From 564e1951d8c5a3bcaf13c1c4a8533beead2099b6 Mon Sep 17 00:00:00 2001 From: 28810 <28810@YEXIANGQIN> Date: Tue, 10 Sep 2019 16:01:01 +0800 Subject: [PATCH] =?UTF-8?q?-=20=E5=A2=9E=E5=8A=A0=20ISelect.ToChunk=20?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E5=88=86=E5=9D=97=E6=9F=A5=E8=AF=A2=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=EF=BC=8C=E5=87=8F=E5=B0=91=E6=95=B0=E6=8D=AE=E8=BF=87?= =?UTF-8?q?=E5=A4=A7=E6=97=B6=E5=86=85=E5=AD=98=E5=8D=A0=E7=94=A8=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Examples/orm_vs/Program.cs | 6 ++ FreeSql/FreeSql.xml | 8 ++ FreeSql/Interface/Curd/ISelect/ISelect0.cs | 7 ++ .../SelectProvider/Select0Provider.cs | 78 +++++++++++++++++++ 4 files changed, 99 insertions(+) diff --git a/Examples/orm_vs/Program.cs b/Examples/orm_vs/Program.cs index 729b9c36..6530eb3e 100644 --- a/Examples/orm_vs/Program.cs +++ b/Examples/orm_vs/Program.cs @@ -48,6 +48,12 @@ namespace orm_vs static void Main(string[] args) { + var testlist1 = fsql.Select().OrderBy(a => a.Id).ToList(); + var testlist2 = new List(); + fsql.Select().OrderBy(a => a.Id).ToChunk(0, list => + { + testlist2.AddRange(list); + }); fsql.CodeFirst.SyncStructure(typeof(Song), typeof(Song_tag), typeof(Tag)); //sugar.CodeFirst.InitTables(typeof(Song), typeof(Song_tag), typeof(Tag)); diff --git a/FreeSql/FreeSql.xml b/FreeSql/FreeSql.xml index aeaf966c..db077ae2 100644 --- a/FreeSql/FreeSql.xml +++ b/FreeSql/FreeSql.xml @@ -787,6 +787,14 @@ false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据 + + + 执行SQL查询，分块返回数据，可减少内存开销。比如读取10万条数据，每次返回100条处理。 + + 数据块的大小 + 处理数据块 + false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据 + 执行SQL查询，返回 field 指定字段的记录，并以元组或基础类型(int,string,long)接收，记录不存在时返回 Count 为 0 的列表 diff --git a/FreeSql/Interface/Curd/ISelect/ISelect0.cs b/FreeSql/Interface/Curd/ISelect/ISelect0.cs index bc10a9ad..eb1773ba 100644 --- a/FreeSql/Interface/Curd/ISelect/ISelect0.cs +++ b/FreeSql/Interface/Curd/ISelect/ISelect0.cs @@ -46,6 +46,13 @@ namespace FreeSql List ToList(bool includeNestedMembers = false); Task> ToListAsync(bool includeNestedMembers = false); /// + /// 执行SQL查询，分块返回数据，可减少内存开销。比如读取10万条数据，每次返回100条处理。 + /// + /// 数据块的大小 + /// 处理数据块 + /// false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据 + void ToChunk(int size, Action> done, bool includeNestedMembers = false); + /// /// 执行SQL查询，返回 field 指定字段的记录，并以元组或基础类型(int,string,long)接收，记录不存在时返回 Count 为 0 的列表 /// /// diff --git a/FreeSql/Internal/CommonProvider/SelectProvider/Select0Provider.cs b/FreeSql/Internal/CommonProvider/SelectProvider/Select0Provider.cs index c087b977..6ad66981 100644 --- a/FreeSql/Internal/CommonProvider/SelectProvider/Select0Provider.cs +++ b/FreeSql/Internal/CommonProvider/SelectProvider/Select0Provider.cs @@ -472,6 +472,84 @@ namespace FreeSql.Internal.CommonProvider return ToListAfPrivateAsync(sql, af, otherData); } + #region ToChunk + internal void ToListAfChunkPrivate(int chunkSize, Action> chunkDone, string sql, GetAllFieldExpressionTreeInfo af, (string field, ReadAnonymousTypeInfo read, List retlist)[] otherData) + { + var dbParms = _params.ToArray(); + var before = new Aop.CurdBeforeEventArgs(_tables[0].Table.Type, Aop.CurdType.Select, sql, dbParms); + _orm.Aop.CurdBefore?.Invoke(this, before); + var ret = new List(); + var retCount = 0; + Exception exception = null; + var checkDoneTimes = 0; + try + { + _orm.Ado.ExecuteReader(_connection, _transaction, dr => + { + ret.Add(af.Read(_orm, dr)); + retCount++; + if (otherData != null) + { + var idx = af.FieldCount - 1; + foreach (var other in otherData) + other.retlist.Add(_commonExpression.ReadAnonymous(other.read, dr, ref idx, false)); + } + if (chunkSize > 0 && chunkSize == ret.Count) + { + checkDoneTimes++; + + foreach (var include in _includeToList) include?.Invoke(ret); + _orm.Aop.ToList?.Invoke(this, new Aop.ToListEventArgs(ret)); + _trackToList?.Invoke(ret); + chunkDone(ret); + + + ret.Clear(); + if (otherData != null) + foreach (var other in otherData) + other.retlist.Clear(); + } + }, CommandType.Text, sql, dbParms); + } + catch (Exception ex) + { + exception = ex; + throw ex; + } + finally + { + var after = new Aop.CurdAfterEventArgs(before, exception, retCount); + _orm.Aop.CurdAfter?.Invoke(this, after); + } + if (ret.Any() || checkDoneTimes == 0) + { + foreach (var include in _includeToList) include?.Invoke(ret); + _orm.Aop.ToList?.Invoke(this, new Aop.ToListEventArgs(ret)); + _trackToList?.Invoke(ret); + chunkDone(ret); + } + } + internal void ToListChunkPrivate(int chunkSize, Action> chunkDone, GetAllFieldExpressionTreeInfo af, (string field, ReadAnonymousTypeInfo read, List retlist)[] otherData) + { + string sql = null; + if (otherData?.Length > 0) + { + var sbField = new StringBuilder().Append(af.Field); + foreach (var other in otherData) + sbField.Append(other.field); + sql = this.ToSql(sbField.ToString()); + } + else + sql = this.ToSql(af.Field); + + ToListAfChunkPrivate(chunkSize, chunkDone, sql, af, otherData); + } + public void ToChunk(int size, Action> done, bool includeNestedMembers = false) + { + if (_selectExpression != null) throw new ArgumentException("Chunk 功能之前不可使用 Select"); + this.ToListChunkPrivate(size, done, includeNestedMembers == false ? this.GetAllFieldExpressionTreeLevel2() : this.GetAllFieldExpressionTreeLevelAll(), null); + } + #endregion public virtual List ToList(bool includeNestedMembers = false) { if (_selectExpression != null) return this.InternalToList(_selectExpression);