- 增加 ISelect.ToChunk 实现分块查询数据，减少数据过大时内存占用；

Examples/orm_vs/Program.cs

@@ -48,6 +48,12 @@ namespace orm_vs
 
         static void Main(string[] args)
         {
+            var testlist1 = fsql.Select<Song>().OrderBy(a => a.Id).ToList();
+            var testlist2 = new List<Song>();
+            fsql.Select<Song>().OrderBy(a => a.Id).ToChunk(0, list =>
+            {
+                testlist2.AddRange(list);
+            });
 
             fsql.CodeFirst.SyncStructure(typeof(Song), typeof(Song_tag), typeof(Tag));
             //sugar.CodeFirst.InitTables(typeof(Song), typeof(Song_tag), typeof(Tag));

FreeSql/FreeSql.xml

@@ -787,6 +787,14 @@
             <param name="includeNestedMembers">false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据</param>
             <returns></returns>
         </member>
+        <member name="M:FreeSql.ISelect0`2.ToChunk(System.Int32,System.Action{System.Collections.Generic.List{`1}},System.Boolean)">
+            <summary>
+            执行SQL查询，分块返回数据，可减少内存开销。比如读取10万条数据，每次返回100条处理。
+            </summary>
+            <param name="size">数据块的大小</param>
+            <param name="done">处理数据块</param>
+            <param name="includeNestedMembers">false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据</param>
+        </member>
         <member name="M:FreeSql.ISelect0`2.ToList``1(System.String)">
             <summary>
             执行SQL查询，返回 field 指定字段的记录，并以元组或基础类型(int,string,long)接收，记录不存在时返回 Count 为 0 的列表

FreeSql/Interface/Curd/ISelect/ISelect0.cs

@@ -46,6 +46,13 @@ namespace FreeSql
         List<T1> ToList(bool includeNestedMembers = false);
         Task<List<T1>> ToListAsync(bool includeNestedMembers = false);
         /// <summary>
+        /// 执行SQL查询，分块返回数据，可减少内存开销。比如读取10万条数据，每次返回100条处理。
+        /// </summary>
+        /// <param name="size">数据块的大小</param>
+        /// <param name="done">处理数据块</param>
+        /// <param name="includeNestedMembers">false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据</param>
+        void ToChunk(int size, Action<List<T1>> done, bool includeNestedMembers = false);
+        /// <summary>
         /// 执行SQL查询，返回 field 指定字段的记录，并以元组或基础类型(int,string,long)接收，记录不存在时返回 Count 为 0 的列表
         /// </summary>
         /// <typeparam name="TTuple"></typeparam>

FreeSql/Internal/CommonProvider/SelectProvider/Select0Provider.cs

@@ -472,6 +472,84 @@ namespace FreeSql.Internal.CommonProvider
 
             return ToListAfPrivateAsync(sql, af, otherData);
         }
+        #region ToChunk
+        internal void ToListAfChunkPrivate(int chunkSize, Action<List<T1>> chunkDone, string sql, GetAllFieldExpressionTreeInfo af, (string field, ReadAnonymousTypeInfo read, List<object> retlist)[] otherData)
+        {
+            var dbParms = _params.ToArray();
+            var before = new Aop.CurdBeforeEventArgs(_tables[0].Table.Type, Aop.CurdType.Select, sql, dbParms);
+            _orm.Aop.CurdBefore?.Invoke(this, before);
+            var ret = new List<T1>();
+            var retCount = 0;
+            Exception exception = null;
+            var checkDoneTimes = 0;
+            try
+            {
+                _orm.Ado.ExecuteReader(_connection, _transaction, dr =>
+                {
+                    ret.Add(af.Read(_orm, dr));
+                    retCount++;
+                    if (otherData != null)
+                    {
+                        var idx = af.FieldCount - 1;
+                        foreach (var other in otherData)
+                            other.retlist.Add(_commonExpression.ReadAnonymous(other.read, dr, ref idx, false));
+                    }
+                    if (chunkSize > 0 && chunkSize == ret.Count)
+                    {
+                        checkDoneTimes++;
+
+                        foreach (var include in _includeToList) include?.Invoke(ret);
+                        _orm.Aop.ToList?.Invoke(this, new Aop.ToListEventArgs(ret));
+                        _trackToList?.Invoke(ret);
+                        chunkDone(ret);
+                        
+
+                        ret.Clear();
+                        if (otherData != null)
+                            foreach (var other in otherData)
+                                other.retlist.Clear();
+                    }
+                }, CommandType.Text, sql, dbParms);
+            }
+            catch (Exception ex)
+            {
+                exception = ex;
+                throw ex;
+            }
+            finally
+            {
+                var after = new Aop.CurdAfterEventArgs(before, exception, retCount);
+                _orm.Aop.CurdAfter?.Invoke(this, after);
+            }
+            if (ret.Any() || checkDoneTimes == 0)
+            {
+                foreach (var include in _includeToList) include?.Invoke(ret);
+                _orm.Aop.ToList?.Invoke(this, new Aop.ToListEventArgs(ret));
+                _trackToList?.Invoke(ret);
+                chunkDone(ret);
+            }
+        }
+        internal void ToListChunkPrivate(int chunkSize, Action<List<T1>> chunkDone, GetAllFieldExpressionTreeInfo af, (string field, ReadAnonymousTypeInfo read, List<object> retlist)[] otherData)
+        {
+            string sql = null;
+            if (otherData?.Length > 0)
+            {
+                var sbField = new StringBuilder().Append(af.Field);
+                foreach (var other in otherData)
+                    sbField.Append(other.field);
+                sql = this.ToSql(sbField.ToString());
+            }
+            else
+                sql = this.ToSql(af.Field);
+
+            ToListAfChunkPrivate(chunkSize, chunkDone, sql, af, otherData);
+        }
+        public void ToChunk(int size, Action<List<T1>> done, bool includeNestedMembers = false)
+        {
+            if (_selectExpression != null) throw new ArgumentException("Chunk 功能之前不可使用 Select");
+            this.ToListChunkPrivate(size, done, includeNestedMembers == false ? this.GetAllFieldExpressionTreeLevel2() : this.GetAllFieldExpressionTreeLevelAll(), null);
+        }
+        #endregion
         public virtual List<T1> ToList(bool includeNestedMembers = false)
         {
             if (_selectExpression != null) return this.InternalToList<T1>(_selectExpression);