tk/FreeSql
1
0
Fork 0
mirror of https://github.com/nsnail/FreeSql.git synced 2025-11-04 09:15:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

- 增加 ISelect.ToChunk 实现分块查询数据，减少数据过大时内存占用；

Browse Source
This commit is contained in:
28810
2019-09-10 16:01:01 +08:00
parent e0a23accb0
commit 564e1951d8
4 changed files with 99 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
FreeSql/FreeSql.xml
View File

@@ -787,6 +787,14 @@
<param name="includeNestedMembers">false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据</param>
<returns></returns>
</member>
<member name="M:FreeSql.ISelect0`2.ToChunk(System.Int32,System.Action{System.Collections.Generic.List{`1}},System.Boolean)">
<summary>
执行SQL查询，分块返回数据，可减少内存开销。比如读取10万条数据，每次返回100条处理。
</summary>
<param name="size">数据块的大小</param>
<param name="done">处理数据块</param>
<param name="includeNestedMembers">false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据</param>
</member>
<member name="M:FreeSql.ISelect0`2.ToList``1(System.String)">
<summary>
执行SQL查询，返回 field 指定字段的记录，并以元组或基础类型(int,string,long)接收，记录不存在时返回 Count 为 0 的列表

7
FreeSql/Interface/Curd/ISelect/ISelect0.cs
View File

@@ -46,6 +46,13 @@ namespace FreeSql
List<T1> ToList(bool includeNestedMembers = false);
Task<List<T1>> ToListAsync(bool includeNestedMembers = false);
/// <summary>
/// 执行SQL查询，分块返回数据，可减少内存开销。比如读取10万条数据，每次返回100条处理。
/// </summary>
/// <param name="size">数据块的大小</param>
/// <param name="done">处理数据块</param>
/// <param name="includeNestedMembers">false: 返回 2级 LeftJoin/InnerJoin/RightJoin 对象；true: 返回所有 LeftJoin/InnerJoin/RightJoin 的导航数据</param>
void ToChunk(int size, Action<List<T1>> done, bool includeNestedMembers = false);
/// <summary>
/// 执行SQL查询，返回 field 指定字段的记录，并以元组或基础类型(int,string,long)接收，记录不存在时返回 Count 为 0 的列表
/// </summary>
/// <typeparam name="TTuple"></typeparam>

78
FreeSql/Internal/CommonProvider/SelectProvider/Select0Provider.cs
View File

@@ -472,6 +472,84 @@ namespace FreeSql.Internal.CommonProvider
return ToListAfPrivateAsync(sql, af, otherData);
}
#region ToChunk
internal void ToListAfChunkPrivate(int chunkSize, Action<List<T1>> chunkDone, string sql, GetAllFieldExpressionTreeInfo af, (string field, ReadAnonymousTypeInfo read, List<object> retlist)[] otherData)
{
var dbParms = _params.ToArray();
var before = new Aop.CurdBeforeEventArgs(_tables[0].Table.Type, Aop.CurdType.Select, sql, dbParms);
_orm.Aop.CurdBefore?.Invoke(this, before);
var ret = new List<T1>();
var retCount = 0;
Exception exception = null;
var checkDoneTimes = 0;
try
{
_orm.Ado.ExecuteReader(_connection, _transaction, dr =>
{
ret.Add(af.Read(_orm, dr));
retCount++;
if (otherData != null)
{
var idx = af.FieldCount - 1;
foreach (var other in otherData)
other.retlist.Add(_commonExpression.ReadAnonymous(other.read, dr, ref idx, false));
}
if (chunkSize > 0 && chunkSize == ret.Count)
{
checkDoneTimes++;
foreach (var include in _includeToList) include?.Invoke(ret);
_orm.Aop.ToList?.Invoke(this, new Aop.ToListEventArgs(ret));
_trackToList?.Invoke(ret);
chunkDone(ret);
ret.Clear();
if (otherData != null)
foreach (var other in otherData)
other.retlist.Clear();
}
}, CommandType.Text, sql, dbParms);
}
catch (Exception ex)
{
exception = ex;
throw ex;
}
finally
{
var after = new Aop.CurdAfterEventArgs(before, exception, retCount);
_orm.Aop.CurdAfter?.Invoke(this, after);
}
if (ret.Any() || checkDoneTimes == 0)
{
foreach (var include in _includeToList) include?.Invoke(ret);
_orm.Aop.ToList?.Invoke(this, new Aop.ToListEventArgs(ret));
_trackToList?.Invoke(ret);
chunkDone(ret);
}
}
internal void ToListChunkPrivate(int chunkSize, Action<List<T1>> chunkDone, GetAllFieldExpressionTreeInfo af, (string field, ReadAnonymousTypeInfo read, List<object> retlist)[] otherData)
{
string sql = null;
if (otherData?.Length > 0)
{
var sbField = new StringBuilder().Append(af.Field);
foreach (var other in otherData)
sbField.Append(other.field);
sql = this.ToSql(sbField.ToString());
}
else
sql = this.ToSql(af.Field);
ToListAfChunkPrivate(chunkSize, chunkDone, sql, af, otherData);
}
public void ToChunk(int size, Action<List<T1>> done, bool includeNestedMembers = false)
{
if (_selectExpression != null) throw new ArgumentException("Chunk 功能之前不可使用 Select");
this.ToListChunkPrivate(size, done, includeNestedMembers == false ? this.GetAllFieldExpressionTreeLevel2() : this.GetAllFieldExpressionTreeLevelAll(), null);
}
#endregion
public virtual List<T1> ToList(bool includeNestedMembers = false)
{
if (_selectExpression != null) return this.InternalToList<T1>(_selectExpression);