tk/FreeSql
1
0
Fork 0
mirror of https://github.com/nsnail/FreeSql.git synced 2025-04-22 18:52:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

测试Array常用查询函数

Browse Source
This commit is contained in:
d4ilys 2023-11-30 15:48:59 +08:00
parent 362d39bdb5
commit 57b0347c94
6 changed files with 416 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

247
FreeSql.Tests/FreeSql.Tests/ClickHouse/ClickHouseTest3.cs
View File

File diff suppressed because one or more lines are too long

72
FreeSql.Tests/FreeSql.Tests/FreeSql.Tests.xml
View File

@ -44,6 +44,78 @@
测试Array类型映射
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectAnySync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectLengthSync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectContainsSync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectConcatSync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArrayFilterFuncTest">
<summary>
测试ArrayFilter测试
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.IsPrimaryTest">
<summary>
测试ArrayFilter测试
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.UriStringIsTooLongTest">
<summary>
https://github.com/dotnetcore/FreeSql/issues/969
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBulkCopySingle">
<summary>
测试BulkCopy单条
</summary>
<returns></returns>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBulkCopyMany">
<summary>
测试BulkCopy多条
</summary>
<returns></returns>
</member>
<member name="T:FreeSql.Tests.ClickHouse.HttpContextRecord">
<summary>
Http请求信息统计
</summary>
</member>
<member name="P:FreeSql.Tests.ClickHouse.HttpContextRecord.RequestTotalKey">
<summary>
请求模板
</summary>
</member>
<member name="P:FreeSql.Tests.ClickHouse.HttpContextRecord.Total">
<summary>
请求量
</summary>
</member>
<member name="P:FreeSql.Tests.ClickHouse.HttpContextRecord.Type">
<summary>
记录请求类型
</summary>
</member>
<member name="P:FreeSql.Tests.ClickHouse.HttpContextRecord.AddTime">
<summary>
添加时间
</summary>
</member>
<member name="T:FreeSql.Tests.ClickHouse.CollectDataEntityUpdate01">
<summary>
实时数据

6
Providers/FreeSql.Provider.ClickHouse/ClickHouseCodeFirst.cs
View File

@ -583,9 +583,13 @@ where a.database in ({0}) and a.table in ({1})", tboldname ?? tbname);
if (isPrimary)
{
if (dbType.Contains("Nullable"))
return dbType.Replace("Nullable(", "")
{
var res = dbType.Replace("Nullable(", "")
.Replace(")", "")
.Replace(" NOT NULL", "");
return res;
}
return dbType;
}

40
Providers/FreeSql.Provider.ClickHouse/ClickHouseExpression.cs
View File

@ -5,6 +5,7 @@ using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
@ -121,27 +122,51 @@ namespace FreeSql.ClickHouse
case "First":
case "FirstOrDefault":
return $"substr({getExp(callExp.Arguments[0])}, 1, 1)";
}
}
}
if (objType == null) objType = callExp.Method.DeclaringType;
if (objType != null || objType.IsArrayOrList())
{
if (argIndex >= callExp.Arguments.Count) break;
//if (argIndex >= callExp.Arguments.Count) break;
//tsc.SetMapColumnTmp(null);
//var args1 = getExp(callExp.Arguments[argIndex]);
//var oldMapType = tsc.SetMapTypeReturnOld(tsc.mapTypeTmp);
//var oldDbParams = objExp?.NodeType == ExpressionType.MemberAccess ? tsc.SetDbParamsReturnOld(null) : null; //#900 UseGenerateCommandParameterWithLambda(true) 子查询 bug、以及 #1173 参数化 bug
//tsc.isNotSetMapColumnTmp = true;
var left = objExp == null ? null : getExp(objExp);
//tsc.isNotSetMapColumnTmp = false;
//tsc.SetMapColumnTmp(null).SetMapTypeReturnOld(oldMapType);
//if (oldDbParams != null) tsc.SetDbParamsReturnOld(oldDbParams);
switch (callExp.Method.Name)
{
case "Count":
left = objExp == null ? null : getExp(objExp);
if (left.StartsWith("(") || left.EndsWith(")")) left = $"array[{left.TrimStart('(').TrimEnd(')')}]";
return $"(case when {left} is null then 0 else length({left}) end)";
case "Any":
left = objExp == null ? null : getExp(objExp);
if (left.StartsWith("(") || left.EndsWith(")")) left = $"array[{left.TrimStart('(').TrimEnd(')')}]";
return $"(case when {left} is null then 0 else length({left}) end > 0)";
case "Contains":
tsc.SetMapColumnTmp(null);
var args1 = getExp(callExp.Arguments[argIndex]);
var oldMapType = tsc.SetMapTypeReturnOld(tsc.mapTypeTmp);
var oldDbParams = objExp?.NodeType == ExpressionType.MemberAccess ? tsc.SetDbParamsReturnOld(null) : null; //#900 UseGenerateCommandParameterWithLambda(true) 子查询 bug、以及 #1173 参数化 bug
tsc.isNotSetMapColumnTmp = true;
var left = objExp == null ? null : getExp(objExp);
left = objExp == null ? null : getExp(objExp);
tsc.isNotSetMapColumnTmp = false;
tsc.SetMapColumnTmp(null).SetMapTypeReturnOld(oldMapType);
if (oldDbParams != null) tsc.SetDbParamsReturnOld(oldDbParams);
switch (callExp.Method.Name)
{
case "Contains":
//判断 in //在各大 Provider AdoProvider 中已约定，500元素分割, 3空格\r\n4空格
return $"(({args1}) in {left.Replace(", \r\n \r\n", $") \r\n OR ({args1}) in (")})";
if (left.StartsWith("(") || left.EndsWith(")")) left = $"array[{left.TrimStart('(').TrimEnd(')')}]";
return $"(hasAny({left}, [{args1}]))";
case "Concat":
left = objExp == null ? null : getExp(objExp);
if (left.StartsWith("(") || left.EndsWith(")")) left = $"array[{left.TrimStart('(').TrimEnd(')')}]";
var right2 = getExp(callExp.Arguments[argIndex]);
if (right2.StartsWith("(") || right2.EndsWith(")")) right2 = $"array[{right2.TrimStart('(').TrimEnd(')')}]";
return $"(arrayConcat({left} || {right2}))";
}
}
break;
@ -181,7 +206,6 @@ namespace FreeSql.ClickHouse
}
return null;
}
public override string ExpressionLambdaToSqlMemberAccessString(MemberExpression exp, ExpTSC tsc)
{
if (exp.Expression == null)

45
Providers/FreeSql.Provider.ClickHouse/ClickHouseExtensions.cs
View File

@ -2,24 +2,30 @@
using FreeSql.ClickHouse.Curd;
using FreeSql.Internal.CommonProvider;
using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Threading;
using System.Threading.Tasks;
using FreeSql.DataAnnotations;
public static partial class FreeSqlClickHouseGlobalExtensions
{
/// <summary>
/// 特殊处理类似 string.Format 的使用方法，防止注入，以及 IS NULL 转换
/// </summary>
/// <param name="that"></param>
/// <param name="args"></param>
/// <returns></returns>
public static string FormatClickHouse(this string that, params object[] args) => _clickHouseAdo.Addslashes(that, args);
public static string FormatClickHouse(this string that, params object[] args) =>
_clickHouseAdo.Addslashes(that, args);
static FreeSql.ClickHouse.ClickHouseAdo _clickHouseAdo = new FreeSql.ClickHouse.ClickHouseAdo();
/// <summary>
/// Clickhouse limit by
/// </summary>
public static ISelect<T> LimitBy<T>(this ISelect<T> that, Expression<Func<T, object>> selector, int limit, int offset = 0)
public static ISelect<T> LimitBy<T>(this ISelect<T> that, Expression<Func<T, object>> selector, int limit,
int offset = 0)
{
if (limit <= 0 && offset <= 0) return that;
var s0p = that as ClickHouseSelect<T>;
@ -34,6 +40,7 @@ public static partial class FreeSqlClickHouseGlobalExtensions
{
s0p._orderby = oldOrderBy;
}
return that;
}
@ -52,4 +59,36 @@ public static partial class FreeSqlClickHouseGlobalExtensions
s0p._sample = $"SAMPLE {k}";
return that;
}
/// <summary>
/// 批量快速插入
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="that"></param>
/// <returns></returns>
public static async Task<int> ExecuteBulkCopyAsync<T>(this IInsert<T> that) where T : class
{
try
{
var insert = that as ClickHouseInsert<T>;
await insert.InternalBulkCopyAsync();
}
catch (Exception e)
{
throw e;
}
return 0;
}
/// <summary>
/// 批量快速插入
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="insert"></param>
/// <returns></returns>
public static int ExecuteBulkCopy<T>(this IInsert<T> insert) where T : class
{
return ExecuteBulkCopyAsync(insert).ConfigureAwait(false).GetAwaiter().GetResult();
}
}

31
Providers/FreeSql.Provider.ClickHouse/Curd/ClickHouseInsert.cs
View File

@ -57,18 +57,7 @@ namespace FreeSql.ClickHouse.Curd
{
before = new Aop.CurdBeforeEventArgs(_table.Type, _table, Aop.CurdType.Insert, null, _params);
_orm.Aop.CurdBeforeHandler?.Invoke(this, before);
var data = ToDataTable();
using (var conn = _orm.Ado.MasterPool.Get())
{
using (var bulkCopyInterface = new ClickHouseBulkCopy(conn.Value as ClickHouseConnection)
{
DestinationTableName = data.TableName,
BatchSize = _source.Count
})
{
bulkCopyInterface.WriteToServerAsync(data, default).Wait();
}
}
InternalBulkCopyAsync().ConfigureAwait(false).GetAwaiter().GetResult();
return affrows;
}
catch (Exception ex)
@ -85,6 +74,24 @@ namespace FreeSql.ClickHouse.Curd
return base.RawExecuteAffrows();
}
internal async Task<int> InternalBulkCopyAsync()
{
var data = ToDataTable();
using (var conn = _orm.Ado.MasterPool.Get())
{
using (var bulkCopyInterface = new ClickHouseBulkCopy(conn.Value as ClickHouseConnection)
{
DestinationTableName = data.TableName,
BatchSize = _source.Count
})
{
await bulkCopyInterface.WriteToServerAsync(data, default);
}
}
return 0;
}
private IDictionary<string, object> GetValue<T>(T u, System.Reflection.PropertyInfo[] columns)
{
try