tk/FreeSql
1
0
Fork 0
mirror of https://github.com/nsnail/FreeSql.git synced 2025-04-22 18:52:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1676 from d4ilys/master

Browse Source 
Clickhouse - Array/Bool数据类型适配
This commit is contained in:
2881099 2023-11-30 16:12:48 +08:00 committed by GitHub
commit 175011a419
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 783 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
FreeSql.Tests/FreeSql.Tests/ClickHouse/ClickHouseTest2.cs
View File

@ -8,11 +8,14 @@ using System.Diagnostics;
using System.ComponentModel.DataAnnotations;
using FreeSql.DataAnnotations;
using Xunit;
using Xunit.Abstractions;
namespace FreeSql.Tests.ClickHouse
{
public class ClickHouseTest2
{
private static IFreeSql fsql = new FreeSqlBuilder().UseConnectionString(DataType.ClickHouse,
"Host=127.0.0.1;Port=8123;Database=test;Compress=True;Min Pool Size=1")
.UseMonitorCommand(cmd => Console.WriteLine($"线程：{cmd.CommandText}\r\n"))
@ -29,6 +32,7 @@ namespace FreeSql.Tests.ClickHouse
{
fsql.CodeFirst.SyncStructure(typeof(PositionInfoModel));
}
[Fact]
public void Issuse1587TestOnePrimary()
{

456
FreeSql.Tests/FreeSql.Tests/ClickHouse/ClickHouseTest3.cs Normal file
View File

File diff suppressed because one or more lines are too long

122
FreeSql.Tests/FreeSql.Tests/FreeSql.Tests.xml
View File

@ -4,6 +4,128 @@
<name>FreeSql.Tests</name>
</assembly>
<members>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBoolMappingSync">
<summary>
测试bool类型映射
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBoolMappingInsert">
<summary>
测试bool类型插入
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBoolMappingUpdateSet">
<summary>
测试bool类型修改
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBoolMappingUpdate">
<summary>
测试bool类型修改
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBoolMappingSelect">
<summary>
测试bool类型查询
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArrayBoolMappingSync">
<summary>
测试Array类型映射
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArrayBoolMappingInsert">
<summary>
测试Array类型插入
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArrayBoolMappingSelect">
<summary>
测试Array类型映射
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectAnySync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectLengthSync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectContainsSync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectConcatSync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectConstContainsSync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArraySelectConstLengthSync">
<summary>
测试Array常用查询函数
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.ArrayFilterFuncTest">
<summary>
测试ArrayFilter测试
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.IsPrimaryTest">
<summary>
测试ArrayFilter测试
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.UriStringIsTooLongTest">
<summary>
https://github.com/dotnetcore/FreeSql/issues/969
</summary>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBulkCopySingle">
<summary>
测试BulkCopy单条
</summary>
<returns></returns>
</member>
<member name="M:FreeSql.Tests.ClickHouse.ClickHouseTest3.TestBulkCopyMany">
<summary>
测试BulkCopy多条
</summary>
<returns></returns>
</member>
<member name="T:FreeSql.Tests.ClickHouse.HttpContextRecord">
<summary>
Http请求信息统计
</summary>
</member>
<member name="P:FreeSql.Tests.ClickHouse.HttpContextRecord.RequestTotalKey">
<summary>
请求模板
</summary>
</member>
<member name="P:FreeSql.Tests.ClickHouse.HttpContextRecord.Total">
<summary>
请求量
</summary>
</member>
<member name="P:FreeSql.Tests.ClickHouse.HttpContextRecord.Type">
<summary>
记录请求类型
</summary>
</member>
<member name="P:FreeSql.Tests.ClickHouse.HttpContextRecord.AddTime">
<summary>
添加时间
</summary>
</member>
<member name="T:FreeSql.Tests.ClickHouse.CollectDataEntityUpdate01">
<summary>
实时数据

2
FreeSql/Internal/CommonProvider/AdoProvider/AdoProviderUtils.cs
View File

@ -43,7 +43,7 @@ namespace FreeSql.Internal.CommonProvider
sb.Append(AddslashesProcessParam(z, mapType, mapColumn));
}
return sb.Length == 0 ? "(NULL)" : sb.Remove(0, 1).Insert(0, "(").Append(")").ToString();
return sb.Length == 0 ? "(NULL)" : sb.Remove(0, 1).Insert(0, "[").Append("]").ToString();
}
public static bool IsFromSlave(string cmdText)

2
Providers/FreeSql.Provider.ClickHouse/ClickHouseAdo/ClickHouseAdo.cs
View File

@ -48,7 +48,7 @@ namespace FreeSql.ClickHouse
param = Utils.GetDataReaderValue(mapType, param);
if (param is bool || param is bool?)
return (bool)param ? 1 : 0;
return (bool)param; //不需要转0/1
else if (param is string)
return string.Concat("'", param.ToString().Replace("'", "''").Replace("\\", "\\\\"), "'"); //只有 mysql 需要处理反斜杠
else if (param is char)

113
Providers/FreeSql.Provider.ClickHouse/ClickHouseCodeFirst.cs
View File

@ -1,6 +1,7 @@
using FreeSql.Internal;
using FreeSql.Internal.Model;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Linq;
@ -23,8 +24,8 @@ namespace FreeSql.ClickHouse
static Dictionary<string, CsToDb<DbType>> _dicCsToDb = new Dictionary<string, CsToDb<DbType>>()
{
{ typeof(bool).FullName, CsToDb.New(DbType.SByte, "Int8", "Int8", null, false, false) },
{ typeof(bool?).FullName, CsToDb.New(DbType.SByte, "Int8", "Nullable(Int8)", null, true, null) },
{ typeof(bool).FullName, CsToDb.New(DbType.SByte, "Bool", "Bool", null, false, false) },
{ typeof(bool?).FullName, CsToDb.New(DbType.SByte, "Bool", "Nullable(Bool)", null, true, null) },
{ typeof(sbyte).FullName, CsToDb.New(DbType.SByte, "Int8", "Int8", false, false, 0) },
{ typeof(sbyte?).FullName, CsToDb.New(DbType.SByte, "Int8", "Nullable(Int8)", false, true, null) },
@ -50,7 +51,8 @@ namespace FreeSql.ClickHouse
{ typeof(float?).FullName, CsToDb.New(DbType.Single, "Float32", "Nullable(Float32)", false, true, null) },
{
typeof(decimal).FullName,
CsToDb.New(DbType.Decimal, "Decimal(38, 19)", "Decimal(38, 19)", false, false, 0) //Nullable(Decimal(38, 19))
CsToDb.New(DbType.Decimal, "Decimal(38, 19)", "Decimal(38, 19)", false, false,
0) //Nullable(Decimal(38, 19))
},
{
typeof(decimal?).FullName,
@ -75,17 +77,84 @@ namespace FreeSql.ClickHouse
{ typeof(Guid?).FullName, CsToDb.New(DbType.String, "String", "Nullable(String)", false, true, null) },
};
public override DbInfoResult GetDbInfo(Type type)
{
if (_dicCsToDb.TryGetValue(type.FullName, out var trydc))
return new DbInfoResult((int)trydc.type, trydc.dbtype, trydc.dbtypeFull, trydc.isnullable,
trydc.defaultValue);
if (type.IsArray)
return null;
//判断是否是集合
var isCollection = IsArray(type);
if (isCollection.Item1)
{
var genericType = isCollection.Item2;
var genericTypeName = genericType?.FullName;
var tryGetValue = _dicCsToDb.TryGetValue(genericTypeName, out var value);
if (tryGetValue)
{
var arrayDbType = $"Array({value.dbtype})";
var defaultArray = new ArrayList(0);
return new DbInfoResult(Convert.ToInt32(DbType.Object), arrayDbType, arrayDbType, false,defaultArray);
}
}
return null;
}
protected override string GetComparisonDDLStatements(params TypeSchemaAndName[] objects)
private Tuple<bool, Type> IsArray(Type type)
{
var flag = false;
Type resultType = null;
if (type.IsArray)
{
flag = true;
resultType = type.GetElementType();
}
return new Tuple<bool, Type>(flag, resultType);
}
private Tuple<bool, Type> IsCollection(Type type)
{
var flag = false;
Type resultType = null;
var interfaces = type.GetInterfaces();
if (interfaces.Any(t => t.Name == "IList"))
flag = true;
if (interfaces.Any(t => t.Name == "ICollection"))
flag = true;
if (interfaces.Any(t => t.Name == "IEnumerable"))
flag = true;
if (type.Name == "Array")
{
flag = true;
resultType = typeof(string);
}
if (type.Name == "ArrayList")
{
flag = true;
resultType = typeof(string);
}
//是否是泛型
if (type.GetGenericArguments().Any())
{
var first = type.GetGenericArguments().First();
resultType = first;
}
return new Tuple<bool, Type>(flag, resultType);
}
protected override string GetComparisonDDLStatements(params TypeAndName[] objects)
{
Object<DbConnection> conn = null;
string database = null;
@ -100,11 +169,11 @@ namespace FreeSql.ClickHouse
{
if (sb.Length > 0)
sb.Append("\r\n");
var tb = obj.tableSchema;
var tb = _commonUtils.GetTableByEntity(obj.entityType);
if (tb == null)
throw new Exception(CoreStrings.S_Type_IsNot_Migrable(obj.tableSchema.Type.FullName));
throw new Exception(CoreStrings.S_Type_IsNot_Migrable(obj.entityType.FullName));
if (tb.Columns.Any() == false)
throw new Exception(CoreStrings.S_Type_IsNot_Migrable_0Attributes(obj.tableSchema.Type.FullName));
throw new Exception(CoreStrings.S_Type_IsNot_Migrable_0Attributes(obj.entityType.FullName));
var tbname = _commonUtils.SplitTableName(tb.DbName);
if (tbname?.Length == 1)
tbname = new[] { database, tbname[0] };
@ -299,7 +368,8 @@ where a.database in ({0}) and a.table in ({1})", tboldname ?? tbname);
//添加列
sbalter.Append("ALTER TABLE ").Append(_commonUtils.QuoteSqlName(tbname[0], tbname[1]))
.Append(" ADD Column ").Append(_commonUtils.QuoteSqlName(tbcol.Attribute.Name)).Append(" ")
.Append(" ADD Column ").Append(_commonUtils.QuoteSqlName(tbcol.Attribute.Name))
.Append(" ")
.Append(tbcol.Attribute.DbType);
if (tbcol.Attribute.IsNullable == false && tbcol.DbDefaultValue != "NULL" &&
tbcol.Attribute.IsIdentity == false)
@ -496,7 +566,8 @@ where a.database in ({0}) and a.table in ({1})", tboldname ?? tbname);
}
finally
{
this._orm?.Aop.CommandAfterHandler?.Invoke(this._orm, new Aop.CommandAfterEventArgs(before, afterException, null));
this._orm?.Aop.CommandAfterHandler?.Invoke(this._orm,
new Aop.CommandAfterEventArgs(before, afterException, null));
}
}
}
@ -512,11 +583,16 @@ where a.database in ({0}) and a.table in ({1})", tboldname ?? tbname);
if (isPrimary)
{
if (dbType.Contains("Nullable"))
return dbType.Replace("Nullable(", "")
{
var res = dbType.Replace("Nullable(", "")
.Replace(")", "")
.Replace(" NOT NULL", "");
return res;
}
return dbType;
}
return dbType.Replace(" NOT NULL", "");
}
@ -524,6 +600,9 @@ where a.database in ({0}) and a.table in ({1})", tboldname ?? tbname);
string CkIntAdapter(string dbType)
{
var result = dbType;
if (dbType.Contains("Array"))
return dbType;
if (dbType.ToLower().Contains("int64"))
{
if (dbType.Contains("Nullable"))
@ -578,13 +657,7 @@ where a.database in ({0}) and a.table in ({1})", tboldname ?? tbname);
internal class ClickHouseTableIndex
{
public string name
{
get; set;
}
public string expr
{
get; set;
}
public string name { get; set; }
public string expr { get; set; }
}
}

40
Providers/FreeSql.Provider.ClickHouse/ClickHouseExpression.cs
View File

@ -5,6 +5,7 @@ using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
@ -121,27 +122,51 @@ namespace FreeSql.ClickHouse
case "First":
case "FirstOrDefault":
return $"substr({getExp(callExp.Arguments[0])}, 1, 1)";
}
}
}
if (objType == null) objType = callExp.Method.DeclaringType;
if (objType != null || objType.IsArrayOrList())
{
if (argIndex >= callExp.Arguments.Count) break;
//if (argIndex >= callExp.Arguments.Count) break;
//tsc.SetMapColumnTmp(null);
//var args1 = getExp(callExp.Arguments[argIndex]);
//var oldMapType = tsc.SetMapTypeReturnOld(tsc.mapTypeTmp);
//var oldDbParams = objExp?.NodeType == ExpressionType.MemberAccess ? tsc.SetDbParamsReturnOld(null) : null; //#900 UseGenerateCommandParameterWithLambda(true) 子查询 bug、以及 #1173 参数化 bug
//tsc.isNotSetMapColumnTmp = true;
var left = objExp == null ? null : getExp(objExp);
//tsc.isNotSetMapColumnTmp = false;
//tsc.SetMapColumnTmp(null).SetMapTypeReturnOld(oldMapType);
//if (oldDbParams != null) tsc.SetDbParamsReturnOld(oldDbParams);
switch (callExp.Method.Name)
{
case "Count":
left = objExp == null ? null : getExp(objExp);
if (left.StartsWith("(") || left.EndsWith(")")) left = $"array[{left.TrimStart('(').TrimEnd(')')}]";
return $"(case when {left} is null then 0 else length({left}) end)";
case "Any":
left = objExp == null ? null : getExp(objExp);
if (left.StartsWith("(") || left.EndsWith(")")) left = $"array[{left.TrimStart('(').TrimEnd(')')}]";
return $"(case when {left} is null then 0 else length({left}) end > 0)";
case "Contains":
tsc.SetMapColumnTmp(null);
var args1 = getExp(callExp.Arguments[argIndex]);
var oldMapType = tsc.SetMapTypeReturnOld(tsc.mapTypeTmp);
var oldDbParams = objExp?.NodeType == ExpressionType.MemberAccess ? tsc.SetDbParamsReturnOld(null) : null; //#900 UseGenerateCommandParameterWithLambda(true) 子查询 bug、以及 #1173 参数化 bug
tsc.isNotSetMapColumnTmp = true;
var left = objExp == null ? null : getExp(objExp);
left = objExp == null ? null : getExp(objExp);
tsc.isNotSetMapColumnTmp = false;
tsc.SetMapColumnTmp(null).SetMapTypeReturnOld(oldMapType);
if (oldDbParams != null) tsc.SetDbParamsReturnOld(oldDbParams);
switch (callExp.Method.Name)
{
case "Contains":
//判断 in //在各大 Provider AdoProvider 中已约定，500元素分割, 3空格\r\n4空格
return $"(({args1}) in {left.Replace(", \r\n \r\n", $") \r\n OR ({args1}) in (")})";
if (left.StartsWith("(") || left.EndsWith(")")) left = $"array[{left.TrimStart('(').TrimEnd(')')}]";
return $"(hasAny({left}, [{args1}]))";
case "Concat":
left = objExp == null ? null : getExp(objExp);
if (left.StartsWith("(") || left.EndsWith(")")) left = $"array[{left.TrimStart('(').TrimEnd(')')}]";
var right2 = getExp(callExp.Arguments[argIndex]);
if (right2.StartsWith("(") || right2.EndsWith(")")) right2 = $"array[{right2.TrimStart('(').TrimEnd(')')}]";
return $"(arrayConcat({left} || {right2}))";
}
}
break;
@ -181,7 +206,6 @@ namespace FreeSql.ClickHouse
}
return null;
}
public override string ExpressionLambdaToSqlMemberAccessString(MemberExpression exp, ExpTSC tsc)
{
if (exp.Expression == null)

45
Providers/FreeSql.Provider.ClickHouse/ClickHouseExtensions.cs
View File

@ -2,24 +2,30 @@
using FreeSql.ClickHouse.Curd;
using FreeSql.Internal.CommonProvider;
using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Threading;
using System.Threading.Tasks;
using FreeSql.DataAnnotations;
public static partial class FreeSqlClickHouseGlobalExtensions
{
/// <summary>
/// 特殊处理类似 string.Format 的使用方法，防止注入，以及 IS NULL 转换
/// </summary>
/// <param name="that"></param>
/// <param name="args"></param>
/// <returns></returns>
public static string FormatClickHouse(this string that, params object[] args) => _clickHouseAdo.Addslashes(that, args);
public static string FormatClickHouse(this string that, params object[] args) =>
_clickHouseAdo.Addslashes(that, args);
static FreeSql.ClickHouse.ClickHouseAdo _clickHouseAdo = new FreeSql.ClickHouse.ClickHouseAdo();
/// <summary>
/// Clickhouse limit by
/// </summary>
public static ISelect<T> LimitBy<T>(this ISelect<T> that, Expression<Func<T, object>> selector, int limit, int offset = 0)
public static ISelect<T> LimitBy<T>(this ISelect<T> that, Expression<Func<T, object>> selector, int limit,
int offset = 0)
{
if (limit <= 0 && offset <= 0) return that;
var s0p = that as ClickHouseSelect<T>;
@ -34,6 +40,7 @@ public static partial class FreeSqlClickHouseGlobalExtensions
{
s0p._orderby = oldOrderBy;
}
return that;
}
@ -52,4 +59,36 @@ public static partial class FreeSqlClickHouseGlobalExtensions
s0p._sample = $"SAMPLE {k}";
return that;
}
/// <summary>
/// 批量快速插入
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="that"></param>
/// <returns></returns>
public static async Task<int> ExecuteBulkCopyAsync<T>(this IInsert<T> that) where T : class
{
try
{
var insert = that as ClickHouseInsert<T>;
await insert.InternalBulkCopyAsync();
}
catch (Exception e)
{
throw e;
}
return 0;
}
/// <summary>
/// 批量快速插入
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="insert"></param>
/// <returns></returns>
public static int ExecuteBulkCopy<T>(this IInsert<T> insert) where T : class
{
return ExecuteBulkCopyAsync(insert).ConfigureAwait(false).GetAwaiter().GetResult();
}
}

7
Providers/FreeSql.Provider.ClickHouse/ClickHouseUtils.cs
View File

@ -6,6 +6,7 @@ using System.Collections.Generic;
using System.Data.Common;
using System.Globalization;
using System.Data;
using System.Text.Json;
using ClickHouse.Client.ADO.Parameters;
using System.Text.RegularExpressions;
@ -43,8 +44,10 @@ namespace FreeSql.ClickHouse
if (col.DbScale != 0) ret.Scale = col.DbScale;
break;
}
if (value is bool)
ret.Value = (bool)value ? 1 : 0;
//if (value.GetType().IsArray)
//{
// ret.DbType = DbType.Object;
//}
}
_params?.Add(ret);
return ret;

31
Providers/FreeSql.Provider.ClickHouse/Curd/ClickHouseInsert.cs
View File

@ -57,18 +57,7 @@ namespace FreeSql.ClickHouse.Curd
{
before = new Aop.CurdBeforeEventArgs(_table.Type, _table, Aop.CurdType.Insert, null, _params);
_orm.Aop.CurdBeforeHandler?.Invoke(this, before);
var data = ToDataTable();
using (var conn = _orm.Ado.MasterPool.Get())
{
using (var bulkCopyInterface = new ClickHouseBulkCopy(conn.Value as ClickHouseConnection)
{
DestinationTableName = data.TableName,
BatchSize = _source.Count
})
{
bulkCopyInterface.WriteToServerAsync(data, default).Wait();
}
}
InternalBulkCopyAsync().ConfigureAwait(false).GetAwaiter().GetResult();
return affrows;
}
catch (Exception ex)
@ -85,6 +74,24 @@ namespace FreeSql.ClickHouse.Curd
return base.RawExecuteAffrows();
}
internal async Task<int> InternalBulkCopyAsync()
{
var data = ToDataTable();
using (var conn = _orm.Ado.MasterPool.Get())
{
using (var bulkCopyInterface = new ClickHouseBulkCopy(conn.Value as ClickHouseConnection)
{
DestinationTableName = data.TableName,
BatchSize = _source.Count
})
{
await bulkCopyInterface.WriteToServerAsync(data, default);
}
}
return 0;
}
private IDictionary<string, object> GetValue<T>(T u, System.Reflection.PropertyInfo[] columns)
{
try