更改批量插入数据的方式(测试100w条5秒不到),更改数据修改时的替换可能替换到数据的问题

This commit is contained in:
ChenBo 2021-11-28 21:37:10 +08:00
parent 4c7e04376f
commit 846c180191
4 changed files with 104 additions and 18 deletions

View File

@ -538,14 +538,5 @@
<param name="that"></param> <param name="that"></param>
<returns></returns> <returns></returns>
</member> </member>
<member name="M:Microsoft.Extensions.DependencyInjection.FreeSqlRepositoryDependencyInjection.AddFreeRepository(Microsoft.Extensions.DependencyInjection.IServiceCollection,System.Action{FreeSql.FluentDataFilter},System.Reflection.Assembly[])">
<summary>
批量注入 Repository可以参考代码自行调整
</summary>
<param name="services"></param>
<param name="globalDataFilter"></param>
<param name="assemblies"></param>
<returns></returns>
</member>
</members> </members>
</doc> </doc>

View File

@ -3,6 +3,9 @@ using System.Collections.Generic;
using System.Reflection; using System.Reflection;
using System.Text; using System.Text;
using Xunit; using Xunit;
using System.Linq;
using System.Collections;
using System.Diagnostics;
namespace FreeSql.Tests.MySql namespace FreeSql.Tests.MySql
{ {
@ -32,13 +35,18 @@ namespace FreeSql.Tests.MySql
public int? Points { get; set; } public int? Points { get; set; }
} }
[FreeSql.DataAnnotations.Table(Name = "ClickHouseTest")] [FreeSql.DataAnnotations.Table(Name = "ClickHouseTest")]
public class TestClickHouse public class TestClickHouse : IEnumerable
{ {
[FreeSql.DataAnnotations.Column(IsPrimary = true)] [FreeSql.DataAnnotations.Column(IsPrimary = true)]
[Now] [Now]
public long Id { get; set; } public long Id { get; set; }
public string Name { get; set; } public string Name { get; set; }
public IEnumerator GetEnumerator()
{
yield return Id;
yield return Name;
}
} }
class NowAttribute: Attribute { } class NowAttribute: Attribute { }
@ -73,9 +81,10 @@ namespace FreeSql.Tests.MySql
[Fact] [Fact]
public void TestInsert() public void TestInsert()
{ {
Stopwatch stopwatch =new Stopwatch();
var fsql = g.clickHouse; var fsql = g.clickHouse;
List<TestClickHouse> list=new List<TestClickHouse>(); List<TestClickHouse> list=new List<TestClickHouse>();
for (int i = 0; i < 1000; i++) for (int i = 0; i < 1000000; i++)
{ {
list.Add(new TestClickHouse() list.Add(new TestClickHouse()
{ {
@ -83,9 +92,13 @@ namespace FreeSql.Tests.MySql
Name = $"测试{i}" Name = $"测试{i}"
}); });
} }
fsql.Delete<TestClickHouse>().Where(t => 1 == 1).ExecuteAffrows();
stopwatch.Start();
fsql.Insert(list).ExecuteAffrows(); fsql.Insert(list).ExecuteAffrows();
var items = fsql.Select<TestClickHouse>().Where(o=>o.Id>900).OrderByDescending(o=>o.Id).ToList(); stopwatch.Stop();
Assert.Equal(100, items.Count); Debug.WriteLine(list.Count+"条用时:" +stopwatch.ElapsedMilliseconds.ToString());
//var items = fsql.Select<TestClickHouse>().Where(o=>o.Id>900).OrderByDescending(o=>o.Id).ToList();
//Assert.Equal(100, items.Count);
} }
[Fact] [Fact]

View File

@ -1,9 +1,12 @@
using FreeSql.Internal; using ClickHouse.Client.ADO;
using ClickHouse.Client.Copy;
using FreeSql.Internal;
using FreeSql.Internal.Model; using FreeSql.Internal.Model;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Data; using System.Data;
using System.Data.Common; using System.Data.Common;
using System.Linq;
using System.Text; using System.Text;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -30,9 +33,9 @@ namespace FreeSql.ClickHouse.Curd
internal Dictionary<string, bool> InternalIgnore => _ignore; internal Dictionary<string, bool> InternalIgnore => _ignore;
internal void InternalClearData() => ClearData(); internal void InternalClearData() => ClearData();
public override int ExecuteAffrows() => base.SplitExecuteAffrows(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000); public override int ExecuteAffrows() => base.SplitExecuteAffrows(int.MaxValue, int.MaxValue);
public override long ExecuteIdentity() => base.SplitExecuteIdentity(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000); public override long ExecuteIdentity() => base.SplitExecuteIdentity(int.MaxValue, int.MaxValue);
public override List<T1> ExecuteInserted() => base.SplitExecuteInserted(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000); public override List<T1> ExecuteInserted() => base.SplitExecuteInserted(int.MaxValue, int.MaxValue);
public override string ToSql() public override string ToSql()
@ -42,6 +45,81 @@ namespace FreeSql.ClickHouse.Curd
return $"INSERT IGNORE INTO {sql.Substring(12)}"; return $"INSERT IGNORE INTO {sql.Substring(12)}";
} }
protected override int RawExecuteAffrows()
{
var affrows = 0;
Exception exception = null;
Aop.CurdBeforeEventArgs before=null;
if (_source.Count>1)
{
try
{
before = new Aop.CurdBeforeEventArgs(_table.Type, _table, Aop.CurdType.Insert, null, _params);
_orm.Aop.CurdBeforeHandler?.Invoke(this, before);
using var bulkCopyInterface = new ClickHouseBulkCopy(_orm.Ado.MasterPool.Get().Value as ClickHouseConnection)
{
DestinationTableName = _table.DbName,
BatchSize = _source.Count
};
bulkCopyInterface.WriteToServerAsync(ToDataTable(),default).Wait();
return affrows;
}
catch (Exception ex)
{
exception = ex;
throw ex;
}
finally
{
var after = new Aop.CurdAfterEventArgs(before, exception, affrows);
_orm.Aop.CurdAfterHandler?.Invoke(this, after);
}
}
else
{
var sql = this.ToSql();
before = new Aop.CurdBeforeEventArgs(_table.Type, _table, Aop.CurdType.Insert, sql, _params);
_orm.Aop.CurdBeforeHandler?.Invoke(this, before);
try
{
affrows = _orm.Ado.ExecuteNonQuery(_connection, _transaction, CommandType.Text, sql, _commandTimeout, _params);
}
catch (Exception ex)
{
exception = ex;
throw ex;
}
finally
{
var after = new Aop.CurdAfterEventArgs(before, exception, affrows);
_orm.Aop.CurdAfterHandler?.Invoke(this, after);
}
return affrows;
}
}
private IDictionary<string, object> GetValue<T>(T u, System.Reflection.PropertyInfo[] columns)
{
try
{
Dictionary<string, object> dic = new Dictionary<string, object>();
foreach (var item in columns)
{
object v = null;
if (u != null)
{
v = item.GetValue(u);
}
dic.TryAdd(item.Name, v);
}
return dic;
}
catch (Exception e)
{
throw;
}
}
protected override long RawExecuteIdentity() protected override long RawExecuteIdentity()
{ {
var sql = this.ToSql(); var sql = this.ToSql();

View File

@ -24,7 +24,11 @@ namespace FreeSql.ClickHouse.Curd
internal Dictionary<string, bool> InternalIgnore => _ignore; internal Dictionary<string, bool> InternalIgnore => _ignore;
public override string ToSql() public override string ToSql()
{ {
return base.ToSql()?.Replace("UPDATE", "ALTER TABLE").Replace("SET", "UPDATE"); var sql = base.ToSql();
sql = sql.Remove(0, 6).Insert(0, "ALTER TABLE");
var index = sql.IndexOf(" SET ");
sql = sql.Remove(index, 5).Insert(index, " UPDATE ");
return sql;
} }
internal void InternalResetSource(List<T1> source) => _source = source; internal void InternalResetSource(List<T1> source) => _source = source;
internal string InternalWhereCaseSource(string CsName, Func<string, string> thenValue) => WhereCaseSource(CsName, thenValue); internal string InternalWhereCaseSource(string CsName, Func<string, string> thenValue) => WhereCaseSource(CsName, thenValue);