mirror of
https://github.com/nsnail/FreeSql.git
synced 2025-04-22 02:32:50 +08:00
更改批量插入数据的方式(测试100w条5秒不到),更改数据修改时的替换可能替换到数据的问题
This commit is contained in:
parent
4c7e04376f
commit
846c180191
@ -538,14 +538,5 @@
|
|||||||
<param name="that"></param>
|
<param name="that"></param>
|
||||||
<returns></returns>
|
<returns></returns>
|
||||||
</member>
|
</member>
|
||||||
<member name="M:Microsoft.Extensions.DependencyInjection.FreeSqlRepositoryDependencyInjection.AddFreeRepository(Microsoft.Extensions.DependencyInjection.IServiceCollection,System.Action{FreeSql.FluentDataFilter},System.Reflection.Assembly[])">
|
|
||||||
<summary>
|
|
||||||
批量注入 Repository,可以参考代码自行调整
|
|
||||||
</summary>
|
|
||||||
<param name="services"></param>
|
|
||||||
<param name="globalDataFilter"></param>
|
|
||||||
<param name="assemblies"></param>
|
|
||||||
<returns></returns>
|
|
||||||
</member>
|
|
||||||
</members>
|
</members>
|
||||||
</doc>
|
</doc>
|
||||||
|
@ -3,6 +3,9 @@ using System.Collections.Generic;
|
|||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using Xunit;
|
using Xunit;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Diagnostics;
|
||||||
|
|
||||||
namespace FreeSql.Tests.MySql
|
namespace FreeSql.Tests.MySql
|
||||||
{
|
{
|
||||||
@ -32,13 +35,18 @@ namespace FreeSql.Tests.MySql
|
|||||||
public int? Points { get; set; }
|
public int? Points { get; set; }
|
||||||
}
|
}
|
||||||
[FreeSql.DataAnnotations.Table(Name = "ClickHouseTest")]
|
[FreeSql.DataAnnotations.Table(Name = "ClickHouseTest")]
|
||||||
public class TestClickHouse
|
public class TestClickHouse : IEnumerable
|
||||||
{
|
{
|
||||||
[FreeSql.DataAnnotations.Column(IsPrimary = true)]
|
[FreeSql.DataAnnotations.Column(IsPrimary = true)]
|
||||||
[Now]
|
[Now]
|
||||||
public long Id { get; set; }
|
public long Id { get; set; }
|
||||||
|
|
||||||
public string Name { get; set; }
|
public string Name { get; set; }
|
||||||
|
public IEnumerator GetEnumerator()
|
||||||
|
{
|
||||||
|
yield return Id;
|
||||||
|
yield return Name;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
class NowAttribute: Attribute { }
|
class NowAttribute: Attribute { }
|
||||||
|
|
||||||
@ -73,9 +81,10 @@ namespace FreeSql.Tests.MySql
|
|||||||
[Fact]
|
[Fact]
|
||||||
public void TestInsert()
|
public void TestInsert()
|
||||||
{
|
{
|
||||||
|
Stopwatch stopwatch =new Stopwatch();
|
||||||
var fsql = g.clickHouse;
|
var fsql = g.clickHouse;
|
||||||
List<TestClickHouse> list=new List<TestClickHouse>();
|
List<TestClickHouse> list=new List<TestClickHouse>();
|
||||||
for (int i = 0; i < 1000; i++)
|
for (int i = 0; i < 1000000; i++)
|
||||||
{
|
{
|
||||||
list.Add(new TestClickHouse()
|
list.Add(new TestClickHouse()
|
||||||
{
|
{
|
||||||
@ -83,9 +92,13 @@ namespace FreeSql.Tests.MySql
|
|||||||
Name = $"测试{i}"
|
Name = $"测试{i}"
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
fsql.Delete<TestClickHouse>().Where(t => 1 == 1).ExecuteAffrows();
|
||||||
|
stopwatch.Start();
|
||||||
fsql.Insert(list).ExecuteAffrows();
|
fsql.Insert(list).ExecuteAffrows();
|
||||||
var items = fsql.Select<TestClickHouse>().Where(o=>o.Id>900).OrderByDescending(o=>o.Id).ToList();
|
stopwatch.Stop();
|
||||||
Assert.Equal(100, items.Count);
|
Debug.WriteLine(list.Count+"条用时:" +stopwatch.ElapsedMilliseconds.ToString());
|
||||||
|
//var items = fsql.Select<TestClickHouse>().Where(o=>o.Id>900).OrderByDescending(o=>o.Id).ToList();
|
||||||
|
//Assert.Equal(100, items.Count);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
@ -1,9 +1,12 @@
|
|||||||
using FreeSql.Internal;
|
using ClickHouse.Client.ADO;
|
||||||
|
using ClickHouse.Client.Copy;
|
||||||
|
using FreeSql.Internal;
|
||||||
using FreeSql.Internal.Model;
|
using FreeSql.Internal.Model;
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Data;
|
using System.Data;
|
||||||
using System.Data.Common;
|
using System.Data.Common;
|
||||||
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Threading;
|
using System.Threading;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
@ -30,9 +33,9 @@ namespace FreeSql.ClickHouse.Curd
|
|||||||
internal Dictionary<string, bool> InternalIgnore => _ignore;
|
internal Dictionary<string, bool> InternalIgnore => _ignore;
|
||||||
internal void InternalClearData() => ClearData();
|
internal void InternalClearData() => ClearData();
|
||||||
|
|
||||||
public override int ExecuteAffrows() => base.SplitExecuteAffrows(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000);
|
public override int ExecuteAffrows() => base.SplitExecuteAffrows(int.MaxValue, int.MaxValue);
|
||||||
public override long ExecuteIdentity() => base.SplitExecuteIdentity(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000);
|
public override long ExecuteIdentity() => base.SplitExecuteIdentity(int.MaxValue, int.MaxValue);
|
||||||
public override List<T1> ExecuteInserted() => base.SplitExecuteInserted(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000);
|
public override List<T1> ExecuteInserted() => base.SplitExecuteInserted(int.MaxValue, int.MaxValue);
|
||||||
|
|
||||||
|
|
||||||
public override string ToSql()
|
public override string ToSql()
|
||||||
@ -42,6 +45,81 @@ namespace FreeSql.ClickHouse.Curd
|
|||||||
return $"INSERT IGNORE INTO {sql.Substring(12)}";
|
return $"INSERT IGNORE INTO {sql.Substring(12)}";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected override int RawExecuteAffrows()
|
||||||
|
{
|
||||||
|
var affrows = 0;
|
||||||
|
Exception exception = null;
|
||||||
|
Aop.CurdBeforeEventArgs before=null;
|
||||||
|
if (_source.Count>1)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
before = new Aop.CurdBeforeEventArgs(_table.Type, _table, Aop.CurdType.Insert, null, _params);
|
||||||
|
_orm.Aop.CurdBeforeHandler?.Invoke(this, before);
|
||||||
|
using var bulkCopyInterface = new ClickHouseBulkCopy(_orm.Ado.MasterPool.Get().Value as ClickHouseConnection)
|
||||||
|
{
|
||||||
|
DestinationTableName = _table.DbName,
|
||||||
|
BatchSize = _source.Count
|
||||||
|
};
|
||||||
|
bulkCopyInterface.WriteToServerAsync(ToDataTable(),default).Wait();
|
||||||
|
return affrows;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
exception = ex;
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
var after = new Aop.CurdAfterEventArgs(before, exception, affrows);
|
||||||
|
_orm.Aop.CurdAfterHandler?.Invoke(this, after);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var sql = this.ToSql();
|
||||||
|
before = new Aop.CurdBeforeEventArgs(_table.Type, _table, Aop.CurdType.Insert, sql, _params);
|
||||||
|
_orm.Aop.CurdBeforeHandler?.Invoke(this, before);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
affrows = _orm.Ado.ExecuteNonQuery(_connection, _transaction, CommandType.Text, sql, _commandTimeout, _params);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
exception = ex;
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
var after = new Aop.CurdAfterEventArgs(before, exception, affrows);
|
||||||
|
_orm.Aop.CurdAfterHandler?.Invoke(this, after);
|
||||||
|
}
|
||||||
|
return affrows;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private IDictionary<string, object> GetValue<T>(T u, System.Reflection.PropertyInfo[] columns)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Dictionary<string, object> dic = new Dictionary<string, object>();
|
||||||
|
foreach (var item in columns)
|
||||||
|
{
|
||||||
|
object v = null;
|
||||||
|
if (u != null)
|
||||||
|
{
|
||||||
|
v = item.GetValue(u);
|
||||||
|
}
|
||||||
|
dic.TryAdd(item.Name, v);
|
||||||
|
}
|
||||||
|
return dic;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
protected override long RawExecuteIdentity()
|
protected override long RawExecuteIdentity()
|
||||||
{
|
{
|
||||||
var sql = this.ToSql();
|
var sql = this.ToSql();
|
||||||
|
@ -24,7 +24,11 @@ namespace FreeSql.ClickHouse.Curd
|
|||||||
internal Dictionary<string, bool> InternalIgnore => _ignore;
|
internal Dictionary<string, bool> InternalIgnore => _ignore;
|
||||||
public override string ToSql()
|
public override string ToSql()
|
||||||
{
|
{
|
||||||
return base.ToSql()?.Replace("UPDATE", "ALTER TABLE").Replace("SET", "UPDATE");
|
var sql = base.ToSql();
|
||||||
|
sql = sql.Remove(0, 6).Insert(0, "ALTER TABLE");
|
||||||
|
var index = sql.IndexOf(" SET ");
|
||||||
|
sql = sql.Remove(index, 5).Insert(index, " UPDATE ");
|
||||||
|
return sql;
|
||||||
}
|
}
|
||||||
internal void InternalResetSource(List<T1> source) => _source = source;
|
internal void InternalResetSource(List<T1> source) => _source = source;
|
||||||
internal string InternalWhereCaseSource(string CsName, Func<string, string> thenValue) => WhereCaseSource(CsName, thenValue);
|
internal string InternalWhereCaseSource(string CsName, Func<string, string> thenValue) => WhereCaseSource(CsName, thenValue);
|
||||||
|
Loading…
x
Reference in New Issue
Block a user