mirror of
				https://github.com/nsnail/FreeSql.git
				synced 2025-11-04 01:05:27 +08:00 
			
		
		
		
	更改批量插入数据的方式(测试100w条5秒不到),更改数据修改时的替换可能替换到数据的问题
This commit is contained in:
		@@ -538,14 +538,5 @@
 | 
				
			|||||||
            <param name="that"></param>
 | 
					            <param name="that"></param>
 | 
				
			||||||
            <returns></returns>
 | 
					            <returns></returns>
 | 
				
			||||||
        </member>
 | 
					        </member>
 | 
				
			||||||
        <member name="M:Microsoft.Extensions.DependencyInjection.FreeSqlRepositoryDependencyInjection.AddFreeRepository(Microsoft.Extensions.DependencyInjection.IServiceCollection,System.Action{FreeSql.FluentDataFilter},System.Reflection.Assembly[])">
 | 
					 | 
				
			||||||
            <summary>
 | 
					 | 
				
			||||||
            批量注入 Repository,可以参考代码自行调整
 | 
					 | 
				
			||||||
            </summary>
 | 
					 | 
				
			||||||
            <param name="services"></param>
 | 
					 | 
				
			||||||
            <param name="globalDataFilter"></param>
 | 
					 | 
				
			||||||
            <param name="assemblies"></param>
 | 
					 | 
				
			||||||
            <returns></returns>
 | 
					 | 
				
			||||||
        </member>
 | 
					 | 
				
			||||||
    </members>
 | 
					    </members>
 | 
				
			||||||
</doc>
 | 
					</doc>
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -3,6 +3,9 @@ using System.Collections.Generic;
 | 
				
			|||||||
using System.Reflection;
 | 
					using System.Reflection;
 | 
				
			||||||
using System.Text;
 | 
					using System.Text;
 | 
				
			||||||
using Xunit;
 | 
					using Xunit;
 | 
				
			||||||
 | 
					using System.Linq;
 | 
				
			||||||
 | 
					using System.Collections;
 | 
				
			||||||
 | 
					using System.Diagnostics;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
namespace FreeSql.Tests.MySql
 | 
					namespace FreeSql.Tests.MySql
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
@@ -32,13 +35,18 @@ namespace FreeSql.Tests.MySql
 | 
				
			|||||||
            public int? Points { get; set; }
 | 
					            public int? Points { get; set; }
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
        [FreeSql.DataAnnotations.Table(Name = "ClickHouseTest")]
 | 
					        [FreeSql.DataAnnotations.Table(Name = "ClickHouseTest")]
 | 
				
			||||||
        public class TestClickHouse
 | 
					        public class TestClickHouse : IEnumerable
 | 
				
			||||||
        {
 | 
					        {
 | 
				
			||||||
            [FreeSql.DataAnnotations.Column(IsPrimary = true)]
 | 
					            [FreeSql.DataAnnotations.Column(IsPrimary = true)]
 | 
				
			||||||
            [Now]
 | 
					            [Now]
 | 
				
			||||||
            public long Id { get; set; }
 | 
					            public long Id { get; set; }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
            public string Name { get; set; }
 | 
					            public string Name { get; set; }
 | 
				
			||||||
 | 
					            public IEnumerator GetEnumerator()
 | 
				
			||||||
 | 
					            {
 | 
				
			||||||
 | 
					                yield return Id;
 | 
				
			||||||
 | 
					                yield return Name;
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
        class NowAttribute: Attribute { }
 | 
					        class NowAttribute: Attribute { }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@@ -73,9 +81,10 @@ namespace FreeSql.Tests.MySql
 | 
				
			|||||||
        [Fact]
 | 
					        [Fact]
 | 
				
			||||||
        public void TestInsert()
 | 
					        public void TestInsert()
 | 
				
			||||||
        {
 | 
					        {
 | 
				
			||||||
 | 
					            Stopwatch stopwatch =new Stopwatch();
 | 
				
			||||||
            var fsql = g.clickHouse;
 | 
					            var fsql = g.clickHouse;
 | 
				
			||||||
            List<TestClickHouse> list=new List<TestClickHouse>();
 | 
					            List<TestClickHouse> list=new List<TestClickHouse>();
 | 
				
			||||||
            for (int i = 0; i < 1000; i++)
 | 
					            for (int i = 0; i < 1000000; i++)
 | 
				
			||||||
            {
 | 
					            {
 | 
				
			||||||
                list.Add(new TestClickHouse()
 | 
					                list.Add(new TestClickHouse()
 | 
				
			||||||
                {
 | 
					                {
 | 
				
			||||||
@@ -83,9 +92,13 @@ namespace FreeSql.Tests.MySql
 | 
				
			|||||||
                    Name = $"测试{i}"
 | 
					                    Name = $"测试{i}"
 | 
				
			||||||
                });
 | 
					                });
 | 
				
			||||||
            }
 | 
					            }
 | 
				
			||||||
 | 
					            fsql.Delete<TestClickHouse>().Where(t => 1 == 1).ExecuteAffrows();
 | 
				
			||||||
 | 
					            stopwatch.Start();
 | 
				
			||||||
            fsql.Insert(list).ExecuteAffrows();
 | 
					            fsql.Insert(list).ExecuteAffrows();
 | 
				
			||||||
            var items = fsql.Select<TestClickHouse>().Where(o=>o.Id>900).OrderByDescending(o=>o.Id).ToList();
 | 
					            stopwatch.Stop();
 | 
				
			||||||
            Assert.Equal(100, items.Count);
 | 
					            Debug.WriteLine(list.Count+"条用时:" +stopwatch.ElapsedMilliseconds.ToString());
 | 
				
			||||||
 | 
					            //var items = fsql.Select<TestClickHouse>().Where(o=>o.Id>900).OrderByDescending(o=>o.Id).ToList();
 | 
				
			||||||
 | 
					            //Assert.Equal(100, items.Count);
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        [Fact]
 | 
					        [Fact]
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -1,9 +1,12 @@
 | 
				
			|||||||
using FreeSql.Internal;
 | 
					using ClickHouse.Client.ADO;
 | 
				
			||||||
 | 
					using ClickHouse.Client.Copy;
 | 
				
			||||||
 | 
					using FreeSql.Internal;
 | 
				
			||||||
using FreeSql.Internal.Model;
 | 
					using FreeSql.Internal.Model;
 | 
				
			||||||
using System;
 | 
					using System;
 | 
				
			||||||
using System.Collections.Generic;
 | 
					using System.Collections.Generic;
 | 
				
			||||||
using System.Data;
 | 
					using System.Data;
 | 
				
			||||||
using System.Data.Common;
 | 
					using System.Data.Common;
 | 
				
			||||||
 | 
					using System.Linq;
 | 
				
			||||||
using System.Text;
 | 
					using System.Text;
 | 
				
			||||||
using System.Threading;
 | 
					using System.Threading;
 | 
				
			||||||
using System.Threading.Tasks;
 | 
					using System.Threading.Tasks;
 | 
				
			||||||
@@ -30,9 +33,9 @@ namespace FreeSql.ClickHouse.Curd
 | 
				
			|||||||
        internal Dictionary<string, bool> InternalIgnore => _ignore;
 | 
					        internal Dictionary<string, bool> InternalIgnore => _ignore;
 | 
				
			||||||
        internal void InternalClearData() => ClearData();
 | 
					        internal void InternalClearData() => ClearData();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        public override int ExecuteAffrows() => base.SplitExecuteAffrows(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000);
 | 
					        public override int ExecuteAffrows() => base.SplitExecuteAffrows(int.MaxValue, int.MaxValue);
 | 
				
			||||||
        public override long ExecuteIdentity() => base.SplitExecuteIdentity(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000);
 | 
					        public override long ExecuteIdentity() => base.SplitExecuteIdentity(int.MaxValue, int.MaxValue);
 | 
				
			||||||
        public override List<T1> ExecuteInserted() => base.SplitExecuteInserted(_batchValuesLimit > 0 ? _batchValuesLimit : 5000, _batchParameterLimit > 0 ? _batchParameterLimit : 3000);
 | 
					        public override List<T1> ExecuteInserted() => base.SplitExecuteInserted(int.MaxValue, int.MaxValue);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        public override string ToSql()
 | 
					        public override string ToSql()
 | 
				
			||||||
@@ -42,6 +45,81 @@ namespace FreeSql.ClickHouse.Curd
 | 
				
			|||||||
            return $"INSERT IGNORE INTO {sql.Substring(12)}";
 | 
					            return $"INSERT IGNORE INTO {sql.Substring(12)}";
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        protected override int RawExecuteAffrows()
 | 
				
			||||||
 | 
					        {
 | 
				
			||||||
 | 
					            var affrows = 0;
 | 
				
			||||||
 | 
					            Exception exception = null;
 | 
				
			||||||
 | 
					            Aop.CurdBeforeEventArgs before=null;
 | 
				
			||||||
 | 
					            if (_source.Count>1)
 | 
				
			||||||
 | 
					            {
 | 
				
			||||||
 | 
					                try
 | 
				
			||||||
 | 
					                {
 | 
				
			||||||
 | 
					                    before = new Aop.CurdBeforeEventArgs(_table.Type, _table, Aop.CurdType.Insert, null, _params);
 | 
				
			||||||
 | 
					                    _orm.Aop.CurdBeforeHandler?.Invoke(this, before);
 | 
				
			||||||
 | 
					                    using var bulkCopyInterface = new ClickHouseBulkCopy(_orm.Ado.MasterPool.Get().Value as ClickHouseConnection)
 | 
				
			||||||
 | 
					                    {
 | 
				
			||||||
 | 
					                        DestinationTableName = _table.DbName,
 | 
				
			||||||
 | 
					                        BatchSize = _source.Count
 | 
				
			||||||
 | 
					                    };
 | 
				
			||||||
 | 
					                    bulkCopyInterface.WriteToServerAsync(ToDataTable(),default).Wait();
 | 
				
			||||||
 | 
					                    return affrows;
 | 
				
			||||||
 | 
					                }
 | 
				
			||||||
 | 
					                catch (Exception ex)
 | 
				
			||||||
 | 
					                {
 | 
				
			||||||
 | 
					                    exception = ex;
 | 
				
			||||||
 | 
					                    throw ex;
 | 
				
			||||||
 | 
					                }
 | 
				
			||||||
 | 
					                finally
 | 
				
			||||||
 | 
					                {
 | 
				
			||||||
 | 
					                    var after = new Aop.CurdAfterEventArgs(before, exception, affrows);
 | 
				
			||||||
 | 
					                    _orm.Aop.CurdAfterHandler?.Invoke(this, after);
 | 
				
			||||||
 | 
					                }
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					            else
 | 
				
			||||||
 | 
					            {
 | 
				
			||||||
 | 
					                var sql = this.ToSql();
 | 
				
			||||||
 | 
					                before = new Aop.CurdBeforeEventArgs(_table.Type, _table, Aop.CurdType.Insert, sql, _params);
 | 
				
			||||||
 | 
					                _orm.Aop.CurdBeforeHandler?.Invoke(this, before);
 | 
				
			||||||
 | 
					                try
 | 
				
			||||||
 | 
					                {
 | 
				
			||||||
 | 
					                    affrows = _orm.Ado.ExecuteNonQuery(_connection, _transaction, CommandType.Text, sql, _commandTimeout, _params);
 | 
				
			||||||
 | 
					                }
 | 
				
			||||||
 | 
					                catch (Exception ex)
 | 
				
			||||||
 | 
					                {
 | 
				
			||||||
 | 
					                    exception = ex;
 | 
				
			||||||
 | 
					                    throw ex;
 | 
				
			||||||
 | 
					                }
 | 
				
			||||||
 | 
					                finally
 | 
				
			||||||
 | 
					                {
 | 
				
			||||||
 | 
					                    var after = new Aop.CurdAfterEventArgs(before, exception, affrows);
 | 
				
			||||||
 | 
					                    _orm.Aop.CurdAfterHandler?.Invoke(this, after);
 | 
				
			||||||
 | 
					                }
 | 
				
			||||||
 | 
					                return affrows;
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        private IDictionary<string, object> GetValue<T>(T u, System.Reflection.PropertyInfo[] columns)
 | 
				
			||||||
 | 
					        {
 | 
				
			||||||
 | 
					            try
 | 
				
			||||||
 | 
					            {
 | 
				
			||||||
 | 
					                Dictionary<string, object> dic = new Dictionary<string, object>();
 | 
				
			||||||
 | 
					                foreach (var item in columns)
 | 
				
			||||||
 | 
					                {
 | 
				
			||||||
 | 
					                    object v = null;
 | 
				
			||||||
 | 
					                    if (u != null)
 | 
				
			||||||
 | 
					                    {
 | 
				
			||||||
 | 
					                        v = item.GetValue(u);
 | 
				
			||||||
 | 
					                    }
 | 
				
			||||||
 | 
					                    dic.TryAdd(item.Name, v);
 | 
				
			||||||
 | 
					                }
 | 
				
			||||||
 | 
					                return dic;
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					            catch (Exception e)
 | 
				
			||||||
 | 
					            {
 | 
				
			||||||
 | 
					                throw;
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        protected override long RawExecuteIdentity()
 | 
					        protected override long RawExecuteIdentity()
 | 
				
			||||||
        {
 | 
					        {
 | 
				
			||||||
            var sql = this.ToSql();
 | 
					            var sql = this.ToSql();
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -24,7 +24,11 @@ namespace FreeSql.ClickHouse.Curd
 | 
				
			|||||||
        internal Dictionary<string, bool> InternalIgnore => _ignore;
 | 
					        internal Dictionary<string, bool> InternalIgnore => _ignore;
 | 
				
			||||||
        public override string ToSql()
 | 
					        public override string ToSql()
 | 
				
			||||||
        {
 | 
					        {
 | 
				
			||||||
            return base.ToSql()?.Replace("UPDATE", "ALTER TABLE").Replace("SET", "UPDATE");
 | 
					            var sql = base.ToSql();
 | 
				
			||||||
 | 
					            sql = sql.Remove(0, 6).Insert(0, "ALTER TABLE");
 | 
				
			||||||
 | 
					            var index = sql.IndexOf(" SET ");
 | 
				
			||||||
 | 
					            sql = sql.Remove(index, 5).Insert(index, " UPDATE ");
 | 
				
			||||||
 | 
					            return sql;
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
        internal void InternalResetSource(List<T1> source) => _source = source;
 | 
					        internal void InternalResetSource(List<T1> source) => _source = source;
 | 
				
			||||||
        internal string InternalWhereCaseSource(string CsName, Func<string, string> thenValue) => WhereCaseSource(CsName, thenValue);
 | 
					        internal string InternalWhereCaseSource(string CsName, Func<string, string> thenValue) => WhereCaseSource(CsName, thenValue);
 | 
				
			||||||
 
 | 
				
			|||||||
		Reference in New Issue
	
	Block a user