From c335eab82c38b70820f33ea68471d292e62c323c Mon Sep 17 00:00:00 2001 From: 28810 <28810@YEXIANGQIN> Date: Sat, 21 Dec 2019 14:43:24 +0800 Subject: [PATCH] =?UTF-8?q?-=20=E5=A2=9E=E5=8A=A0=20IInsert.ExecuteSqlBulk?= =?UTF-8?q?Copy=20=E6=89=A9=E5=B1=95=E6=96=B9=E6=B3=95=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=20SqlBulkCopy=20=E6=89=B9=E9=87=8F=E6=8F=92=E5=85=A5=EF=BC=8C?= =?UTF-8?q?=E5=9C=A8=20FreeSql.Provider.SqlServer=20=E5=8F=AF=E7=94=A8?= =?UTF-8?q?=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- FreeSql/FreeSql.xml | 9 ++ .../Internal/CommonProvider/InsertProvider.cs | 17 +++- .../Curd/SqlServerInsert.cs | 5 ++ .../SqlServerExtensions.cs | 84 +++++++++++++++++++ 4 files changed, 113 insertions(+), 2 deletions(-) diff --git a/FreeSql/FreeSql.xml b/FreeSql/FreeSql.xml index 9033ba51..5a5f4b82 100644 --- a/FreeSql/FreeSql.xml +++ b/FreeSql/FreeSql.xml @@ -910,6 +910,15 @@ + + + 返回 DataTable 以便做 BulkCopy 数据做准备 + 此方法会处理： + 类型、表名、字段名映射 + IgnoreColumns、InsertColumns + + + 自动产生 as1, as2, as3 .... 字段别名 diff --git a/FreeSql/Internal/CommonProvider/InsertProvider.cs b/FreeSql/Internal/CommonProvider/InsertProvider.cs index d634ad30..296dac53 100644 --- a/FreeSql/Internal/CommonProvider/InsertProvider.cs +++ b/FreeSql/Internal/CommonProvider/InsertProvider.cs @@ -1,4 +1,5 @@ using FreeSql.Internal.Model; +using FreeSql.Extensions.EntityUtil; using SafeObjectPool; using System; using System.Collections.Generic; @@ -136,8 +137,20 @@ namespace FreeSql.Internal.CommonProvider foreach (var col in table.Columns.Values) { object val = col.GetMapValue(data); - if (col.Attribute.IsPrimary && col.Attribute.MapType.NullableTypeOrThis() == typeof(Guid) && (val == null || (Guid)val == Guid.Empty)) - col.SetMapValue(data, val = FreeUtil.NewMongodbId()); + if (col.Attribute.IsPrimary) + { + if (col.Attribute.MapType.NullableTypeOrThis() == typeof(Guid) && (val == null || (Guid)val == Guid.Empty)) + col.SetMapValue(data, val = FreeUtil.NewMongodbId()); + else if (col.CsType.NullableTypeOrThis() == typeof(Guid)) + { + var guidVal = orm.GetEntityValueWithPropertyName(table.Type, data, col.CsName); + if (guidVal == null || (Guid)guidVal == Guid.Empty) + { + orm.SetEntityValueWithPropertyName(table.Type, data, col.CsName, FreeUtil.NewMongodbId()); + val = col.GetMapValue(data); + } + } + } if (orm.Aop.AuditValue != null) { var auditArgs = new Aop.AuditValueEventArgs(Aop.AuditValueType.Insert, col, table.Properties[col.CsName], val); diff --git a/Providers/FreeSql.Provider.SqlServer/Curd/SqlServerInsert.cs b/Providers/FreeSql.Provider.SqlServer/Curd/SqlServerInsert.cs index df48b84a..f70d791e 100644 --- a/Providers/FreeSql.Provider.SqlServer/Curd/SqlServerInsert.cs +++ b/Providers/FreeSql.Provider.SqlServer/Curd/SqlServerInsert.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Data; using System.Data.Common; +using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -18,6 +19,10 @@ namespace FreeSql.SqlServer.Curd { } + internal IFreeSql InternalOrm => _orm as IFreeSql; + internal SqlConnection InternalConnection => _connection as SqlConnection; + internal SqlTransaction InternalTransaction => _transaction as SqlTransaction; + public override int ExecuteAffrows() => base.SplitExecuteAffrows(_batchValuesLimit > 0 ? _batchValuesLimit : 1000, _batchParameterLimit > 0 ? _batchParameterLimit : 2100); public override long ExecuteIdentity() => base.SplitExecuteIdentity(_batchValuesLimit > 0 ? _batchValuesLimit : 1000, _batchParameterLimit > 0 ? _batchParameterLimit : 2100); public override List ExecuteInserted() => base.SplitExecuteInserted(_batchValuesLimit > 0 ? _batchValuesLimit : 1000, _batchParameterLimit > 0 ? _batchParameterLimit : 2100); diff --git a/Providers/FreeSql.Provider.SqlServer/SqlServerExtensions.cs b/Providers/FreeSql.Provider.SqlServer/SqlServerExtensions.cs index ab503d90..38cb29d5 100644 --- a/Providers/FreeSql.Provider.SqlServer/SqlServerExtensions.cs +++ b/Providers/FreeSql.Provider.SqlServer/SqlServerExtensions.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Data.SqlClient; public static partial class FreeSqlSqlServerGlobalExtensions { @@ -40,6 +41,89 @@ public static partial class FreeSqlSqlServerGlobalExtensions return that; } internal static ConcurrentDictionary)> _dicSetGlobalSelectWithLock = new ConcurrentDictionary)>(); + + /// + /// SqlServer SqlCopyBulk 批量插入功能 + /// 使用 IgnoreColumns/InsertColumns 设置忽略/指定导入的列 + /// 使用 WithConnection/WithTransaction 传入连接/事务对象 + /// 提示：若本方法不能满足，请使用 IInsert<T>.ToDataTable 方法得到 DataTable 对象后，自行处理。 + /// SqlCopyBulk 与 insert into t values(..),(..),(..) 性能测试参考： + /// 插入180000行，52列：21,065ms 与 402,355ms，10列：4,248ms 与 47,204ms + /// 插入10000行，52列：578ms 与 24,847ms，10列：127ms 与 2,275ms + /// 插入5000行，52列：326ms 与 11,465ms，10列：71ms 与 1,108ms + /// 插入2000行，52列：139ms 与 4,971ms，10列：30ms 与 488ms + /// 插入1000行，52列：105ms 与 2,437ms，10列：48ms 与 279ms + /// 插入500行，52列：79ms 与 915ms，10列：14ms 与 123ms + /// 插入100行，52列：60ms 与 138ms，10列：11ms 与 35ms + /// 插入50行，52列：48ms 与 88ms，10列：10ms 与 16ms + /// + /// + /// + /// + /// + /// + public static void ExecuteSqlBulkCopy(this IInsert that, SqlBulkCopyOptions copyOptions = SqlBulkCopyOptions.Default, int? batchSize = null, int? bulkCopyTimeout = null) where T : class + { + var insert = that as FreeSql.SqlServer.Curd.SqlServerInsert; + if (insert == null) throw new Exception("ExecuteSqlBulkCopy 是 FreeSql.Provider.SqlServer 特有的功能"); + + var dt = that.ToDataTable(); + if (dt.Rows.Count == 0) return; + + Action writeToServer = bulkCopy => + { + if (batchSize.HasValue) bulkCopy.BatchSize = batchSize.Value; + if (bulkCopyTimeout.HasValue) bulkCopy.BulkCopyTimeout = bulkCopyTimeout.Value; + bulkCopy.DestinationTableName = dt.TableName; + bulkCopy.WriteToServer(dt); + }; + + if (insert.InternalConnection == null && insert.InternalTransaction == null) + { + using (var conn = insert.InternalOrm.Ado.MasterPool.Get()) + { + using (var bulkCopy = copyOptions == SqlBulkCopyOptions.Default ? + new SqlBulkCopy(conn.Value as SqlConnection) : + new SqlBulkCopy(conn.Value as SqlConnection, copyOptions, null)) + { + writeToServer(bulkCopy); + } + } + } + else if (insert.InternalTransaction != null) + { + using (var bulkCopy = copyOptions == SqlBulkCopyOptions.Default ? + new SqlBulkCopy(insert.InternalTransaction.Connection) : + new SqlBulkCopy(insert.InternalTransaction.Connection, copyOptions, insert.InternalTransaction)) + { + writeToServer(bulkCopy); + } + } + else if (insert.InternalConnection != null) + { + var conn = insert.InternalConnection; + var isNotOpen = false; + if (conn.State != System.Data.ConnectionState.Open) + { + isNotOpen = true; + conn.Open(); + } + using (var bulkCopy = copyOptions == SqlBulkCopyOptions.Default ? + new SqlBulkCopy(insert.InternalConnection) : + new SqlBulkCopy(insert.InternalConnection, copyOptions, null)) + { + writeToServer(bulkCopy); + } + if (isNotOpen) + { + conn.Close(); + } + } + else + { + throw new NotImplementedException("未实现错误，请反馈给作者"); + } + } } [Flags]