From 09f3f1a20c68c5551d4fbbdfe7978bd6e6145947 Mon Sep 17 00:00:00 2001 From: 2881099 <2881099@qq.com> Date: Sat, 25 Mar 2023 14:50:13 +0800 Subject: [PATCH] =?UTF-8?q?-=20=E5=A2=9E=E5=8A=A0=20IInsertOrUpdate=20?= =?UTF-8?q?=E9=AB=98=E6=80=A7=E8=83=BD=E6=96=B9=E6=B3=95=20ExecuteOracleBu?= =?UTF-8?q?lkCopy/ExecuteDmBulkCopy=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Examples/base_entity/Entities/User.cs | 2 +- Examples/base_entity/Program.cs | 12 +++-- FreeSql/Internal/Model/ColumnInfo.cs | 2 +- FreeSql/Internal/Model/TableInfo.cs | 2 +- .../DamengExtensions.cs | 51 ++++++++++++++++++- .../OracleExtensions.cs | 48 +++++++++++++++++ .../SqlServerExtensions.cs | 14 ++--- 7 files changed, 112 insertions(+), 19 deletions(-) diff --git a/Examples/base_entity/Entities/User.cs b/Examples/base_entity/Entities/User.cs index faf9a460..70982857 100644 --- a/Examples/base_entity/Entities/User.cs +++ b/Examples/base_entity/Entities/User.cs @@ -55,7 +55,7 @@ public class User1 : BaseEntity /// /// 描述 /// - [MaxLength(4000)] + [MaxLength(2000)] public string Description { get; set; } } diff --git a/Examples/base_entity/Program.cs b/Examples/base_entity/Program.cs index b2701460..17a9e949 100644 --- a/Examples/base_entity/Program.cs +++ b/Examples/base_entity/Program.cs @@ -552,8 +552,8 @@ namespace base_entity //.UseConnectionString(FreeSql.DataType.Oracle, "user id=user1;password=123456;data source=//127.0.0.1:1521/XE;Pooling=true;Max Pool Size=2") //.UseNameConvert(FreeSql.Internal.NameConvertType.ToUpper) - //.UseConnectionString(FreeSql.DataType.Dameng, "server=127.0.0.1;port=5236;user id=2user;password=123456789;database=2user;poolsize=5;min pool size=1") - //.UseNameConvert(FreeSql.Internal.NameConvertType.ToUpper) + .UseConnectionString(FreeSql.DataType.Dameng, "server=127.0.0.1;port=5236;user id=2user;password=123456789;database=2user;poolsize=5;min pool size=1") + .UseNameConvert(FreeSql.Internal.NameConvertType.ToUpper) //.UseConnectionString(FreeSql.DataType.OdbcMySql, "Driver={MySQL ODBC 8.0 Unicode Driver};Server=127.0.0.1;Persist Security Info=False;Trusted_Connection=Yes;UID=root;PWD=root;DATABASE=cccddd_odbc;Charset=utf8;SslMode=none;Max pool size=2") @@ -578,9 +578,15 @@ namespace base_entity BaseEntity.Initialization(fsql, () => _asyncUow.Value); #endregion + fsql.CodeFirst.GetTableByEntity(typeof(User1)).Columns.Values.ToList().ForEach(col => + { + col.Comment = ""; + }); + fsql.Insert(Enumerable.Range(0, 100).Select(a => new User1 { Id = Guid.NewGuid(), Nickname = $"nickname{a}", Username = $"username{a}", Description = $"desc{a}" }).ToArray()).ExecuteAffrows(); + fsql.InsertOrUpdate() .SetSource(fsql.Select().ToList()) - .ExecuteSqlBulkCopy(); + .ExecuteDmBulkCopy(); var updatejoin01 = fsql.Update() .Join(fsql.Select(), (a, b) => a.GroupId == b.Id) diff --git a/FreeSql/Internal/Model/ColumnInfo.cs b/FreeSql/Internal/Model/ColumnInfo.cs index d8a5c42c..98c5ed90 100644 --- a/FreeSql/Internal/Model/ColumnInfo.cs +++ b/FreeSql/Internal/Model/ColumnInfo.cs @@ -13,7 +13,7 @@ namespace FreeSql.Internal.Model public string CsName { get; set; } public Type CsType { get; set; } public ColumnAttribute Attribute { get; set; } - public string Comment { get; internal set; } + public string Comment { get; set; } public string DbTypeText { get; internal set; } public string DbDefaultValue { get; internal set; } public string DbInsertValue { get; internal set; } diff --git a/FreeSql/Internal/Model/TableInfo.cs b/FreeSql/Internal/Model/TableInfo.cs index 56db4fbf..e4b0a30f 100644 --- a/FreeSql/Internal/Model/TableInfo.cs +++ b/FreeSql/Internal/Model/TableInfo.cs @@ -24,7 +24,7 @@ namespace FreeSql.Internal.Model public string DbName { get; set; } public string DbOldName { get; set; } public bool DisableSyncStructure { get; set; } - public string Comment { get; internal set; } + public string Comment { get; set; } public bool IsRereadSql { get; internal set; } public bool IsDictionaryType { get; internal set; } diff --git a/Providers/FreeSql.Provider.Dameng/DamengExtensions.cs b/Providers/FreeSql.Provider.Dameng/DamengExtensions.cs index cfb256cb..c8ed319e 100644 --- a/Providers/FreeSql.Provider.Dameng/DamengExtensions.cs +++ b/Providers/FreeSql.Provider.Dameng/DamengExtensions.cs @@ -20,10 +20,57 @@ public static partial class FreeSqlDamengGlobalExtensions static FreeSql.Dameng.DamengAdo _damengAdo = new FreeSql.Dameng.DamengAdo(); #region ExecuteDmBulkCopy - + /// + /// 批量插入或更新（操作的字段数量超过 2000 时收益大） + /// 实现原理：使用 DmBulkCopy 插入临时表，再执行 MERGE INTO t1 using (select * from #temp) ... + /// + /// + /// + /// + public static int ExecuteDmBulkCopy(this IInsertOrUpdate that) where T : class + { + var upsert = that as InsertOrUpdateProvider; + if (upsert._source.Any() != true || upsert._tempPrimarys.Any() == false) return 0; + var state = ExecuteDmBulkCopyState(upsert); + return UpdateProvider.ExecuteBulkUpsert(upsert, state, insert => insert.ExecuteDmBulkCopy()); + } + static NativeTuple ExecuteDmBulkCopyState(InsertOrUpdateProvider upsert) where T : class + { + if (upsert._source.Any() != true) return null; + var _table = upsert._table; + var _commonUtils = upsert._commonUtils; + var updateTableName = upsert._tableRule?.Invoke(_table.DbName) ?? _table.DbName; + var tempTableName = $"Temp_{Guid.NewGuid().ToString("N").ToUpper().Substring(0, 24)}"; + if (upsert._orm.CodeFirst.IsSyncStructureToLower) tempTableName = tempTableName.ToLower(); + if (upsert._orm.CodeFirst.IsSyncStructureToUpper) tempTableName = tempTableName.ToUpper(); + if (upsert._connection == null && upsert._orm.Ado.TransactionCurrentThread != null) + upsert.WithTransaction(upsert._orm.Ado.TransactionCurrentThread); + var sb = new StringBuilder().Append("CREATE GLOBAL TEMPORARY TABLE ").Append(_commonUtils.QuoteSqlName(tempTableName)).Append(" ( "); + foreach (var col in _table.Columns.Values) + { + sb.Append(" \r\n ").Append(_commonUtils.QuoteSqlName(col.Attribute.Name)).Append(" ").Append(col.Attribute.DbType.Replace("NOT NULL", "")); + sb.Append(","); + } + var sql1 = sb.Remove(sb.Length - 1, 1).Append("\r\n) ON COMMIT PRESERVE ROWS").ToString(); + sb.Clear(); + try + { + upsert._sourceSql = $"select * from {tempTableName}"; + var sql2 = upsert.ToSql(); + var sql3 = $"BEGIN \r\n" + + $"execute immediate 'TRUNCATE TABLE {_commonUtils.QuoteSqlName(tempTableName)}';\r\n" + + $"execute immediate 'DROP TABLE {_commonUtils.QuoteSqlName(tempTableName)}';\r\n" + + $"END;"; + return NativeTuple.Create(sql1, sql2, sql3, tempTableName, _table.Columns.Values.Select(a => a.Attribute.Name).ToArray()); + } + finally + { + upsert._sourceSql = null; + } + } /// /// 批量更新（更新字段数量超过 2000 时收益大） - /// 实现原理：使用 OracleBulkCopy 插入临时表，再使用 MERGE INTO 联表更新 + /// 实现原理：使用 DmBulkCopy 插入临时表，再使用 MERGE INTO 联表更新 /// /// /// diff --git a/Providers/FreeSql.Provider.Oracle/OracleExtensions.cs b/Providers/FreeSql.Provider.Oracle/OracleExtensions.cs index 13ff188f..a61edb86 100644 --- a/Providers/FreeSql.Provider.Oracle/OracleExtensions.cs +++ b/Providers/FreeSql.Provider.Oracle/OracleExtensions.cs @@ -27,6 +27,54 @@ public static partial class FreeSqlOracleGlobalExtensions #else #region ExecuteOracleBulkCopy /// + /// 批量插入或更新（操作的字段数量超过 2000 时收益大） + /// 实现原理：使用 OracleBulkCopy 插入临时表，再执行 MERGE INTO t1 using (select * from #temp) ... + /// + /// + /// + /// + public static int ExecuteOracleBulkCopy(this IInsertOrUpdate that) where T : class + { + var upsert = that as InsertOrUpdateProvider; + if (upsert._source.Any() != true || upsert._tempPrimarys.Any() == false) return 0; + var state = ExecuteOracleBulkCopyState(upsert); + return UpdateProvider.ExecuteBulkUpsert(upsert, state, insert => insert.ExecuteOracleBulkCopy()); + } + static NativeTuple ExecuteOracleBulkCopyState(InsertOrUpdateProvider upsert) where T : class + { + if (upsert._source.Any() != true) return null; + var _table = upsert._table; + var _commonUtils = upsert._commonUtils; + var updateTableName = upsert._tableRule?.Invoke(_table.DbName) ?? _table.DbName; + var tempTableName = $"Temp_{Guid.NewGuid().ToString("N").ToUpper().Substring(0, 24)}"; + if (upsert._orm.CodeFirst.IsSyncStructureToLower) tempTableName = tempTableName.ToLower(); + if (upsert._orm.CodeFirst.IsSyncStructureToUpper) tempTableName = tempTableName.ToUpper(); + if (upsert._connection == null && upsert._orm.Ado.TransactionCurrentThread != null) + upsert.WithTransaction(upsert._orm.Ado.TransactionCurrentThread); + var sb = new StringBuilder().Append("CREATE GLOBAL TEMPORARY TABLE ").Append(_commonUtils.QuoteSqlName(tempTableName)).Append(" ( "); + foreach (var col in _table.Columns.Values) + { + sb.Append(" \r\n ").Append(_commonUtils.QuoteSqlName(col.Attribute.Name)).Append(" ").Append(col.Attribute.DbType.Replace("NOT NULL", "")); + sb.Append(","); + } + var sql1 = sb.Remove(sb.Length - 1, 1).Append("\r\n) ON COMMIT PRESERVE ROWS").ToString(); + sb.Clear(); + try + { + upsert._sourceSql = $"select * from {tempTableName}"; + var sql2 = upsert.ToSql(); + var sql3 = $"BEGIN \r\n" + + $"execute immediate 'TRUNCATE TABLE {_commonUtils.QuoteSqlName(tempTableName)}';\r\n" + + $"execute immediate 'DROP TABLE {_commonUtils.QuoteSqlName(tempTableName)}';\r\n" + + $"END;"; + return NativeTuple.Create(sql1, sql2, sql3, tempTableName, _table.Columns.Values.Select(a => a.Attribute.Name).ToArray()); + } + finally + { + upsert._sourceSql = null; + } + } + /// /// 批量更新（更新字段数量超过 2000 时收益大） /// 实现原理：使用 OracleBulkCopy 插入临时表，再使用 MERGE INTO 联表更新 /// diff --git a/Providers/FreeSql.Provider.SqlServer/SqlServerExtensions.cs b/Providers/FreeSql.Provider.SqlServer/SqlServerExtensions.cs index 5b135927..ef79c803 100644 --- a/Providers/FreeSql.Provider.SqlServer/SqlServerExtensions.cs +++ b/Providers/FreeSql.Provider.SqlServer/SqlServerExtensions.cs @@ -135,7 +135,6 @@ public static partial class FreeSqlSqlServerGlobalExtensions if (upsert._source.Any() != true || upsert._tempPrimarys.Any() == false) return 0; var state = ExecuteSqlBulkCopyState(upsert); return UpdateProvider.ExecuteBulkUpsert(upsert, state, insert => insert.ExecuteSqlBulkCopy(copyOptions, batchSize, bulkCopyTimeout)); - } static NativeTuple ExecuteSqlBulkCopyState(InsertOrUpdateProvider upsert) where T : class { @@ -148,20 +147,13 @@ public static partial class FreeSqlSqlServerGlobalExtensions if (upsert._orm.CodeFirst.IsSyncStructureToUpper) tempTableName = tempTableName.ToUpper(); if (upsert._connection == null && upsert._orm.Ado.TransactionCurrentThread != null) upsert.WithTransaction(upsert._orm.Ado.TransactionCurrentThread); - var setColumns = new List(); - var pkColumns = new List(); - foreach (var col in _table.Columns.Values) - { - if (upsert._tempPrimarys.Any(a => a.CsName == col.CsName)) pkColumns.Add(col.Attribute.Name); - else if (col.Attribute.IsIdentity == false && col.Attribute.IsVersion == false && upsert._updateIgnore.ContainsKey(col.Attribute.Name) == false) setColumns.Add(col.Attribute.Name); - } - var sql1 = $"SELECT {string.Join(", ", pkColumns.Select(a => _commonUtils.QuoteSqlName(a)))}, {string.Join(", ", setColumns.Select(a => _commonUtils.QuoteSqlName(a)))} INTO {tempTableName} FROM {_commonUtils.QuoteSqlName(updateTableName)} WHERE 1=2"; + var sql1 = $"SELECT {string.Join(", ", _table.Columns.Values.Select(a => _commonUtils.QuoteSqlName(a.Attribute.Name)))} INTO {tempTableName} FROM {_commonUtils.QuoteSqlName(updateTableName)} WHERE 1=2"; try { upsert._sourceSql = $"select * from {tempTableName}"; var sql2 = upsert.ToSql(); var sql3 = $"DROP TABLE {tempTableName}"; - return NativeTuple.Create(sql1, sql2, sql3, tempTableName, pkColumns.Concat(setColumns).ToArray()); + return NativeTuple.Create(sql1, sql2, sql3, tempTableName, _table.Columns.Values.Select(a => a.Attribute.Name).ToArray()); } finally { @@ -195,7 +187,7 @@ public static partial class FreeSqlSqlServerGlobalExtensions if (update._orm.CodeFirst.IsSyncStructureToLower) tempTableName = tempTableName.ToLower(); if (update._orm.CodeFirst.IsSyncStructureToUpper) tempTableName = tempTableName.ToUpper(); if (update._connection == null && update._orm.Ado.TransactionCurrentThread != null) - update.WithTransaction(update._orm.Ado.TransactionCurrentThread); + update.WithTransaction(update._orm.Ado.TransactionCurrentThread); var setColumns = new List(); var pkColumns = new List(); foreach (var col in _table.Columns.Values)