From df6dc0411c874c43c912b0e2b7b5190b93a0812f Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Wed, 12 Aug 2026 13:36:31 -0500 Subject: [PATCH 1/4] :robot: Only perform graph column mapping if bulk copy includes graph tables. --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 174 ++++++++++++------ .../ManualTests/BulkCopy/CopyAllFromReader.cs | 10 +- 2 files changed, 119 insertions(+), 65 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 44c74fc3f0..106c9abd57 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -82,6 +82,8 @@ internal BulkCopySimpleResultSet() _results = new List(); } + internal int Count => _results.Count; + internal Result this[int idx] => _results[idx]; // Callback function for the tdsparser @@ -151,11 +153,11 @@ public SourceColumnMetadata(ValueMethod method, bool isSqlType, bool isDataFeed) public readonly bool IsDataFeed; } - // The initial query will return three tables. + // The initial query will return three tables, and may return a fourth for column aliases. // Transaction count has only one value in one column and one row // MetaData has n columns but no rows // Collation has 4 columns and n rows - // Column aliases has 3 columns and n rows + // Column aliases has 2 columns and n rows private const int MetaDataResultId = 1; @@ -484,6 +486,50 @@ private string CreateInitialQuery() string objectName = ADP.BuildMultiPartName(parts); string escapedObjectName = SqlServerEscapeHelper.EscapeStringAsLiteral(objectName); string catalogNameStringLiteral = CatalogName is null ? null : SqlServerEscapeHelper.EscapeStringAsLiteral(CatalogName); + bool resolveColumnAliases = ShouldResolveColumnAliases(); + string createColumnAliasesTableQuery = resolveColumnAliases + ? """ + +CREATE TABLE #Column_Aliases +( + [Canonical_Column_Name] SYSNAME, + [Canonical_Column_Id] INT, + [Aliased_Column_Name] SYSNAME +) +""" + : string.Empty; + string populateColumnAliasesQuery = resolveColumnAliases + ? $""" + + EXEC sp_executesql N' + INSERT INTO #Column_Aliases ([Canonical_Column_Name], [Canonical_Column_Id], [Aliased_Column_Name]) + SELECT [name], [column_id], ''$to_id'' FROM {catalogNameStringLiteral}.[sys].[all_columns] WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) = 8 + UNION ALL + SELECT [name], [column_id], ''$from_id'' FROM {catalogNameStringLiteral}.[sys].[all_columns] WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) = 5 + UNION ALL + SELECT [name], [column_id], ''$edge_id'' FROM {catalogNameStringLiteral}.[sys].[all_columns] WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) = 2 AND [name] LIKE ''$edge[_]id[_]%'' + UNION ALL + SELECT [name], [column_id], ''$node_id'' FROM {catalogNameStringLiteral}.[sys].[all_columns] WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) = 2 AND [name] LIKE ''$node[_]id[_]%''', + N'@Object_ID INT', @Object_ID = @Object_ID +""" + : string.Empty; + string removeShadowedColumnAliasesQuery = resolveColumnAliases + ? $""" + + DELETE FROM #Column_Aliases + WHERE [Aliased_Column_Name] IN (SELECT [name] FROM {CatalogName}.[sys].[all_columns] WHERE [object_id] = @Object_ID) +""" + : string.Empty; + string selectColumnAliasesQuery = resolveColumnAliases + ? """ + +SELECT [Canonical_Column_Name], [Aliased_Column_Name] +FROM #Column_Aliases +ORDER BY [Canonical_Column_Id] ASC + +DROP TABLE #Column_Aliases +""" + : string.Empty; // Specify the column names explicitly. This is to ensure that we can map to hidden // columns (e.g. columns in temporal tables.) If the target table doesn't exist, // OBJECT_ID will return NULL and @Column_Names will remain non-null. The subsequent @@ -543,13 +589,7 @@ private string CreateInitialQuery() DECLARE @Column_Name_Query NVARCHAR(MAX); DECLARE @Column_Names NVARCHAR(MAX) = NULL; DECLARE @Has_Sys_All_Columns_Permissions INT = HAS_PERMS_BY_NAME('{catalogNameStringLiteral}.[sys].[all_columns]', 'OBJECT', 'SELECT'); - -CREATE TABLE #Column_Aliases -( - [Canonical_Column_Name] SYSNAME, - [Canonical_Column_Id] INT, - [Aliased_Column_Name] SYSNAME -) +{createColumnAliasesTableQuery} IF CAST(SERVERPROPERTY('EngineEdition') AS INT) = 6 BEGIN @@ -567,17 +607,7 @@ IF CAST(SERVERPROPERTY('EngineEdition') AS INT) = 6 IF EXISTS (SELECT TOP 1 * FROM {CatalogName}.[sys].[all_columns] WHERE [object_id] = OBJECT_ID('{catalogNameStringLiteral}.[sys].[all_columns]') AND [name] = 'graph_type') BEGIN SET @Column_Name_Query_FILTER = N'WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) NOT IN (1, 3, 4, 6, 7)'; - - EXEC sp_executesql N' - INSERT INTO #Column_Aliases ([Canonical_Column_Name], [Canonical_Column_Id], [Aliased_Column_Name]) - SELECT [name], [column_id], ''$to_id'' FROM {catalogNameStringLiteral}.[sys].[all_columns] WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) = 8 - UNION ALL - SELECT [name], [column_id], ''$from_id'' FROM {catalogNameStringLiteral}.[sys].[all_columns] WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) = 5 - UNION ALL - SELECT [name], [column_id], ''$edge_id'' FROM {catalogNameStringLiteral}.[sys].[all_columns] WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) = 2 AND [name] LIKE ''$edge[_]id[_]%'' - UNION ALL - SELECT [name], [column_id], ''$node_id'' FROM {catalogNameStringLiteral}.[sys].[all_columns] WHERE [object_id] = @Object_ID AND COALESCE([graph_type], 0) = 2 AND [name] LIKE ''$node[_]id[_]%''', - N'@Object_ID INT', @Object_ID = @Object_ID +{populateColumnAliasesQuery} END ELSE BEGIN @@ -586,9 +616,7 @@ UNION ALL SET @Column_Name_Query = @Column_Name_Query_SELECT + ' FROM {catalogNameStringLiteral}.[sys].[all_columns] ' + @Column_Name_Query_FILTER + ' ' + @Column_Name_Query_SORT + ';' EXEC sp_executesql @Column_Name_Query, N'@Object_ID INT, @Column_Names NVARCHAR(MAX) OUTPUT', @Object_ID = @Object_ID, @Column_Names = @Column_Names OUTPUT; - - DELETE FROM #Column_Aliases - WHERE [Aliased_Column_Name] IN (SELECT [name] FROM {CatalogName}.[sys].[all_columns] WHERE [object_id] = @Object_ID) +{removeShadowedColumnAliasesQuery} END SELECT @Column_Names = COALESCE(@Column_Names, '*'); @@ -598,12 +626,7 @@ WHERE [Aliased_Column_Name] IN (SELECT [name] FROM {CatalogName}.[sys].[all_colu SET FMTONLY OFF; EXEC {CatalogName}..{TableCollationsStoredProc} N'{SchemaName}.{TableName}'; - -SELECT [Canonical_Column_Name], [Aliased_Column_Name] -FROM #Column_Aliases -ORDER BY [Canonical_Column_Id] ASC - -DROP TABLE #Column_Aliases +{selectColumnAliasesQuery} """; } @@ -614,7 +637,8 @@ DROP TABLE #Column_Aliases private Task CreateAndExecuteInitialQueryAsync(out BulkCopySimpleResultSet result) { // Check if we have valid cached metadata for the current destination table - if (CachedMetadata != null) + if (CachedMetadata != null + && (!ShouldResolveColumnAliases() || CachedMetadata.Count > ColumnAliasesResultId)) { SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName); result = CachedMetadata; @@ -714,44 +738,47 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i // Apply any necessary column aliases. If an aliased name exists in the // local column mappings but the canonical name does not, update them. - Result columnAliasResults = internalResults[ColumnAliasesResultId]; - for (int i = 0; i < columnAliasResults.Count; i++) + if (internalResults.Count > ColumnAliasesResultId) { - Row aliasRow = columnAliasResults[i]; - SqlString canonicalName = (SqlString)aliasRow[ColumnCanonicalNameColumnId]; - SqlString aliasedName = (SqlString)aliasRow[ColumnAliasColumnId]; - - if (canonicalName.IsNull || aliasedName.IsNull) + Result columnAliasResults = internalResults[ColumnAliasesResultId]; + for (int i = 0; i < columnAliasResults.Count; i++) { - continue; - } + Row aliasRow = columnAliasResults[i]; + SqlString canonicalName = (SqlString)aliasRow[ColumnCanonicalNameColumnId]; + SqlString aliasedName = (SqlString)aliasRow[ColumnAliasColumnId]; - string canonical = canonicalName.Value; - bool canonicalNameExists = unmatchedColumns.Contains(canonical) - // The destination columns might be escaped. If so, search for those instead - || unmatchedColumns.Contains(SqlServerEscapeHelper.EscapeIdentifier(canonical)); + if (canonicalName.IsNull || aliasedName.IsNull) + { + continue; + } - if (canonicalNameExists) - { - continue; - } + string canonical = canonicalName.Value; + bool canonicalNameExists = unmatchedColumns.Contains(canonical) + // The destination columns might be escaped. If so, search for those instead + || unmatchedColumns.Contains(SqlServerEscapeHelper.EscapeIdentifier(canonical)); - // The canonical name does not exist. Look for a local column mapping which matches - // the alias (or its escaped variant) and replace its name with its canonical name. - string alias = aliasedName.Value; - string escapedAlias = SqlServerEscapeHelper.EscapeIdentifier(alias); + if (canonicalNameExists) + { + continue; + } - for (int j = 0; j < _localColumnMappings.Count; j++) - { - if (unmatchedColumns.Comparer.Equals(_localColumnMappings[j].DestinationColumn, alias) - || unmatchedColumns.Comparer.Equals(_localColumnMappings[j].DestinationColumn, escapedAlias)) + // The canonical name does not exist. Look for a local column mapping which matches + // the alias (or its escaped variant) and replace its name with its canonical name. + string alias = aliasedName.Value; + string escapedAlias = SqlServerEscapeHelper.EscapeIdentifier(alias); + + for (int j = 0; j < _localColumnMappings.Count; j++) { - unmatchedColumns.Remove(_localColumnMappings[j].DestinationColumn); + if (unmatchedColumns.Comparer.Equals(_localColumnMappings[j].DestinationColumn, alias) + || unmatchedColumns.Comparer.Equals(_localColumnMappings[j].DestinationColumn, escapedAlias)) + { + unmatchedColumns.Remove(_localColumnMappings[j].DestinationColumn); - unmatchedColumns.Add(canonical); - _localColumnMappings[j].MappedDestinationColumn = canonical; + unmatchedColumns.Add(canonical); + _localColumnMappings[j].MappedDestinationColumn = canonical; - break; + break; + } } } } @@ -1632,6 +1659,33 @@ private void AppendColumnNameAndTypeName(StringBuilder query, string columnName, query.Append(typeName); } + private bool ShouldResolveColumnAliases() + { + if (_localColumnMappings is null) + { + return false; + } + + for (int i = 0; i < _localColumnMappings.Count; i++) + { + if (IsGraphColumnAlias(_localColumnMappings[i].DestinationColumn)) + { + return true; + } + } + + return false; + } + + private bool IsGraphColumnAlias(string name) + { + string unquotedName = UnquotedName(name); + return string.Equals(unquotedName, "$node_id", StringComparison.OrdinalIgnoreCase) + || string.Equals(unquotedName, "$edge_id", StringComparison.OrdinalIgnoreCase) + || string.Equals(unquotedName, "$from_id", StringComparison.OrdinalIgnoreCase) + || string.Equals(unquotedName, "$to_id", StringComparison.OrdinalIgnoreCase); + } + private string UnquotedName(string name) { if (string.IsNullOrEmpty(name)) @@ -2330,6 +2384,8 @@ private void WriteRowSourceToServerCommon(int columnCount) _localColumnMappings.ValidateCollection(); foreach (SqlBulkCopyColumnMapping bulkCopyColumn in _localColumnMappings) { + bulkCopyColumn.MappedDestinationColumn = null; + if (bulkCopyColumn._internalSourceColumnOrdinal == -1) { unspecifiedColumnOrdinals = true; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs index fd6b06bafb..ba94cc1274 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs @@ -45,10 +45,8 @@ public void Test() using (DbDataReader reader = srcCmd.ExecuteReader()) { IDictionary stats; - long expectedIduCount = DataTestUtility.IsAzureSynapse || DataTestUtility.IsAtLeastSQL2017() ? 2 : 1; - long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 4 : 13; - long expectedSelectRows = DataTestUtility.IsAzureSynapse ? 4 : 15; - long expectedTransactions = DataTestUtility.IsAzureSynapse || DataTestUtility.IsAtLeastSQL2017() ? 2 : 1; + long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 4 : 11; + long expectedSelectRows = DataTestUtility.IsAzureSynapse ? 4 : 14; using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { bulkcopy.DestinationTableName = dstTable; @@ -69,12 +67,12 @@ public void Test() DataTestUtility.AssertEqualsWithDescription((long)3, stats["BuffersReceived"], "Unexpected BuffersReceived value."); DataTestUtility.AssertEqualsWithDescription((long)3, stats["BuffersSent"], "Unexpected BuffersSent value."); - DataTestUtility.AssertEqualsWithDescription(expectedIduCount, stats["IduCount"], "Unexpected IduCount value."); + DataTestUtility.AssertEqualsWithDescription((long)0, stats["IduCount"], "Unexpected IduCount value."); DataTestUtility.AssertEqualsWithDescription(expectedSelectCount, stats["SelectCount"], "Unexpected SelectCount value."); DataTestUtility.AssertEqualsWithDescription((long)3, stats["ServerRoundtrips"], "Unexpected ServerRoundtrips value."); DataTestUtility.AssertEqualsWithDescription(expectedSelectRows, stats["SelectRows"], "Unexpected SelectRows value."); DataTestUtility.AssertEqualsWithDescription((long)2, stats["SumResultSets"], "Unexpected SumResultSets value."); - DataTestUtility.AssertEqualsWithDescription(expectedTransactions, stats["Transactions"], "Unexpected Transactions value."); + DataTestUtility.AssertEqualsWithDescription((long)0, stats["Transactions"], "Unexpected Transactions value."); } } } From acfaf72224a22f4128b4cef26ed065f62eb4d48f Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Thu, 13 Aug 2026 13:50:41 -0500 Subject: [PATCH 2/4] Fix for failing manual test --- .../tests/ManualTests/BulkCopy/CopyAllFromReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs index ba94cc1274..c6e2b71846 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs @@ -45,7 +45,7 @@ public void Test() using (DbDataReader reader = srcCmd.ExecuteReader()) { IDictionary stats; - long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 4 : 11; + long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 4 : 12; long expectedSelectRows = DataTestUtility.IsAzureSynapse ? 4 : 14; using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { From d7046247ee0e7196e02431d7ad331e0fcccc0597 Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Thu, 13 Aug 2026 18:37:43 -0500 Subject: [PATCH 3/4] Fix for the other manual test failure --- .../tests/ManualTests/BulkCopy/CopyAllFromReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs index c6e2b71846..a055fee30a 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs @@ -46,7 +46,7 @@ public void Test() { IDictionary stats; long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 4 : 12; - long expectedSelectRows = DataTestUtility.IsAzureSynapse ? 4 : 14; + long expectedSelectRows = DataTestUtility.IsAzureSynapse ? 4 : 15; using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { bulkcopy.DestinationTableName = dstTable; From fc7fdd564a078b7b483dc0a7d6fcf98dfb7f342d Mon Sep 17 00:00:00 2001 From: Ben Russell Date: Mon, 17 Aug 2026 14:23:30 -0500 Subject: [PATCH 4/4] Addressing PR comments (round 1) --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 52 +++-- .../ManualTests/BulkCopy/CopyAllFromReader.cs | 2 +- .../ManualTests/BulkCopy/SqlGraphTables.cs | 203 ++++++++++++++++++ .../SqlClient/SqlBulkCopyCacheMetadataTest.cs | 30 +++ 4 files changed, 269 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 106c9abd57..5e4e76b387 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -193,6 +193,7 @@ public SourceColumnMetadata(ValueMethod method, bool isSqlType, bool isDataFeed) private SqlBulkCopyColumnMappingCollection _columnMappings; private SqlBulkCopyColumnMappingCollection _localColumnMappings; + private bool _localColumnMappingsResolveAliases; private SqlConnection _connection; private SqlTransaction _internalTransaction; @@ -246,6 +247,7 @@ private int RowNumber // Metadata caching fields for CacheMetadata option internal BulkCopySimpleResultSet CachedMetadata { get; private set; } + private bool _cachedMetadataResolveAliases; // Per-operation clone of the destination table metadata, used when CacheMetadata is // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not // mutate the cached BulkCopySimpleResultSet. @@ -375,6 +377,7 @@ public string DestinationTableName } CachedMetadata = null; + _cachedMetadataResolveAliases = false; _destinationTableName = value; } } @@ -486,9 +489,7 @@ private string CreateInitialQuery() string objectName = ADP.BuildMultiPartName(parts); string escapedObjectName = SqlServerEscapeHelper.EscapeStringAsLiteral(objectName); string catalogNameStringLiteral = CatalogName is null ? null : SqlServerEscapeHelper.EscapeStringAsLiteral(CatalogName); - bool resolveColumnAliases = ShouldResolveColumnAliases(); - string createColumnAliasesTableQuery = resolveColumnAliases - ? """ + string createColumnAliasesTableQuery = """ CREATE TABLE #Column_Aliases ( @@ -496,9 +497,8 @@ CREATE TABLE #Column_Aliases [Canonical_Column_Id] INT, [Aliased_Column_Name] SYSNAME ) -""" - : string.Empty; - string populateColumnAliasesQuery = resolveColumnAliases +"""; + string populateColumnAliasesQuery = _localColumnMappingsResolveAliases ? $""" EXEC sp_executesql N' @@ -513,23 +513,21 @@ UNION ALL N'@Object_ID INT', @Object_ID = @Object_ID """ : string.Empty; - string removeShadowedColumnAliasesQuery = resolveColumnAliases + string removeShadowedColumnAliasesQuery = _localColumnMappingsResolveAliases ? $""" DELETE FROM #Column_Aliases WHERE [Aliased_Column_Name] IN (SELECT [name] FROM {CatalogName}.[sys].[all_columns] WHERE [object_id] = @Object_ID) """ : string.Empty; - string selectColumnAliasesQuery = resolveColumnAliases - ? """ + string selectColumnAliasesQuery = """ SELECT [Canonical_Column_Name], [Aliased_Column_Name] FROM #Column_Aliases ORDER BY [Canonical_Column_Id] ASC DROP TABLE #Column_Aliases -""" - : string.Empty; +"""; // Specify the column names explicitly. This is to ensure that we can map to hidden // columns (e.g. columns in temporal tables.) If the target table doesn't exist, // OBJECT_ID will return NULL and @Column_Names will remain non-null. The subsequent @@ -637,8 +635,7 @@ IF EXISTS (SELECT TOP 1 * FROM {CatalogName}.[sys].[all_columns] WHERE [object_i private Task CreateAndExecuteInitialQueryAsync(out BulkCopySimpleResultSet result) { // Check if we have valid cached metadata for the current destination table - if (CachedMetadata != null - && (!ShouldResolveColumnAliases() || CachedMetadata.Count > ColumnAliasesResultId)) + if (IsCachedMetadataValid()) { SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName); result = CachedMetadata; @@ -679,11 +676,18 @@ private Task CreateAndExecuteInitialQueryAsync(out Bulk } } + internal bool IsCachedMetadataValid() + { + return CachedMetadata != null + && (!_localColumnMappingsResolveAliases || _cachedMetadataResolveAliases); + } + private void CacheMetadataIfEnabled(BulkCopySimpleResultSet result) { if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata)) { CachedMetadata = result; + _cachedMetadataResolveAliases = _localColumnMappingsResolveAliases; SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CacheMetadataIfEnabled | Info | Cached metadata for table '{0}'", _destinationTableName); } } @@ -1095,6 +1099,7 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults) public void ClearCachedMetadata() { CachedMetadata = null; + _cachedMetadataResolveAliases = false; SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.ClearCachedMetadata | Info | Metadata cache cleared"); } @@ -1119,6 +1124,7 @@ private void Dispose(bool disposing) _columnMappings = null; _parser = null; CachedMetadata = null; + _cachedMetadataResolveAliases = false; _operationMetaData = null; try { @@ -1659,6 +1665,12 @@ private void AppendColumnNameAndTypeName(StringBuilder query, string columnName, query.Append(typeName); } + private void ResetLocalColumnMappings() + { + _localColumnMappings = null; + _localColumnMappingsResolveAliases = false; + } + private bool ShouldResolveColumnAliases() { if (_localColumnMappings is null) @@ -2385,7 +2397,10 @@ private void WriteRowSourceToServerCommon(int columnCount) foreach (SqlBulkCopyColumnMapping bulkCopyColumn in _localColumnMappings) { bulkCopyColumn.MappedDestinationColumn = null; + } + foreach (SqlBulkCopyColumnMapping bulkCopyColumn in _localColumnMappings) + { if (bulkCopyColumn._internalSourceColumnOrdinal == -1) { unspecifiedColumnOrdinals = true; @@ -2399,6 +2414,8 @@ private void WriteRowSourceToServerCommon(int columnCount) _localColumnMappings.CreateDefaultMapping(columnCount); } + _localColumnMappingsResolveAliases = ShouldResolveColumnAliases(); + // perf: If the user specified all column ordinals we do not need to get a schematable if (unspecifiedColumnOrdinals) { @@ -3113,7 +3130,7 @@ private void WriteToServerInternalRestContinuedAsync(BulkCopySimpleResultSet int // Bulk copy task is completed at this moment. if (task.IsCanceled) { - sqlBulkCopy._localColumnMappings = null; + sqlBulkCopy.ResetLocalColumnMappings(); try { sqlBulkCopy.CleanUpStateObject(); @@ -3125,11 +3142,12 @@ private void WriteToServerInternalRestContinuedAsync(BulkCopySimpleResultSet int } else if (task.Exception != null) { + sqlBulkCopy.ResetLocalColumnMappings(); source.SetException(task.Exception.InnerException); } else { - sqlBulkCopy._localColumnMappings = null; + sqlBulkCopy.ResetLocalColumnMappings(); try { sqlBulkCopy.CleanUpStateObject(isCancelRequested: false); @@ -3154,7 +3172,7 @@ private void WriteToServerInternalRestContinuedAsync(BulkCopySimpleResultSet int } else { - _localColumnMappings = null; + ResetLocalColumnMappings(); try { @@ -3173,7 +3191,7 @@ private void WriteToServerInternalRestContinuedAsync(BulkCopySimpleResultSet int } catch (Exception ex) when (ADP.IsCatchableExceptionType(ex)) { - _localColumnMappings = null; + ResetLocalColumnMappings(); try { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs index a055fee30a..67db8d5ae0 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs @@ -45,7 +45,7 @@ public void Test() using (DbDataReader reader = srcCmd.ExecuteReader()) { IDictionary stats; - long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 4 : 12; + long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 5 : 13; long expectedSelectRows = DataTestUtility.IsAzureSynapse ? 4 : 15; using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/SqlGraphTables.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/SqlGraphTables.cs index 12353e8695..39e99a5b1d 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/SqlGraphTables.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/SqlGraphTables.cs @@ -12,6 +12,39 @@ namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy [Trait("Set", "2")] public class SqlGraphTables { + /// + /// Verifies that copying to a SQL Graph table without graph aliases in the mappings does not + /// require alias resolution. + /// + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsAtLeastSQL2017))] + public void WriteToServer_CopyToSqlGraphNodeTableBySourceOrdinal_Succeeds() + { + string connectionString = DataTestUtility.TCPConnectionString; + + using SqlConnection dstConn = new(connectionString); + using DataTable nodes = new() + { + Columns = { new DataColumn("Name", typeof(string)) } + }; + + dstConn.Open(); + + for (int i = 0; i < 5; i++) + { + nodes.Rows.Add($"Name {i}"); + } + + using Table dstNodeTable = new(dstConn, "SqlGraphNodeTableByOrdinal", "([Name] VARCHAR(100)) AS NODE"); + using SqlBulkCopy nodeCopy = new(dstConn); + + nodeCopy.DestinationTableName = dstNodeTable.Name; + nodeCopy.ColumnMappings.Add(0, "Name"); + nodeCopy.WriteToServer(nodes); + + using SqlCommand verifyCommand = new($"SELECT COUNT(*) FROM {dstNodeTable.Name}", dstConn); + Assert.Equal(5, (int)verifyCommand.ExecuteScalar()); + } + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsAtLeastSQL2017))] public void WriteToServer_CopyToSqlGraphNodeTable_Succeeds() { @@ -170,5 +203,175 @@ public void WriteToServer_CopyToTableWithSameNameAsColumnAlias_Succeeds() } } } + + /// + /// Reuses one SqlBulkCopy after graph alias mappings have been resolved, then changes those + /// mappings to ordinary destination columns. This guards against stale resolved graph + /// canonical names being reused on the next operation. + /// + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsAtLeastSQL2017))] + public void WriteToServer_ReuseAfterAliasMappingsThenOrdinaryMappings_Succeeds() + { + string connectionString = DataTestUtility.TCPConnectionString; + + using SqlConnection dstConn = new(connectionString); + dstConn.Open(); + + using Table srcNodeTable = new(dstConn, "SqlGraph_ReuseAliasFirstNodes", "(Id INT PRIMARY KEY IDENTITY(1,1), [Name] VARCHAR(100)) AS NODE"); + using Table dstEdgeTable = new(dstConn, "SqlGraph_ReuseAliasFirstEdges", "([Description] VARCHAR(100)) AS EDGE"); + using Table dstNormalTable = new(dstConn, "SqlGraph_ReuseAliasFirstNormal", "([Description] VARCHAR(100), [ToId] NVARCHAR(MAX) NOT NULL, [FromId] NVARCHAR(MAX) NOT NULL)"); + using DataTable edges = CreateEdgeSourceData(dstConn, srcNodeTable.Name); + + using SqlBulkCopy edgeCopy = new(dstConn); + edgeCopy.ColumnMappings.Add("Description", "Description"); + edgeCopy.ColumnMappings.Add("ToId", "$to_id"); + edgeCopy.ColumnMappings.Add("FromId", "$from_id"); + + edgeCopy.DestinationTableName = dstEdgeTable.Name; + edgeCopy.WriteToServer(edges); + + edgeCopy.ColumnMappings[0].SourceColumn = "Description"; + edgeCopy.ColumnMappings[1].DestinationColumn = "ToId"; + edgeCopy.ColumnMappings[2].DestinationColumn = "FromId"; + edgeCopy.DestinationTableName = dstNormalTable.Name; + edgeCopy.WriteToServer(edges); + + VerifyNormalEdgeData(dstConn, dstNormalTable.Name, edges); + } + + /// + /// Reuses one SqlBulkCopy after ordinary mappings, then changes those mappings to graph + /// aliases. This ensures alias-resolution bypass state is recomputed for each operation. + /// + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsAtLeastSQL2017))] + public void WriteToServer_ReuseAfterOrdinaryMappingsThenAliasMappings_Succeeds() + { + string connectionString = DataTestUtility.TCPConnectionString; + + using SqlConnection dstConn = new(connectionString); + dstConn.Open(); + + using Table srcNodeTable = new(dstConn, "SqlGraph_ReuseOrdinaryFirstNodes", "(Id INT PRIMARY KEY IDENTITY(1,1), [Name] VARCHAR(100)) AS NODE"); + using Table dstNormalTable = new(dstConn, "SqlGraph_ReuseOrdinaryFirstNormal", "([Description] VARCHAR(100), [ToId] NVARCHAR(MAX) NOT NULL, [FromId] NVARCHAR(MAX) NOT NULL)"); + using Table dstEdgeTable = new(dstConn, "SqlGraph_ReuseOrdinaryFirstEdges", "([Description] VARCHAR(100)) AS EDGE"); + using DataTable edges = CreateEdgeSourceData(dstConn, srcNodeTable.Name); + + using SqlBulkCopy edgeCopy = new(dstConn); + edgeCopy.ColumnMappings.Add("Description", "Description"); + edgeCopy.ColumnMappings.Add("ToId", "ToId"); + edgeCopy.ColumnMappings.Add("FromId", "FromId"); + + edgeCopy.DestinationTableName = dstNormalTable.Name; + edgeCopy.WriteToServer(edges); + + edgeCopy.ColumnMappings[0].SourceColumn = "Description"; + edgeCopy.ColumnMappings[1].DestinationColumn = "$to_id"; + edgeCopy.ColumnMappings[2].DestinationColumn = "$from_id"; + edgeCopy.DestinationTableName = dstEdgeTable.Name; + edgeCopy.WriteToServer(edges); + + VerifyGraphEdgeData(dstConn, dstEdgeTable.Name, edges); + } + + /// + /// Exercises CacheMetadata with graph alias mappings against a real SQL Graph edge table, + /// including reuse of the cached alias result set on a later operation. + /// + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsAtLeastSQL2017))] + public void WriteToServer_CacheMetadataWithSqlGraphAliasMappings_Succeeds() + { + string connectionString = DataTestUtility.TCPConnectionString; + + using SqlConnection dstConn = new(connectionString); + dstConn.Open(); + + using Table srcNodeTable = new(dstConn, "SqlGraph_CacheAliasNodes", "(Id INT PRIMARY KEY IDENTITY(1,1), [Name] VARCHAR(100)) AS NODE"); + using Table dstEdgeTable = new(dstConn, "SqlGraph_CacheAliasEdges", "([Description] VARCHAR(100)) AS EDGE"); + using DataTable edges = CreateEdgeSourceData(dstConn, srcNodeTable.Name); + + using SqlBulkCopy cachedCopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + + cachedCopy.DestinationTableName = dstEdgeTable.Name; + cachedCopy.ColumnMappings.Add("Description", "Description"); + cachedCopy.ColumnMappings.Add("ToId", "$to_id"); + cachedCopy.ColumnMappings.Add("FromId", "$from_id"); + cachedCopy.WriteToServer(edges); + VerifyGraphEdgeData(dstConn, dstEdgeTable.Name, edges); + + cachedCopy.ColumnMappings[0].SourceColumn = "Description"; + cachedCopy.ColumnMappings[1].DestinationColumn = "$to_id"; + cachedCopy.ColumnMappings[2].DestinationColumn = "$from_id"; + cachedCopy.WriteToServer(edges); + VerifyGraphEdgeRowCount(dstConn, dstEdgeTable.Name, edges.Rows.Count * 2); + } + + private static DataTable CreateEdgeSourceData(SqlConnection connection, string nodeTableName) + { + using SqlCommand insertSampleNodes = new($"INSERT INTO {nodeTableName} ([Name]) VALUES ('A'), ('B'), ('C')", connection); + insertSampleNodes.ExecuteNonQuery(); + + DataTable edges = new() + { + Columns = + { + new DataColumn("Description", typeof(string)), + new DataColumn("ToId", typeof(string)), + new DataColumn("FromId", typeof(string)) + } + }; + + using SqlCommand nodeQuery = new($"SELECT $node_id FROM {nodeTableName} ORDER BY Id", connection); + using SqlDataReader reader = nodeQuery.ExecuteReader(); + Assert.True(reader.Read()); + string firstNodeId = reader.GetString(0); + + Assert.True(reader.Read()); + string secondNodeId = reader.GetString(0); + edges.Rows.Add("First edge", firstNodeId, secondNodeId); + + Assert.True(reader.Read()); + string thirdNodeId = reader.GetString(0); + edges.Rows.Add("Second edge", secondNodeId, thirdNodeId); + + return edges; + } + + private static void VerifyGraphEdgeData(SqlConnection connection, string edgeTableName, DataTable expectedEdges) + { + using SqlCommand verificationCommand = new($"SELECT [Description], $to_id, $from_id FROM {edgeTableName} ORDER BY [Description]", connection); + using SqlDataReader reader = verificationCommand.ExecuteReader(); + + foreach (DataRow expectedRow in expectedEdges.Select(filterExpression: null, sort: "Description")) + { + Assert.True(reader.Read()); + Assert.Equal(expectedRow["Description"], reader.GetString(0)); + Assert.Equal(expectedRow["ToId"], reader.GetString(1)); + Assert.Equal(expectedRow["FromId"], reader.GetString(2)); + } + + Assert.False(reader.Read()); + } + + private static void VerifyNormalEdgeData(SqlConnection connection, string tableName, DataTable expectedEdges) + { + using SqlCommand verificationCommand = new($"SELECT [Description], [ToId], [FromId] FROM {tableName} ORDER BY [Description]", connection); + using SqlDataReader reader = verificationCommand.ExecuteReader(); + + foreach (DataRow expectedRow in expectedEdges.Select(filterExpression: null, sort: "Description")) + { + Assert.True(reader.Read()); + Assert.Equal(expectedRow["Description"], reader.GetString(0)); + Assert.Equal(expectedRow["ToId"], reader.GetString(1)); + Assert.Equal(expectedRow["FromId"], reader.GetString(2)); + } + + Assert.False(reader.Read()); + } + + private static void VerifyGraphEdgeRowCount(SqlConnection connection, string edgeTableName, int expectedRows) + { + using SqlCommand countCommand = new($"SELECT COUNT(*) FROM {edgeTableName}", connection); + Assert.Equal(expectedRows, (int)countCommand.ExecuteScalar()); + } } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs index 90aa4a8f5e..5eef28c471 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -18,6 +18,20 @@ private static void SetCachedMetadata(SqlBulkCopy bulkCopy, BulkCopySimpleResult .SetValue(bulkCopy, value); } + private static void SetResolveColumnAliases(SqlBulkCopy bulkCopy, bool value) + { + typeof(SqlBulkCopy) + .GetField("_localColumnMappingsResolveAliases", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(bulkCopy, value); + } + + private static void SetCachedMetadataResolveAliases(SqlBulkCopy bulkCopy, bool value) + { + typeof(SqlBulkCopy) + .GetField("_cachedMetadataResolveAliases", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(bulkCopy, value); + } + [Fact] public void CacheMetadata_FlagValue_IsCorrect() { @@ -117,6 +131,22 @@ public void DestinationTableName_Change_ClearsCachedMetadata() Assert.Null(bulkCopy.CachedMetadata); } + [Theory] + [InlineData(false, false, true)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, true, true)] + public void IsCachedMetadataValid_DependsOnOperationAndCachedAliasResolution(bool resolveAliases, bool cachedMetadataResolveAliases, bool expected) + { + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); + + SetCachedMetadata(bulkCopy, new BulkCopySimpleResultSet()); + SetCachedMetadataResolveAliases(bulkCopy, cachedMetadataResolveAliases); + SetResolveColumnAliases(bulkCopy, resolveAliases); + + Assert.Equal(expected, bulkCopy.IsCachedMetadataValid()); + } + [Fact] public void Constructor_WithCacheMetadataOption_Succeeds() {