diff --git a/BUILDGUIDE.md b/BUILDGUIDE.md index 6298bbc9eb..ba20249392 100644 --- a/BUILDGUIDE.md +++ b/BUILDGUIDE.md @@ -497,7 +497,7 @@ The top-level flags control global runner behavior: | `WaitForProfiler` | Pauses at startup and prints the process ID so you can attach an external profiler (e.g. `dotnet-trace`) before benchmarks run. | | `UseNativeMemoryAndETWProfiler` | Attaches the `NativeMemoryProfiler` and `EtwProfiler` BenchmarkDotNet diagnosers. Windows only; has no effect on other OSes. | -Some benchmarks (e.g. `DataTypeReaderRunner`, `DataTypeReaderAsyncRunner`) also +Some benchmarks (e.g. `DataTypeReaderRunner`) also read per-type test values from `datatypes.json` in the `PerformanceTests` directory. Like `runnerconfig.jsonc`, this file's location can be overridden with the `DATATYPES_CONFIG` environment variable. diff --git a/eng/pipelines/perf/README.md b/eng/pipelines/perf/README.md index 09ad48788f..be925f5431 100644 --- a/eng/pipelines/perf/README.md +++ b/eng/pipelines/perf/README.md @@ -209,7 +209,7 @@ supplies the isolated dedicated host, the tuned SQL instance, and the disjoint c | Client CPU pin | Pins the benchmark process to `PERF_CLIENT_CPUS` (`taskset` on Linux, `ProcessorAffinity` on Windows). | | Fail loud | Preflight `SELECT 1` before any pass, **and** a post-pass guard that fails the run if a pass produced **zero** benchmark results — so an empty comparison can never be reported green. | | Warm-up | Touches the target DB in the preflight to warm the buffer pool / plan cache before the first measured benchmark. | -| Allocator tuning (Linux) | Exports `MALLOC_MMAP_THRESHOLD_=128MiB` and `MALLOC_TRIM_THRESHOLD_=-1` so large-buffer benches (`AsyncLargeDataRead`, `SqlBulkCopy`) stop re-`mmap`ing per iteration. | +| Allocator tuning (Linux) | Exports `MALLOC_MMAP_THRESHOLD_=128MiB` and `MALLOC_TRIM_THRESHOLD_=-1` so large-buffer benches (`LargeDataRead`, `SqlBulkCopy`) stop re-`mmap`ing per iteration. | | Network tuning (Linux) | Best-effort `sysctl` to widen the ephemeral port range and enable `tcp_tw_reuse` for churn benches (`ConnectionPoolStress`, `ParallelAsyncConnection`). Never fails the run. | | Diagnostics | Writes `results/diagnostics/`: SQL instance config (MAXDOP, memory, affinity, tempdb files, `@@VERSION`), host CPU topology, and per-pass CPU-clock/thermal telemetry (before/after each pass). | | Regression gate | `failOnRegression` threads `--fail-on-regression`; only a **candidate-slower** delta past the threshold fails, and in interleaved mode only after best-of-N confirmation. Default off. | diff --git a/eng/pipelines/perf/scripts/run-perf-tests.sh b/eng/pipelines/perf/scripts/run-perf-tests.sh index 4df2eec8e3..afa61525ce 100755 --- a/eng/pipelines/perf/scripts/run-perf-tests.sh +++ b/eng/pipelines/perf/scripts/run-perf-tests.sh @@ -285,7 +285,7 @@ DIAG_DIR="${RESULTS_DIR}/diagnostics" mkdir -p "${DIAG_DIR}" # --- §2.8 Allocator tuning (exported so the 'dotnet run' children inherit it) --------------------- -# Large-buffer benches (AsyncLargeDataRead, SqlBulkCopy) re-mmap a big buffer every iteration under +# Large-buffer benches (LargeDataRead, SqlBulkCopy) re-mmap a big buffer every iteration under # glibc malloc; keep those allocations on the heap and stop trimming freed pages so they are reused, # which removes a major source of per-iteration variance. export MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-134217728}" # 128 MiB diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnMasterKeyCertificateFixture.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnMasterKeyCertificateFixture.cs index c4fbbad613..d396251103 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnMasterKeyCertificateFixture.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnMasterKeyCertificateFixture.cs @@ -22,6 +22,8 @@ public ColumnMasterKeyCertificateFixture() public X509Certificate2? ColumnMasterKeyCertificate { get; } + public string? ColumnMasterKeyCertificatePath { get; } + protected ColumnMasterKeyCertificateFixture(bool createCertificate) { if (createCertificate) @@ -29,6 +31,8 @@ protected ColumnMasterKeyCertificateFixture(bool createCertificate) ColumnMasterKeyCertificate = CreateCertificate(nameof(ColumnMasterKeyCertificate), Array.Empty(), Array.Empty()); AddToStore(ColumnMasterKeyCertificate, StoreLocation.CurrentUser, StoreName.My); + + ColumnMasterKeyCertificatePath = $"{StoreLocation.CurrentUser}/{StoreName.My}/{ColumnMasterKeyCertificate.Thumbprint}"; } } } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnEncryptionKey.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnEncryptionKey.cs new file mode 100644 index 0000000000..af402428b0 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnEncryptionKey.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Security.Cryptography; + +namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +/// +/// A column encryption key, created at the start of its scope and dropped when disposed. +/// +public sealed class ColumnEncryptionKey : DatabaseObject +{ + private const int PlaintextKeyLength = 32; + + private const string DefinitionTemplate = "CREATE COLUMN ENCRYPTION KEY {0} WITH VALUES" + + " (COLUMN_MASTER_KEY = {1}, ALGORITHM = 'RSA_OAEP', ENCRYPTED_VALUE = 0x{2})"; + + private ColumnMasterKey ColumnMasterKey => State; + + /// + /// Initializes a new instance of the ColumnEncryptionKey class using the specified SQL connection, + /// name and a column master key. + /// + /// The SQL connection used to interact with the database. + /// The column encryption key name. + /// The column master key which backs this encryption key. + public ColumnEncryptionKey(SqlConnection connection, string namePrefix, ColumnMasterKey cmkOrigin) + : base(connection, GenerateLongName(namePrefix), definition: DefinitionTemplate, + state: cmkOrigin, shouldCreate: true, shouldDrop: true) + { + } + + protected override void CreateObject(string definition) + { + string encryptedValue; + + using (RandomNumberGenerator rnd = RandomNumberGenerator.Create()) + { + byte[] randomPlaintext = new byte[PlaintextKeyLength]; + byte[] encryptedPlaintext; + + rnd.GetBytes(randomPlaintext); + encryptedPlaintext = ColumnMasterKey.Encrypt(randomPlaintext); + + encryptedValue = BitConverter.ToString(encryptedPlaintext).Replace("-", ""); + } + + definition = string.Format(definition, Name, ColumnMasterKey.Name, encryptedValue); + using SqlCommand createCommand = new(definition, Connection); + + createCommand.ExecuteNonQuery(); + } + + protected override void DropObject() + { + using SqlCommand dropCommand = new($"IF EXISTS (SELECT 1 FROM sys.column_encryption_keys where name = @Name) DROP COLUMN ENCRYPTION KEY {Name}", Connection); + dropCommand.Parameters.AddWithValue("@Name", UnescapedName); + + dropCommand.ExecuteNonQuery(); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnMasterKey.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnMasterKey.cs new file mode 100644 index 0000000000..de607c0f68 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnMasterKey.cs @@ -0,0 +1,179 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text; + +namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +/// +/// A column master key, created at the start of its scope and dropped when disposed. +/// +public abstract class ColumnMasterKey : DatabaseObject +{ + private const string DefinitionTemplate = "CREATE COLUMN MASTER KEY {0} WITH (KEY_STORE_PROVIDER_NAME = '{1}', KEY_PATH = '{2}'{3})"; + + public sealed class CreationParameters + { + public SqlColumnEncryptionKeyStoreProvider Provider { get; } + + public string ProviderName { get; } + + public string KeyPath { get; } + + public bool AllowEnclaveComputations { get; } + + internal CreationParameters(SqlColumnEncryptionKeyStoreProvider provider, + string providerName, + string keyPath, + bool allowEnclaveComputations) + { + Provider = provider; + ProviderName = providerName; + KeyPath = keyPath; + AllowEnclaveComputations = allowEnclaveComputations; + } + } + + protected ColumnMasterKey(SqlConnection connection, string namePrefix, CreationParameters creationParameters) + : base(connection, name: GenerateLongName(namePrefix), definition: DefinitionTemplate, + state: creationParameters, shouldCreate: true, shouldDrop: true) + { + } + + protected override void CreateObject(string definition) + { + string enclaveStatement; + + if (State.AllowEnclaveComputations) + { + byte[] signature = State.Provider.SignColumnMasterKeyMetadata(State.KeyPath, State.AllowEnclaveComputations); + string signatureString = BitConverter.ToString(signature).Replace("-", ""); + + enclaveStatement = ", ENCLAVE_COMPUTATIONS (SIGNATURE = 0x" + signatureString + ")"; + } + else + { + enclaveStatement = string.Empty; + } + + definition = string.Format(definition, Name, State.ProviderName, State.KeyPath, enclaveStatement); + + using SqlCommand createCommand = new(definition, Connection); + + createCommand.ExecuteNonQuery(); + } + + protected override void DropObject() + { + using SqlCommand dropCommand = new($"IF EXISTS (SELECT 1 FROM sys.column_master_keys where name = @Name) DROP COLUMN MASTER KEY {Name}", Connection); + dropCommand.Parameters.AddWithValue("@Name", UnescapedName); + + dropCommand.ExecuteNonQuery(); + } + + public byte[] Encrypt(byte[] columnEncryptionKey) => + State.Provider.EncryptColumnEncryptionKey(State.KeyPath, "RSA_OAEP", columnEncryptionKey); + + public byte[] Decrypt(byte[] encryptedColumnEncryptionKey) => + State.Provider.DecryptColumnEncryptionKey(State.KeyPath, "RSA_OAEP", encryptedColumnEncryptionKey); +} + +/// +/// A column master key backed by a Cryptographic Service Provider. Created at the start of its +/// scope and dropped when disposed. +/// +public sealed class CspProviderBackedColumnMasterKey : ColumnMasterKey +{ + /// + /// Initializes a new instance of the CspProviderBackedColumnMasterKey class using the specified + /// SQL connection, name and a certificate containing a CSP-backed private key. + /// + /// + /// + /// If a column master key with the specified name already exists, it will be dropped automatically + /// before creation. + /// + /// + /// This column master key will be backed by the class. + /// + /// + /// The SQL connection used to interact with the database. + /// The column master key name. + /// The certificate to wrap. Must contain a CSP-backed private key. + /// true to enable enclave computations. + public CspProviderBackedColumnMasterKey(SqlConnection connection, string namePrefix, + CspCertificateFixture cspProvider, bool allowEnclaveComputations) + : base(connection, namePrefix, GenerateCreationParameters(cspProvider, allowEnclaveComputations)) + { + } + + private static CreationParameters GenerateCreationParameters(CspCertificateFixture cspProvider, bool allowEnclaveComputations) => + new(provider: new SqlColumnEncryptionCspProvider(), + providerName: SqlColumnEncryptionCspProvider.ProviderName, + cspProvider.CspKeyPath ?? throw new InvalidOperationException("Certificate lacks a CSP key."), + allowEnclaveComputations); +} + +/// +/// A column master key backed by a certificate. Created at the start of its scope and dropped when disposed. +/// +public sealed class CertificateBackedColumnMasterKey : ColumnMasterKey +{ + /// + /// Initializes a new instance of the CertificateBackedColumnMasterKey class using the specified + /// SQL connection, name and a certificate. + /// + /// + /// + /// If a column master key with the specified name already exists, it will be dropped automatically + /// before creation. + /// + /// + /// This column master key will be backed by the + /// class. + /// + /// + /// The SQL connection used to interact with the database. + /// The column master key name. + /// The certificate to wrap. Must contain a private key. + /// true to enable enclave computations. + public CertificateBackedColumnMasterKey(SqlConnection connection, string namePrefix, + CspCertificateFixture cspCertificate, bool allowEnclaveComputations) + : base(connection, namePrefix, GenerateCreationParameters(cspCertificate.CspCertificatePath, allowEnclaveComputations)) + { + } + + /// + /// Initializes a new instance of the ColumnMasterKey class using the specified SQL connection, + /// name and a certificate. + /// + /// + /// + /// If a column master key with the specified name already exists, it will be dropped automatically + /// before creation. + /// + /// + /// This column master key will be backed by the + /// class. + /// + /// + /// The SQL connection used to interact with the database. + /// The column master key name. + /// The certificate to wrap. Must contain a private key. + /// true to enable enclave computations. + public CertificateBackedColumnMasterKey(SqlConnection connection, string namePrefix, + ColumnMasterKeyCertificateFixture cmkCertificate, bool allowEnclaveComputations) + : base(connection, namePrefix, GenerateCreationParameters( + cmkCertificate.ColumnMasterKeyCertificatePath + ?? throw new InvalidOperationException("Certificate has not been created."), + allowEnclaveComputations)) + { + } + + private static CreationParameters GenerateCreationParameters(string certificatePath, bool allowEnclaveComputations) => + new(provider: new SqlColumnEncryptionCertificateStoreProvider(), + providerName: SqlColumnEncryptionCertificateStoreProvider.ProviderName, + certificatePath, + allowEnclaveComputations); +} diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Microsoft.Data.SqlClient.TestCommon.csproj b/src/Microsoft.Data.SqlClient/tests/Common/Microsoft.Data.SqlClient.TestCommon.csproj index de44a0c64b..77b1d1163b 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Microsoft.Data.SqlClient.TestCommon.csproj +++ b/src/Microsoft.Data.SqlClient/tests/Common/Microsoft.Data.SqlClient.TestCommon.csproj @@ -37,8 +37,31 @@ + + + + + + + + + Condition="'$(ReferenceType)' == 'Package' and '$(MdsPackageVersion)' == ''" /> + + + + + + + + + + + + + diff --git a/src/Microsoft.Data.SqlClient/tests/Common/SqlDataReaderExtensions.cs b/src/Microsoft.Data.SqlClient/tests/Common/SqlDataReaderExtensions.cs index 8851cb5210..daa690ce2d 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/SqlDataReaderExtensions.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/SqlDataReaderExtensions.cs @@ -45,7 +45,8 @@ public static void FlushResultSet(this SqlDataReader dataReader) { while (dataReader.Read()) { - // Discard results. + // Read all row data and discard. + _ = dataReader.IsDBNull(0); } } @@ -58,7 +59,8 @@ public static async Task FlushResultSetAsync(this SqlDataReader dataReader) { while (await dataReader.ReadAsync()) { - // Discard results. + // Read all row data and discard. + _ = await dataReader.IsDBNullAsync(0); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/AsyncLargeDataReadRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/AsyncLargeDataReadRunner.cs deleted file mode 100644 index 851b5b8d66..0000000000 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/AsyncLargeDataReadRunner.cs +++ /dev/null @@ -1,119 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Data; -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; - -namespace Microsoft.Data.SqlClient.PerformanceTests -{ - /// - /// Benchmarks for sync vs async reading of large VARBINARY(MAX) values. - /// Reproduces issues #593 and #1562. - /// - public class AsyncLargeDataReadRunner : BaseRunner - { - private string _tableName; - private string _connectionString; - - /// - /// Size of the data to read in bytes. - /// - [Params(1_048_576, 5_242_880, 10_485_760, 20_971_520)] - public int DataSizeBytes { get; set; } - - /// - /// Size of the client-side read buffer used to drain the VARBINARY(MAX) column. - /// Kept small (8 KB) and large (1 MB) to observe whether buffer size relative to - /// the payload materially changes throughput. - /// - [Params(8_192, 1_048_576)] - public int ReadBufferBytes { get; set; } - - [GlobalSetup] - public void Setup() - { - _connectionString = s_config.ConnectionString; - string machineHash = ((uint)Environment.MachineName.GetHashCode()).ToString("x8"); - _tableName = $"[perf_AsyncLargeData_{machineHash}_{Guid.NewGuid():N}]"; - - using var conn = new SqlConnection(_connectionString); - conn.Open(); - - using var createCmd = new SqlCommand( - $"CREATE TABLE {_tableName} (Id INT IDENTITY PRIMARY KEY, Data VARBINARY(MAX))", conn); - createCmd.ExecuteNonQuery(); - - // Generate the payload entirely server-side via CRYPT_GEN_RANDOM so we don't - // allocate a multi-megabyte byte[] on the client and don't ship the payload - // over the wire just to seed the benchmark. - using var insertCmd = new SqlCommand( - $@"INSERT INTO {_tableName} (Data) - SELECT SUBSTRING( - CONVERT( - varbinary(max), - REPLICATE( - CONVERT(varchar(max), CRYPT_GEN_RANDOM(8000), 2), - (@dataSizeBytes + 7999) / 8000 - ), - 2 - ), - 1, - @dataSizeBytes - );", conn); - insertCmd.Parameters.Add("@dataSizeBytes", SqlDbType.Int).Value = DataSizeBytes; - insertCmd.ExecuteNonQuery(); - } - - [GlobalCleanup] - public void Cleanup() - { - using var conn = new SqlConnection(_connectionString); - conn.Open(); - using var cmd = new SqlCommand($"DROP TABLE IF EXISTS {_tableName}", conn); - cmd.ExecuteNonQuery(); - SqlConnection.ClearAllPools(); - } - - [Benchmark] - public void ReadLargeDataSync() - { - using var conn = new SqlConnection(_connectionString); - conn.Open(); - using var cmd = new SqlCommand($"SELECT Data FROM {_tableName}", conn); - using var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess); - while (reader.Read()) - { - byte[] buffer = new byte[ReadBufferBytes]; - long offset = 0; - long bytesRead; - do - { - bytesRead = reader.GetBytes(0, offset, buffer, 0, buffer.Length); - offset += bytesRead; - } while (bytesRead > 0); - } - } - - [Benchmark] - public async Task ReadLargeDataAsync() - { - using var conn = new SqlConnection(_connectionString); - await conn.OpenAsync(); - using var cmd = new SqlCommand($"SELECT Data FROM {_tableName}", conn); - using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess); - while (await reader.ReadAsync()) - { - using var stream = reader.GetStream(0); - byte[] buffer = new byte[ReadBufferBytes]; - int bytesRead; - do - { - bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length); - } while (bytesRead > 0); - } - } - } -} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderAsyncRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderAsyncRunner.cs deleted file mode 100644 index b74a277258..0000000000 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderAsyncRunner.cs +++ /dev/null @@ -1,130 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; -using static Microsoft.Data.SqlClient.PerformanceTests.Constants; - -namespace Microsoft.Data.SqlClient.PerformanceTests -{ - public class DataTypeReaderAsyncRunner : BaseRunner - { - private static long s_rowCount; - private static string _query(string name) => $"SELECT * FROM {name}"; - - [GlobalSetup] - public static void Setup() - { - s_rowCount = s_config.Benchmarks.DataTypeReaderRunnerConfig.RowCount; - } - - [GlobalCleanup] - public static void Dispose() - { - SqlConnection.ClearAllPools(); - } - - [IterationCleanup] - public static void ResetConnection() - { - SqlConnection.ClearAllPools(); - } - - private static async Task RunBenchmarkAsync(DataType type) - { - using SqlConnection sqlConnection = new(s_config.ConnectionString); - sqlConnection.Open(); - Table t = Table.Build(nameof(SqlCommandRunner)) - .AddColumn(new Column(type)) - .CreateTable(sqlConnection) - .InsertBulkRows(s_rowCount, sqlConnection); - try - { - using SqlCommand sqlCommand = new(_query(t.Name), sqlConnection); - using SqlDataReader reader = await sqlCommand.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - { } - } - finally - { - t.DropTable(sqlConnection); - } - } - - [Benchmark] - public static Task BitAsync() => RunBenchmarkAsync(s_datatypes.Numerics[n_bit]); - - [Benchmark] - public static Task IntAsync() => RunBenchmarkAsync(s_datatypes.Numerics[n_int]); - - [Benchmark] - public static Task TinyIntAsync() => RunBenchmarkAsync(s_datatypes.Numerics[n_tinyint]); - - [Benchmark] - public static Task SmallIntAsync() => RunBenchmarkAsync(s_datatypes.Numerics[n_smallint]); - - [Benchmark] - public static Task BigIntAsync() => RunBenchmarkAsync(s_datatypes.Numerics[n_bigint]); - - [Benchmark] - public static Task MoneyAsync() => RunBenchmarkAsync(s_datatypes.Numerics[n_money]); - - [Benchmark] - public static Task SmallMoneyAsync() => RunBenchmarkAsync(s_datatypes.Numerics[n_smallmoney]); - - [Benchmark] - public static Task DecimalAsync() => RunBenchmarkAsync(s_datatypes.Decimals[d_decimal]); - - [Benchmark] - public static Task NumericAsync() => RunBenchmarkAsync(s_datatypes.Decimals[d_numeric]); - - [Benchmark] - public static Task FloatAsync() => RunBenchmarkAsync(s_datatypes.Decimals[d_float]); - - [Benchmark] - public static Task RealAsync() => RunBenchmarkAsync(s_datatypes.Decimals[d_real]); - - [Benchmark] - public static Task DateAsync() => RunBenchmarkAsync(s_datatypes.DateTimes[t_date]); - - [Benchmark] - public static Task DatetimeAsync() => RunBenchmarkAsync(s_datatypes.DateTimes[t_datetime]); - - [Benchmark] - public static Task Datetime2Async() => RunBenchmarkAsync(s_datatypes.DateTimes[t_datetime2]); - - [Benchmark] - public static Task TimeAsync() => RunBenchmarkAsync(s_datatypes.DateTimes[t_time]); - - [Benchmark] - public static Task SmallDateTimeAsync() => RunBenchmarkAsync(s_datatypes.DateTimes[t_smalldatetime]); - - [Benchmark] - public static Task DateTimeOffsetAsync() => RunBenchmarkAsync(s_datatypes.DateTimes[t_datetimeoffset]); - - [Benchmark] - public static Task CharAsync() => RunBenchmarkAsync(s_datatypes.Characters[c_char]); - - [Benchmark] - public static Task NCharAsync() => RunBenchmarkAsync(s_datatypes.Characters[c_nchar]); - - [Benchmark] - public static Task BinaryAsync() => RunBenchmarkAsync(s_datatypes.Binary[b_binary]); - - [Benchmark] - public static Task VarCharAsync() => RunBenchmarkAsync(s_datatypes.MaxTypes[m_varchar]); - - [Benchmark] - public static Task NVarCharAsync() => RunBenchmarkAsync(s_datatypes.MaxTypes[m_nvarchar]); - - [Benchmark] - public static Task VarBinaryAsync() => RunBenchmarkAsync(s_datatypes.MaxTypes[m_varbinary]); - - [Benchmark] - public static Task UniqueIdentifierAsync() => RunBenchmarkAsync(s_datatypes.Others[o_uniqueidentifier]); - - [Benchmark] - public static Task XmlAsync() => RunBenchmarkAsync(s_datatypes.Others[o_xml]); - } -} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner.cs deleted file mode 100644 index 00834425c8..0000000000 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner.cs +++ /dev/null @@ -1,126 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using BenchmarkDotNet.Attributes; -using static Microsoft.Data.SqlClient.PerformanceTests.Constants; - -namespace Microsoft.Data.SqlClient.PerformanceTests -{ - public class DataTypeReaderRunner : BaseRunner - { - private static SqlConnection s_sqlConnection; - private static long s_rowCount; - - private static string _query(string name) => $"SELECT * FROM {name}"; - - [GlobalSetup] - public static void Setup() - { - s_sqlConnection = new(s_config.ConnectionString); - s_sqlConnection.Open(); - s_rowCount = s_config.Benchmarks.DataTypeReaderRunnerConfig.RowCount; - } - - [GlobalCleanup] - public static void Dispose() - { - s_sqlConnection.Close(); - SqlConnection.ClearAllPools(); - } - - private static void RunBenchmark(DataType type) - { - Table t = Table.Build(nameof(SqlCommandRunner)) - .AddColumn(new Column(type)) - .CreateTable(s_sqlConnection) - .InsertBulkRows(s_rowCount, s_sqlConnection); - try - { - using SqlCommand sqlCommand = new(_query(t.Name), s_sqlConnection); - using SqlDataReader reader = sqlCommand.ExecuteReader(); - while (reader.Read()) - { } - } - finally - { - t.DropTable(s_sqlConnection); - } - } - - [Benchmark] - public static void Bit() => RunBenchmark(s_datatypes.Numerics[n_bit]); - - [Benchmark] - public static void Int() => RunBenchmark(s_datatypes.Numerics[n_int]); - - [Benchmark] - public static void TinyInt() => RunBenchmark(s_datatypes.Numerics[n_tinyint]); - - [Benchmark] - public static void SmallInt() => RunBenchmark(s_datatypes.Numerics[n_smallint]); - - [Benchmark] - public static void BigInt() => RunBenchmark(s_datatypes.Numerics[n_bigint]); - - [Benchmark] - public static void Money() => RunBenchmark(s_datatypes.Numerics[n_money]); - - [Benchmark] - public static void SmallMoney() => RunBenchmark(s_datatypes.Numerics[n_smallmoney]); - - [Benchmark] - public static void Decimal() => RunBenchmark(s_datatypes.Decimals[d_decimal]); - - [Benchmark] - public static void Numeric() => RunBenchmark(s_datatypes.Decimals[d_numeric]); - - [Benchmark] - public static void Float() => RunBenchmark(s_datatypes.Decimals[d_float]); - - [Benchmark] - public static void Real() => RunBenchmark(s_datatypes.Decimals[d_real]); - - [Benchmark] - public static void Date() => RunBenchmark(s_datatypes.DateTimes[t_date]); - - [Benchmark] - public static void Datetime() => RunBenchmark(s_datatypes.DateTimes[t_datetime]); - - [Benchmark] - public static void Datetime2() => RunBenchmark(s_datatypes.DateTimes[t_datetime2]); - - [Benchmark] - public static void Time() => RunBenchmark(s_datatypes.DateTimes[t_time]); - - [Benchmark] - public static void SmallDateTime() => RunBenchmark(s_datatypes.DateTimes[t_smalldatetime]); - - [Benchmark] - public static void DateTimeOffset() => RunBenchmark(s_datatypes.DateTimes[t_datetimeoffset]); - - [Benchmark] - public static void Char() => RunBenchmark(s_datatypes.Characters[c_char]); - - [Benchmark] - public static void NChar() => RunBenchmark(s_datatypes.Characters[c_nchar]); - - [Benchmark] - public static void Binary() => RunBenchmark(s_datatypes.Binary[b_binary]); - - [Benchmark] - public static void VarChar() => RunBenchmark(s_datatypes.MaxTypes[m_varchar]); - - [Benchmark] - public static void NVarChar() => RunBenchmark(s_datatypes.MaxTypes[m_nvarchar]); - - [Benchmark] - public static void VarBinary() => RunBenchmark(s_datatypes.MaxTypes[m_varbinary]); - - [Benchmark] - public static void UniqueIdentifier() => RunBenchmark(s_datatypes.Others[o_uniqueidentifier]); - - [Benchmark] - public static void Xml() => RunBenchmark(s_datatypes.Others[o_xml]); - } -} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/AlwaysEncrypted.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/AlwaysEncrypted.cs new file mode 100644 index 0000000000..9bcc20549d --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/AlwaysEncrypted.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +namespace Microsoft.Data.SqlClient.PerformanceTests.BenchmarkRunners.DataTypeReaderRunner; + +public class AlwaysEncrypted : DataTypeReaderRunnerBase +{ + private ColumnMasterKeyCertificateFixture _cmkCertificate; + private ColumnMasterKey _masterKey; + private ColumnEncryptionKey _encryptionKey; + + public override IEnumerable ExecutedTypes => AvailableTypes.Where(t => t.EncryptionSupported); + + protected override RunnerJob Configuration => s_config.Benchmarks.AlwaysEncryptedDataTypeReaderRunnerConfig; + + protected override SqlConnection OpenConnection() + { + SqlConnectionStringBuilder builder = new(s_config.ConnectionString) + { + ColumnEncryptionSetting = SqlConnectionColumnEncryptionSetting.Enabled + }; + SqlConnection conn = new(builder.ToString()); + + conn.Open(); + return conn; + } + + protected override Table CreateTable() + { + _cmkCertificate = new ColumnMasterKeyCertificateFixture(); + _masterKey = new CertificateBackedColumnMasterKey(_connection, nameof(_masterKey), _cmkCertificate, false); + _encryptionKey = new ColumnEncryptionKey(_connection, nameof(AlwaysEncrypted), _masterKey); + + return Table.Build(Type.Name) + .AddColumn(new Column(Type, encryptionKey: _encryptionKey)) + .CreateTable(_connection); + } + + protected override void OnCleanup() + { + using (_cmkCertificate) + using (_masterKey) + using (_encryptionKey) + { + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/DataTypeReaderRunnerBase.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/DataTypeReaderRunnerBase.cs new file mode 100644 index 0000000000..56c6487533 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/DataTypeReaderRunnerBase.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using Microsoft.Data.SqlClient.Tests.Common; + +namespace Microsoft.Data.SqlClient.PerformanceTests.BenchmarkRunners.DataTypeReaderRunner; + +public abstract class DataTypeReaderRunnerBase : BaseRunner +{ + protected SqlConnection _connection; + protected Table _table; + + public abstract IEnumerable ExecutedTypes { get; } + + [ParamsSource(nameof(ExecutedTypes))] + public DataType Type { get; set; } + + protected IEnumerable AvailableTypes => + s_datatypes.Others + .Concat(s_datatypes.Numerics) + .Concat(s_datatypes.Decimals) + .Concat(s_datatypes.DateTimes) + .Concat(s_datatypes.Characters) + .Concat(s_datatypes.Binary) + .Concat(s_datatypes.MaxTypes); + + protected abstract RunnerJob Configuration { get; } + + protected abstract SqlConnection OpenConnection(); + + protected abstract Table CreateTable(); + + protected virtual void OnCleanup() { } + + [GlobalSetup] + public void Setup() + { + long rowCount = Configuration.RowCount; + + _connection = OpenConnection(); + + _table = CreateTable() + .InsertBulkRows(rowCount, _connection); + } + + [GlobalCleanup] + public void Cleanup() + { + using (_connection) + { + _table.DropTable(_connection); + + OnCleanup(); + } + + SqlConnection.ClearAllPools(); + } + + [IterationCleanup] + public void ResetConnection() + { + SqlConnection.ClearAllPools(); + } + + [Benchmark] + public async Task ReadAsync() + { + await using SqlCommand sqlCommand = new($"SELECT * FROM {_table.Name}", _connection); + await using SqlDataReader reader = await sqlCommand.ExecuteReaderAsync(); + + await reader.FlushResultSetAsync(); + } + + [Benchmark] + public void Read() + { + using SqlCommand sqlCommand = new($"SELECT * FROM {_table.Name}", _connection); + using SqlDataReader reader = sqlCommand.ExecuteReader(); + + reader.FlushResultSet(); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/Plaintext.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/Plaintext.cs new file mode 100644 index 0000000000..e3d8829896 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/DataTypeReaderRunner/Plaintext.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace Microsoft.Data.SqlClient.PerformanceTests.BenchmarkRunners.DataTypeReaderRunner; + +public class Plaintext : DataTypeReaderRunnerBase +{ + public override IEnumerable ExecutedTypes => AvailableTypes; + + protected override RunnerJob Configuration => s_config.Benchmarks.DataTypeReaderRunnerConfig; + + protected override SqlConnection OpenConnection() + { + SqlConnection conn = new(s_config.ConnectionString); + + conn.Open(); + return conn; + } + + protected override Table CreateTable() => + Table.Build(Type.Name) + .AddColumn(new Column(Type)) + .CreateTable(_connection); +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/AlwaysEncrypted.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/AlwaysEncrypted.cs new file mode 100644 index 0000000000..09e789767e --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/AlwaysEncrypted.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Data; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +namespace Microsoft.Data.SqlClient.PerformanceTests.BenchmarkRunners.LargeDataReadRunner; + +public class AlwaysEncrypted : LargeDataReadRunnerBase +{ + private ColumnMasterKeyCertificateFixture _cmkCertificate; + private ColumnMasterKey _masterKey; + private ColumnEncryptionKey _encryptionKey; + + public override IEnumerable ExecutedCommandBehaviors => [CommandBehavior.Default]; + + protected override SqlConnection OpenConnection() + { + SqlConnectionStringBuilder builder = new(s_config.ConnectionString) + { + ColumnEncryptionSetting = SqlConnectionColumnEncryptionSetting.Enabled + }; + SqlConnection conn = new(builder.ToString()); + + conn.Open(); + return conn; + } + + protected override Tests.Common.Fixtures.DatabaseObjects.Table CreateTable() + { + _cmkCertificate = new ColumnMasterKeyCertificateFixture(); + _masterKey = new CertificateBackedColumnMasterKey(_connection, nameof(_masterKey), _cmkCertificate, false); + _encryptionKey = new ColumnEncryptionKey(_connection, nameof(AlwaysEncrypted), _masterKey); + + return new Tests.Common.Fixtures.DatabaseObjects.Table(_connection, nameof(AlwaysEncrypted), + "(" + + "Id INT IDENTITY PRIMARY KEY," + + "Data VARBINARY(MAX) ENCRYPTED WITH" + + "(" + + $"COLUMN_ENCRYPTION_KEY = {_encryptionKey.Name}," + + "ENCRYPTION_TYPE = DETERMINISTIC," + + "ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256'" + + ")" + + ")"); + } + + protected override void OnCleanup() + { + using (_cmkCertificate) + using (_masterKey) + using (_encryptionKey) + { + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/LargeDataReadRunnerBase.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/LargeDataReadRunnerBase.cs new file mode 100644 index 0000000000..f34be9f546 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/LargeDataReadRunnerBase.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Data; +using System.Data.SqlTypes; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Data.SqlClient.PerformanceTests.BenchmarkRunners.LargeDataReadRunner; + +/// +/// Benchmarks for sync vs async reading of large VARBINARY(MAX) values. +/// Reproduces issues #593 and #1562. +/// +public abstract class LargeDataReadRunnerBase : BaseRunner +{ + protected SqlConnection _connection; + protected Tests.Common.Fixtures.DatabaseObjects.Table _table; + + public abstract IEnumerable ExecutedCommandBehaviors { get; } + + /// + /// Size of the data to read in bytes. + /// + [Params(1_048_576, 5_242_880, 10_485_760, 20_971_520)] + public int DataSizeBytes { get; set; } + + /// + /// CommandBehavior to use when executing the reader. + /// SequentialAccess is expected to be faster for large payloads. Default is included + /// to facilitate comparison with Always Encrypted (which doesn't support SequentialAccess.) + /// + [ParamsSource(nameof(ExecutedCommandBehaviors))] + public CommandBehavior CommandBehavior { get; set; } + + protected abstract SqlConnection OpenConnection(); + + protected abstract Tests.Common.Fixtures.DatabaseObjects.Table CreateTable(); + + protected virtual void OnCleanup() { } + + [GlobalSetup] + public void Setup() + { + _connection = OpenConnection(); + + _table = CreateTable(); + + // Cannot generate the payload server-side (and avoid a multi-megabyte byte[] allocation + // on the client) because Always Encrypted values must be generated client-side. + using SqlCommand insertCmd = new($"INSERT INTO {_table.Name} (Data) VALUES (@data)", _connection); + insertCmd.Parameters.Add("@data", SqlDbType.VarBinary, -1).Value = new byte[DataSizeBytes]; + insertCmd.ExecuteNonQuery(); + } + + [GlobalCleanup] + public void Cleanup() + { + using (_connection) + { + try + { + _table.Dispose(); + } + finally + { + OnCleanup(); + } + } + + SqlConnection.ClearAllPools(); + } + + [Benchmark] + public void ReadLargeDataSync_GetFieldValue() + { + using SqlCommand cmd = new($"SELECT Data FROM {_table.Name}", _connection); + using SqlDataReader reader = cmd.ExecuteReader(CommandBehavior); + + while (reader.Read()) + { + _ = reader.GetFieldValue(0); + } + } + + [Benchmark] + public async Task ReadLargeDataAsync_GetFieldValue() + { + await using SqlCommand cmd = new($"SELECT Data FROM {_table.Name}", _connection); + await using SqlDataReader reader = await cmd.ExecuteReaderAsync(CommandBehavior); + + while (await reader.ReadAsync()) + { + _ = await reader.GetFieldValueAsync(0); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/Plaintext.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/Plaintext.cs new file mode 100644 index 0000000000..17ea039de1 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/LargeDataReadRunner/Plaintext.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Data.SqlClient.PerformanceTests.BenchmarkRunners.LargeDataReadRunner; + +public class Plaintext : LargeDataReadRunnerBase +{ + public override IEnumerable ExecutedCommandBehaviors => + [CommandBehavior.Default, CommandBehavior.SequentialAccess]; + + /// + /// Size of the client-side read buffer used to drain the VARBINARY(MAX) column. + /// Kept small (8 KB) and large (1 MB) to observe whether buffer size relative to + /// the payload materially changes throughput. + /// + public IEnumerable ReadBufferBytes => [8_192, 1_048_576]; + + protected override SqlConnection OpenConnection() + { + SqlConnection conn = new(s_config.ConnectionString); + + conn.Open(); + return conn; + } + + protected override Tests.Common.Fixtures.DatabaseObjects.Table CreateTable() => + new(_connection, nameof(Plaintext), "(Id INT IDENTITY PRIMARY KEY, Data VARBINARY(MAX))"); + + [Benchmark] + [ArgumentsSource(nameof(ReadBufferBytes))] + public void ReadLargeDataSync_GetBytes(int readBufferBytes) + { + using SqlCommand cmd = new($"SELECT Data FROM {_table.Name}", _connection); + using SqlDataReader reader = cmd.ExecuteReader(CommandBehavior); + byte[] buffer = new byte[readBufferBytes]; + + while (reader.Read()) + { + long offset = 0; + long bytesRead; + do + { + bytesRead = reader.GetBytes(0, offset, buffer, 0, buffer.Length); + offset += bytesRead; + } while (bytesRead > 0); + } + } + + [Benchmark] + [ArgumentsSource(nameof(ReadBufferBytes))] + public async Task ReadLargeDataAsync_GetStream(int readBufferBytes) + { + await using SqlCommand cmd = new($"SELECT Data FROM {_table.Name}", _connection); + await using SqlDataReader reader = await cmd.ExecuteReaderAsync(CommandBehavior); + byte[] buffer = new byte[readBufferBytes]; + + while (await reader.ReadAsync()) + { + await using Stream stream = reader.GetStream(0); + int bytesRead; + + do + { + bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length); + } while (bytesRead > 0); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/BenchmarkConfig.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/BenchmarkConfig.cs index 9a53f97651..01fd580651 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/BenchmarkConfig.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/BenchmarkConfig.cs @@ -2,11 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Json; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.InProcess.Emit; #if WINDOWS using BenchmarkDotNet.Diagnostics.Windows; @@ -25,6 +28,32 @@ public static class BenchmarkConfig /// public static bool UseNativeMemoryAndEtwProfiler { get; set; } + /// + /// Builds the in-process toolchain for a benchmark, honouring + /// . + /// + /// The in-process toolchain is required (rather than merely convenient): the + /// AppContext switches applied in only + /// affect the process that runs Main, so an out-of-process toolchain would run the + /// benchmarks with those switches at their default values. + /// + private static IToolchain BuildToolchain(RunnerJob runnerJob) => + runnerJob.TimeoutMinutes > 0 + ? new InProcessEmitToolchain( + TimeSpan.FromMinutes(runnerJob.TimeoutMinutes), + logOutput: true) + : InProcessEmitToolchain.Instance; + + /// + /// Resolves the configured name, defaulting to + /// when unset or unrecognised. + /// + private static RunStrategy ResolveRunStrategy(RunnerJob runnerJob) => + !string.IsNullOrWhiteSpace(runnerJob.RunStrategy) + && Enum.TryParse(runnerJob.RunStrategy, ignoreCase: true, out RunStrategy strategy) + ? strategy + : RunStrategy.Throughput; + public static ManualConfig s_instance(RunnerJob runnerJob) { ManualConfig config = DefaultConfig.Instance @@ -37,13 +66,13 @@ public static ManualConfig s_instance(RunnerJob runnerJob) // pipeline can translate results into the Kusto performance-results schema. .AddExporter(JsonExporter.Full) .AddJob( - Job.MediumRun.WithToolchain(InProcessEmitToolchain.Instance) + Job.MediumRun.WithToolchain(BuildToolchain(runnerJob)) .WithLaunchCount(runnerJob.LaunchCount) .WithInvocationCount(runnerJob.InvocationCount) .WithIterationCount(runnerJob.IterationCount) .WithWarmupCount(runnerJob.WarmupCount) .WithUnrollFactor(1) - .WithStrategy(BenchmarkDotNet.Engines.RunStrategy.Throughput) + .WithStrategy(ResolveRunStrategy(runnerJob)) .WithEnvironmentVariable("COMPlus_gcServer", "1") ) .WithOptions(ConfigOptions.JoinSummary); diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs index 2bbd1f1c23..7b136c6f2c 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -59,8 +59,9 @@ public class Benchmarks public RunnerJob SqlCommandRunnerConfig; public RunnerJob SqlBulkCopyRunnerConfig; public RunnerJob DataTypeReaderRunnerConfig; - public RunnerJob DataTypeReaderAsyncRunnerConfig; - public RunnerJob AsyncLargeDataReadRunnerConfig; + public RunnerJob AlwaysEncryptedDataTypeReaderRunnerConfig; + public RunnerJob LargeDataReadRunnerConfig; + public RunnerJob AlwaysEncryptedLargeDataReadRunnerConfig; public RunnerJob MarsOverheadRunnerConfig; public RunnerJob ParallelAsyncConnectionRunnerConfig; public RunnerJob CancellationTokenReadAsyncRunnerConfig; @@ -80,5 +81,40 @@ public class RunnerJob public int InvocationCount; public int WarmupCount; public long RowCount; + + /// + /// Per-benchmark execution timeout, in minutes, for the in-process toolchain. + /// + /// BenchmarkDotNet's in-process executor aborts a benchmark case that exceeds its + /// timeout with "takes too long to run. Prefer to use out-of-process toolchains for + /// long-running benchmarks." The built-in default is 5 minutes, which is not enough + /// for benchmarks whose single operation is inherently slow (for example reading a + /// 20 MB VARBINARY(MAX) value 20 times, plus the additional iterations that + /// MemoryDiagnoser and ThreadingDiagnoser each add). + /// + /// Switching to an out-of-process toolchain - BenchmarkDotNet's own suggestion - is + /// deliberately NOT an option for this suite: the AppContext switches configured in + /// (managed SNI, connection pool V2, + /// optimized async behaviour) only apply to the process that runs Main. An + /// out-of-process toolchain spawns a generated host where those switches revert to + /// their defaults, so the benchmark would silently measure the wrong code paths. + /// + /// When 0 or negative, BenchmarkDotNet's default timeout is used. + /// + public int TimeoutMinutes; + + /// + /// BenchmarkDotNet name to use for + /// this benchmark, for example "Throughput" or "Monitoring". + /// + /// "Throughput" (the default) is designed for fast operations and spends extra + /// iterations measuring and subtracting harness overhead. For benchmarks whose single + /// operation takes seconds, that overhead measurement is pure cost and is dwarfed by + /// the operation itself, so "Monitoring" - which skips it and simply runs the + /// requested iterations - is both faster and more appropriate. + /// + /// When null or empty, "Throughput" is used. + /// + public string RunStrategy; } } diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs index 54b59a1015..bd764571aa 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -52,6 +52,7 @@ public class DataType /// public object DefaultValue; public string Name; + public bool EncryptionSupported; public override string ToString() => Name; } @@ -121,6 +122,7 @@ public class ValueFormatType : DataType public class MaxLengthValueType : DataType { public int MaxLength; + public bool CharacterType; } public class MaxLengthBinaryType : MaxLengthValueType diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/DBFramework/Column.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/DBFramework/Column.cs index 3caed947c9..84b7c1a2ce 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/DBFramework/Column.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/DBFramework/Column.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data; using System.Text; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; namespace Microsoft.Data.SqlClient.PerformanceTests { @@ -12,18 +13,28 @@ public class Column public string Name; public DataType Type; public object Value; + public ColumnEncryptionKey EncryptionKey; - public Column(DataType type, string prefix = null, object value = null) + public Column(DataType type, string prefix = null, object value = null, ColumnEncryptionKey encryptionKey = null) { Type = type; Name = (prefix ?? "c_") + type.Name; Value = value ?? Type.DefaultValue; + EncryptionKey = Type.EncryptionSupported ? encryptionKey : null; } - public string QueryString - { - get => new StringBuilder(Name).Append(' ').Append(Type.ToString()).ToString(); - } + public string QueryString => + EncryptionKey is null + ? $"{Name} {Type}" + : $"{Name} {Type} {EncryptedCollation}ENCRYPTED WITH " + + $"(COLUMN_ENCRYPTION_KEY = {EncryptionKey.Name}," + + $"ENCRYPTION_TYPE = DETERMINISTIC," + + $"ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256')"; + + private string EncryptedCollation => + Type is MaxLengthValueType maxLengthValueType && maxLengthValueType.CharacterType + ? "COLLATE Latin1_General_BIN2 " + : string.Empty; public DataColumn AsDataColumn() => new(Name); } diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj index 77fb6130dc..1acb981002 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj @@ -74,6 +74,8 @@ + + diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs index bf42ebaf75..b5ef4d76c6 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -42,9 +42,10 @@ public BenchmarkUnit(string name, Func selector, Type run new BenchmarkUnit("SqlConnection", b => b.SqlConnectionRunnerConfig, typeof(SqlConnectionRunner)), new BenchmarkUnit("SqlCommand", b => b.SqlCommandRunnerConfig, typeof(SqlCommandRunner)), new BenchmarkUnit("SqlBulkCopy", b => b.SqlBulkCopyRunnerConfig, typeof(SqlBulkCopyRunner)), - new BenchmarkUnit("DataTypeReader", b => b.DataTypeReaderRunnerConfig, typeof(DataTypeReaderRunner)), - new BenchmarkUnit("DataTypeReaderAsync", b => b.DataTypeReaderAsyncRunnerConfig, typeof(DataTypeReaderAsyncRunner)), - new BenchmarkUnit("AsyncLargeDataRead", b => b.AsyncLargeDataReadRunnerConfig, typeof(AsyncLargeDataReadRunner)), + new BenchmarkUnit("DataTypeReader", b => b.DataTypeReaderRunnerConfig, typeof(BenchmarkRunners.DataTypeReaderRunner.Plaintext)), + new BenchmarkUnit("AlwaysEncryptedDataTypeReader", b => b.AlwaysEncryptedDataTypeReaderRunnerConfig, typeof(BenchmarkRunners.DataTypeReaderRunner.AlwaysEncrypted)), + new BenchmarkUnit("LargeDataRead", b => b.LargeDataReadRunnerConfig, typeof(BenchmarkRunners.LargeDataReadRunner.Plaintext)), + new BenchmarkUnit("AlwaysEncryptedLargeDataRead", b => b.AlwaysEncryptedLargeDataReadRunnerConfig, typeof(BenchmarkRunners.LargeDataReadRunner.AlwaysEncrypted)), new BenchmarkUnit("MarsOverhead", b => b.MarsOverheadRunnerConfig, typeof(MarsOverheadRunner)), new BenchmarkUnit("ParallelAsyncConnection", b => b.ParallelAsyncConnectionRunnerConfig, typeof(ParallelAsyncConnectionRunner)), new BenchmarkUnit("CancellationTokenReadAsync", b => b.CancellationTokenReadAsyncRunnerConfig, typeof(CancellationTokenReadAsyncRunner)), diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/datatypes.json b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/datatypes.json index f99664c86b..0eb444650d 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/datatypes.json +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/datatypes.json @@ -1,46 +1,53 @@ -{ +{ "Numerics": [ { "Name": "bit", "MinValue": false, "MaxValue": true, - "DefaultValue": true + "DefaultValue": true, + "EncryptionSupported": true }, { "Name": "int", "MinValue": -2147483648, "MaxValue": 2147483647, - "DefaultValue": 123456 + "DefaultValue": 123456, + "EncryptionSupported": true }, { "Name": "tinyint", "MinValue": 0, "MaxValue": 255, - "DefaultValue": 123 + "DefaultValue": 123, + "EncryptionSupported": true }, { "Name": "smallint", "MinValue": -32768, "MaxValue": 32767, - "DefaultValue": 1234 + "DefaultValue": 1234, + "EncryptionSupported": true }, { "Name": "bigint", "MinValue": -9223372036854775808, "MaxValue": 9223372036854775807, - "DefaultValue": 1234567890 + "DefaultValue": 1234567890, + "EncryptionSupported": true }, { "Name": "money", "MinValue": -922337203685477.58, "MaxValue": 922337203685477.58, - "DefaultValue": 12334567.89 + "DefaultValue": 12334567.89, + "EncryptionSupported": true }, { "Name": "smallmoney", "MinValue": -214748.3648, "MaxValue": 214748.3647, - "DefaultValue": 123345.67 + "DefaultValue": 123345.67, + "EncryptionSupported": true } ], "Decimals": [ @@ -50,7 +57,8 @@ "MaxScale": 38, "Precision": 10, "Scale": 4, - "DefaultValue": 12345.6789 + "DefaultValue": 12345.6789, + "EncryptionSupported": true }, { "Name": "numeric", @@ -58,7 +66,8 @@ "MaxScale": 38, "Precision": 10, "Scale": 4, - "DefaultValue": 12345.6789 + "DefaultValue": 12345.6789, + "EncryptionSupported": true }, { "Name": "float", @@ -66,7 +75,8 @@ "MaxScale": -1, "Precision": 10, "Scale": -1, - "DefaultValue": 12345.6789 + "DefaultValue": 12345.6789, + "EncryptionSupported": true }, { "Name": "real", @@ -74,85 +84,106 @@ "MaxScale": -1, "Precision": 10, "Scale": -1, - "DefaultValue": 12345.6789 + "DefaultValue": 12345.6789, + "EncryptionSupported": true } ], "DateTimes": [ { "Name": "date", "DefaultValue": "1970-01-01", - "Format": "yyyy-mm-dd" + "Format": "yyyy-mm-dd", + "EncryptionSupported": true }, { "Name": "datetime", "DefaultValue": "1970-01-01 00:00:00", - "Format": "yyyy-mm-dd hh:mm:ss[.nnn]" + "Format": "yyyy-mm-dd hh:mm:ss[.nnn]", + "EncryptionSupported": true }, { "Name": "datetime2", "DefaultValue": "9999-12-31 23:59:59.9999999", - "Format": "yyyy-mm-dd hh:mm:ss[.nnnnnnn]" + "Format": "yyyy-mm-dd hh:mm:ss[.nnnnnnn]", + "EncryptionSupported": true }, { "Name": "time", "DefaultValue": "23:59:59.9999999", - "Format": "hh:mm:ss[.nnnnnnn]" + "Format": "hh:mm:ss[.nnnnnnn]", + "EncryptionSupported": true }, { "Name": "smalldatetime", "DefaultValue": "1970-01-01 00:00:00", - "Format": "yyyy-mm-dd hh:mm:ss" + "Format": "yyyy-mm-dd hh:mm:ss", + "EncryptionSupported": true }, { "Name": "datetimeoffset", "DefaultValue": "1970-01-01 00:00:00 +00:00", - "Format": "YYYY-MM-DD hh:mm:ss[.nnnnnnn] [{+|-}hh:mm]" + "Format": "YYYY-MM-DD hh:mm:ss[.nnnnnnn] [{+|-}hh:mm]", + "EncryptionSupported": true } ], "Characters": [ { "Name": "char", "MaxLength": 1, - "DefaultValue": "a" + "DefaultValue": "a", + "EncryptionSupported": true, + "CharacterType": true }, { "Name": "nchar", "MaxLength": 1, - "DefaultValue": "Ş" + "DefaultValue": "Ş", + "EncryptionSupported": true, + "CharacterType": true } ], "Binary": [ { "Name": "binary", "MaxLength": 8000, - "DefaultValue": "0x0001e240" + "DefaultValue": "0x0001e240", + "EncryptionSupported": true, + "CharacterType": false } ], "MaxTypes": [ { "Name": "varchar", "MaxLength": 8000, - "DefaultValue": "abcde12345 .,!@#" + "DefaultValue": "abcde12345 .,!@#", + "EncryptionSupported": true, + "CharacterType": true }, { "Name": "nvarchar", "MaxLength": 8000, - "DefaultValue": "abcde12345 .,!@ŞԛȽClient" + "DefaultValue": "abcde12345 .,!@ŞԛȽClient", + "EncryptionSupported": true, + "CharacterType": true }, { "Name": "varbinary", "MaxLength": 8000, - "DefaultValue": "0x0001e240" + "DefaultValue": "0x0001e240", + "EncryptionSupported": true, + "CharacterType": false } ], "Others": [ { "Name": "uniqueidentifier", - "DefaultValue": "6F9619FF-8B86-D011-B42D-00C04FC964FF" + "DefaultValue": "6F9619FF-8B86-D011-B42D-00C04FC964FF", + "EncryptionSupported": true }, { "Name": "xml", - "DefaultValue": "

Hello World

" + "DefaultValue": "

Hello World

", + "EncryptionSupported": false } ] } diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc index 5777db9652..5df16e3ff5 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc @@ -22,6 +22,17 @@ // This section contains the configuration for each benchmark. // You can enable/disable benchmarks and configure the number of iterations, invocations, // warmup runs, and row count for each benchmark. + // + // Two optional per-benchmark settings are available for slow benchmarks: + // "TimeoutMinutes" - overrides BenchmarkDotNet's 5 minute in-process execution timeout. + // Omit (or set 0) to keep the default. Benchmarks that exceed it fail + // with "takes too long to run. Prefer to use out-of-process + // toolchains for long-running benchmarks." Note that switching to an + // out-of-process toolchain is NOT viable for this suite, because the + // AppContext switches above only apply to the process running Main. + // "RunStrategy" - "Throughput" (default) or "Monitoring". Use "Monitoring" when a + // single operation takes seconds, to skip the harness-overhead + // measurement that "Throughput" performs. "Benchmarks": { "SqlConnectionRunnerConfig": { "Enabled": true, @@ -55,7 +66,7 @@ "WarmupCount": 1, "RowCount": 1000 }, - "DataTypeReaderAsyncRunnerConfig": { + "AlwaysEncryptedDataTypeReaderRunnerConfig": { "Enabled": true, "LaunchCount": 1, "IterationCount": 20, @@ -63,13 +74,33 @@ "WarmupCount": 1, "RowCount": 1000 }, - "AsyncLargeDataReadRunnerConfig": { + // The large-data read benchmarks move up to 20 MB per operation, so a single + // operation takes seconds rather than microseconds. Three settings account for that: + // IterationCount is reduced (20 network-bound iterations add little statistical + // value over 5, but cost 4x the wall time); + // RunStrategy is "Monitoring" so BenchmarkDotNet skips the harness-overhead + // measurement that "Throughput" performs, which is meaningless at this scale; + // TimeoutMinutes raises the in-process executor's 5 minute default, which these + // benchmarks otherwise exceed and fail with "takes too long to run". + "LargeDataReadRunnerConfig": { "Enabled": true, "LaunchCount": 1, - "IterationCount": 20, + "IterationCount": 5, "InvocationCount": 1, "WarmupCount": 1, - "RowCount": 0 + "RowCount": 0, + "RunStrategy": "Monitoring", + "TimeoutMinutes": 30 + }, + "AlwaysEncryptedLargeDataReadRunnerConfig": { + "Enabled": true, + "LaunchCount": 1, + "IterationCount": 5, + "InvocationCount": 1, + "WarmupCount": 1, + "RowCount": 0, + "RunStrategy": "Monitoring", + "TimeoutMinutes": 30 }, "MarsOverheadRunnerConfig": { "Enabled": true,