From f42a364453d9dc4f3d2fddfc3a2bf2ad64cd554d Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 27 Jul 2023 21:17:54 +0200 Subject: [PATCH 1/7] To prevent lock escalation to page/table locks the number of rows must not be over 4.000 Google states multiple resources that either going over 4.000 or 5.000 rows results into lock escalation from row to page/table locks --- src/SqlPersistence/Outbox/OutboxPersister.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SqlPersistence/Outbox/OutboxPersister.cs b/src/SqlPersistence/Outbox/OutboxPersister.cs index a880f46dd..5efa84d14 100644 --- a/src/SqlPersistence/Outbox/OutboxPersister.cs +++ b/src/SqlPersistence/Outbox/OutboxPersister.cs @@ -20,7 +20,7 @@ class OutboxPersister : IOutboxStorage public OutboxPersister(IConnectionManager connectionManager, SqlDialect sqlDialect, OutboxCommands outboxCommands, Func outboxTransactionFactory, - int cleanupBatchSize = 10000) + int cleanupBatchSize = 4000) { this.connectionManager = connectionManager; this.sqlDialect = sqlDialect; @@ -144,4 +144,4 @@ public async Task RemoveEntriesOlderThan(DateTime dateTime, CancellationToken ca } } } -} \ No newline at end of file +} From 28498ba047b0f21d1d80b2dda0bf62201707f882 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 27 Jul 2023 19:23:29 +0000 Subject: [PATCH 2/7] Add `with (rowlock)` to outbox delete query to prevent lock escalation and prevent dead-locks with regular outbox operations as outbox deletes do not have to be efficient --- src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs b/src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs index fe95f8eec..b5ef66352 100644 --- a/src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs +++ b/src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs @@ -76,7 +76,7 @@ internal override string GetOutboxPessimisticCompleteCommand(string tableName) internal override string GetOutboxCleanupCommand(string tableName) { return $@" -delete top (@BatchSize) from {tableName} +delete top (@BatchSize) from {tableName} with (rowlock) where Dispatched = 'true' and DispatchedAt < @DispatchedBefore"; } From e2b9bee261ae2234009b83ab8047c67e11cbf290 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 31 Jul 2023 12:58:26 +0200 Subject: [PATCH 3/7] =?UTF-8?q?approvals=20=F0=9F=A5=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ApprovalFiles/OutboxCommandTests.Cleanup.MsSql.approved.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SqlPersistence.Tests/ApprovalFiles/OutboxCommandTests.Cleanup.MsSql.approved.txt b/src/SqlPersistence.Tests/ApprovalFiles/OutboxCommandTests.Cleanup.MsSql.approved.txt index c44e2fa0e..fb8f9ac97 100644 --- a/src/SqlPersistence.Tests/ApprovalFiles/OutboxCommandTests.Cleanup.MsSql.approved.txt +++ b/src/SqlPersistence.Tests/ApprovalFiles/OutboxCommandTests.Cleanup.MsSql.approved.txt @@ -1,4 +1,4 @@  -delete top (@BatchSize) from [TheSchema].[TheTablePrefixOutboxData] +delete top (@BatchSize) from [TheSchema].[TheTablePrefixOutboxData] with (rowlock) where Dispatched = 'true' and DispatchedAt < @DispatchedBefore \ No newline at end of file From d3c5dda8593ce2ed4ec9445a4bec9779b27e4513 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 1 Aug 2023 09:44:28 +0200 Subject: [PATCH 4/7] Added code comments to explain the number value and query hint --- src/SqlPersistence/Outbox/OutboxPersister.cs | 2 +- src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SqlPersistence/Outbox/OutboxPersister.cs b/src/SqlPersistence/Outbox/OutboxPersister.cs index 5efa84d14..ef5e752a3 100644 --- a/src/SqlPersistence/Outbox/OutboxPersister.cs +++ b/src/SqlPersistence/Outbox/OutboxPersister.cs @@ -20,7 +20,7 @@ class OutboxPersister : IOutboxStorage public OutboxPersister(IConnectionManager connectionManager, SqlDialect sqlDialect, OutboxCommands outboxCommands, Func outboxTransactionFactory, - int cleanupBatchSize = 4000) + int cleanupBatchSize = 4000) // Keep below 4000 to prevent lock escalation { this.connectionManager = connectionManager; this.sqlDialect = sqlDialect; diff --git a/src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs b/src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs index b5ef66352..21723ed56 100644 --- a/src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs +++ b/src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs @@ -75,6 +75,7 @@ internal override string GetOutboxPessimisticCompleteCommand(string tableName) internal override string GetOutboxCleanupCommand(string tableName) { + // Rowlock hint to prevent lock escalation which can result in INSERT's and competing cleanup to dead-lock return $@" delete top (@BatchSize) from {tableName} with (rowlock) where Dispatched = 'true' and From 9d43f8d9d603ccc9b7295aae022e3ddd93966355 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 1 Aug 2023 09:51:52 +0200 Subject: [PATCH 5/7] Hardcode cleanup batch size --- src/SqlPersistence/Outbox/OutboxPersister.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/SqlPersistence/Outbox/OutboxPersister.cs b/src/SqlPersistence/Outbox/OutboxPersister.cs index ef5e752a3..d4c8ab904 100644 --- a/src/SqlPersistence/Outbox/OutboxPersister.cs +++ b/src/SqlPersistence/Outbox/OutboxPersister.cs @@ -14,19 +14,16 @@ class OutboxPersister : IOutboxStorage { readonly IConnectionManager connectionManager; readonly SqlDialect sqlDialect; - readonly int cleanupBatchSize; readonly OutboxCommands outboxCommands; readonly Func outboxTransactionFactory; public OutboxPersister(IConnectionManager connectionManager, SqlDialect sqlDialect, OutboxCommands outboxCommands, - Func outboxTransactionFactory, - int cleanupBatchSize = 4000) // Keep below 4000 to prevent lock escalation + Func outboxTransactionFactory) { this.connectionManager = connectionManager; this.sqlDialect = sqlDialect; this.outboxCommands = outboxCommands; this.outboxTransactionFactory = outboxTransactionFactory; - this.cleanupBatchSize = cleanupBatchSize; } public Task BeginTransaction(ContextBag context, CancellationToken cancellationToken = default) @@ -126,6 +123,8 @@ public Task Store(OutboxMessage message, IOutboxTransaction outboxTransaction, C public async Task RemoveEntriesOlderThan(DateTime dateTime, CancellationToken cancellationToken = default) { + const int CleanupBatchSize = 4000; // Keep below 4000 to prevent lock escalation + using (var connection = await connectionManager.OpenNonContextualConnection(cancellationToken).ConfigureAwait(false)) { var continuePurging = true; @@ -137,11 +136,11 @@ public async Task RemoveEntriesOlderThan(DateTime dateTime, CancellationToken ca { command.CommandText = outboxCommands.Cleanup; command.AddParameter("DispatchedBefore", dateTime); - command.AddParameter("BatchSize", cleanupBatchSize); + command.AddParameter("BatchSize", CleanupBatchSize); var rowCount = await command.ExecuteNonQueryEx(cancellationToken).ConfigureAwait(false); continuePurging = rowCount != 0; } } } } -} +} \ No newline at end of file From e72979775c22fc607bea76fc622d974270ebbea8 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 1 Aug 2023 09:55:54 +0200 Subject: [PATCH 6/7] whitespace --- src/SqlPersistence/Outbox/OutboxPersister.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SqlPersistence/Outbox/OutboxPersister.cs b/src/SqlPersistence/Outbox/OutboxPersister.cs index d4c8ab904..3ae081908 100644 --- a/src/SqlPersistence/Outbox/OutboxPersister.cs +++ b/src/SqlPersistence/Outbox/OutboxPersister.cs @@ -18,7 +18,7 @@ class OutboxPersister : IOutboxStorage readonly Func outboxTransactionFactory; public OutboxPersister(IConnectionManager connectionManager, SqlDialect sqlDialect, OutboxCommands outboxCommands, - Func outboxTransactionFactory) + Func outboxTransactionFactory) { this.connectionManager = connectionManager; this.sqlDialect = sqlDialect; From 90a1e777716652f2999b2f71d2b0bed3db68602d Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 1 Aug 2023 10:32:10 +0200 Subject: [PATCH 7/7] revert last commit --- src/SqlPersistence/Outbox/OutboxPersister.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/SqlPersistence/Outbox/OutboxPersister.cs b/src/SqlPersistence/Outbox/OutboxPersister.cs index 3ae081908..424e8f945 100644 --- a/src/SqlPersistence/Outbox/OutboxPersister.cs +++ b/src/SqlPersistence/Outbox/OutboxPersister.cs @@ -14,16 +14,19 @@ class OutboxPersister : IOutboxStorage { readonly IConnectionManager connectionManager; readonly SqlDialect sqlDialect; + readonly int cleanupBatchSize; readonly OutboxCommands outboxCommands; readonly Func outboxTransactionFactory; public OutboxPersister(IConnectionManager connectionManager, SqlDialect sqlDialect, OutboxCommands outboxCommands, - Func outboxTransactionFactory) + Func outboxTransactionFactory, + int cleanupBatchSize = 4000) // Keep below 4000 to prevent lock escalation { this.connectionManager = connectionManager; this.sqlDialect = sqlDialect; this.outboxCommands = outboxCommands; this.outboxTransactionFactory = outboxTransactionFactory; + this.cleanupBatchSize = cleanupBatchSize; } public Task BeginTransaction(ContextBag context, CancellationToken cancellationToken = default) @@ -123,8 +126,6 @@ public Task Store(OutboxMessage message, IOutboxTransaction outboxTransaction, C public async Task RemoveEntriesOlderThan(DateTime dateTime, CancellationToken cancellationToken = default) { - const int CleanupBatchSize = 4000; // Keep below 4000 to prevent lock escalation - using (var connection = await connectionManager.OpenNonContextualConnection(cancellationToken).ConfigureAwait(false)) { var continuePurging = true; @@ -136,7 +137,7 @@ public async Task RemoveEntriesOlderThan(DateTime dateTime, CancellationToken ca { command.CommandText = outboxCommands.Cleanup; command.AddParameter("DispatchedBefore", dateTime); - command.AddParameter("BatchSize", CleanupBatchSize); + command.AddParameter("BatchSize", cleanupBatchSize); var rowCount = await command.ExecuteNonQueryEx(cancellationToken).ConfigureAwait(false); continuePurging = rowCount != 0; }