From 2e4a2e2ef60b6c9c871c165573c9039f320bf244 Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Tue, 28 Nov 2023 08:13:30 -0600 Subject: [PATCH 1/9] Add unified tests for making ExceededTimeLimit a retryable error --- .../tests/unified/ExceededTimeLimit.json | 170 ++++++++++++++++++ .../tests/unified/ExceededTimeLimit.yml | 76 ++++++++ 2 files changed, 246 insertions(+) create mode 100644 source/retryable-reads/tests/unified/ExceededTimeLimit.json create mode 100644 source/retryable-reads/tests/unified/ExceededTimeLimit.yml diff --git a/source/retryable-reads/tests/unified/ExceededTimeLimit.json b/source/retryable-reads/tests/unified/ExceededTimeLimit.json new file mode 100644 index 0000000000..4ac30c816a --- /dev/null +++ b/source/retryable-reads/tests/unified/ExceededTimeLimit.json @@ -0,0 +1,170 @@ +{ + "description": "ExceededTimeLimit is a retryable read", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.2", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], + "auth": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "find-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "find-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + } + ] + } + ], + "tests": [ + { + "description": "Find succeeds on second attempt after ExceededTimeLimit", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "closeConnection": false, + "errorCode": 262 + } + } + } + }, + { + "name": "find", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "batchSize": 10 + }, + "object": "collection0", + "expectResult": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "batchSize": 10 + }, + "commandName": "find", + "databaseName": "find-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "batchSize": 10 + }, + "commandName": "find", + "databaseName": "find-tests" + } + } + ] + } + ] + } + ] +} diff --git a/source/retryable-reads/tests/unified/ExceededTimeLimit.yml b/source/retryable-reads/tests/unified/ExceededTimeLimit.yml new file mode 100644 index 0000000000..c78ac64eab --- /dev/null +++ b/source/retryable-reads/tests/unified/ExceededTimeLimit.yml @@ -0,0 +1,76 @@ +description: "ExceededTimeLimit is a retryable read" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.2" + topologies: [replicaset, sharded, load-balanced] + auth: true + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name find-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + - { _id: 5, x: 55 } + - { _id: 6, x: 66 } + +tests: + - description: "Find succeeds on second attempt after ExceededTimeLimit" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ "find" ] + closeConnection: false + errorCode: 262 + - name: find + arguments: + filter: { _id: { $gt: 1 } } + batchSize: 10 + object: *collection0 + expectResult: + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + - { _id: 5, x: 55 } + - { _id: 6, x: 66 } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: { $gt: 1 } } + batchSize: 10 + commandName: find + databaseName: *database0Name + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: { $gt: 1 } } + batchSize: 10 + commandName: find + databaseName: *database0Name From 7641533cce821cea007dfc85dd27d72e95277423 Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Tue, 28 Nov 2023 08:27:23 -0600 Subject: [PATCH 2/9] update retryable reads spec to add ExceededTimeLimit to the list of error codes that should receive a RetryableReadError label --- source/retryable-reads/retryable-reads.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/retryable-reads/retryable-reads.rst b/source/retryable-reads/retryable-reads.rst index 161789830b..c5f4febc9b 100644 --- a/source/retryable-reads/retryable-reads.rst +++ b/source/retryable-reads/retryable-reads.rst @@ -54,6 +54,7 @@ An error is considered retryable if it meets any of the following criteria: =============================== ============== **Error Name** **Error Code** =============================== ============== +ExceededTimeLimit 262 InterruptedAtShutdown 11600 InterruptedDueToReplStateChange 11602 NotWritablePrimary 10107 @@ -703,6 +704,8 @@ degraded performance can simply disable ``retryableReads``. Changelog ========= +:2023-11-28: Add ExceededTimeLimit to the list of error codes that should + receive a RetryableReadError label. :2023-08-26: Require that in a sharded cluster the server on which the operation failed MUST be provided to the server selection mechanism as a deprioritized server. From 550f7567db005502c19444331742c0e3876a9790 Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Tue, 28 Nov 2023 12:44:04 -0600 Subject: [PATCH 3/9] fix schema version, use 1.3 instead of 1.0 --- source/retryable-reads/tests/unified/ExceededTimeLimit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/retryable-reads/tests/unified/ExceededTimeLimit.yml b/source/retryable-reads/tests/unified/ExceededTimeLimit.yml index c78ac64eab..ed569f2d30 100644 --- a/source/retryable-reads/tests/unified/ExceededTimeLimit.yml +++ b/source/retryable-reads/tests/unified/ExceededTimeLimit.yml @@ -1,6 +1,6 @@ description: "ExceededTimeLimit is a retryable read" -schemaVersion: "1.0" +schemaVersion: "1.3" runOnRequirements: - minServerVersion: "4.2" From 5a2c3fded9ad97e6f8368cf2d2a22e091341dfc1 Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Wed, 29 Nov 2023 10:56:16 -0600 Subject: [PATCH 4/9] Update source/retryable-reads/tests/unified/ExceededTimeLimit.yml Co-authored-by: Jeremy Mikola --- source/retryable-reads/tests/unified/ExceededTimeLimit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/retryable-reads/tests/unified/ExceededTimeLimit.yml b/source/retryable-reads/tests/unified/ExceededTimeLimit.yml index ed569f2d30..ca1a2d0b0a 100644 --- a/source/retryable-reads/tests/unified/ExceededTimeLimit.yml +++ b/source/retryable-reads/tests/unified/ExceededTimeLimit.yml @@ -45,7 +45,7 @@ tests: data: failCommands: [ "find" ] closeConnection: false - errorCode: 262 + errorCode: 262 # ExceededTimeLimit - name: find arguments: filter: { _id: { $gt: 1 } } From 6718a4492e33c707b28d7fc7f0a21a22756575dd Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Wed, 29 Nov 2023 13:02:14 -0600 Subject: [PATCH 5/9] implement Jeremy's suggestions --- ...dTimeLimit.json => exceededTimeLimit.json} | 54 +++++-------------- ...dedTimeLimit.yml => exceededTimeLimit.yml} | 14 +---- 2 files changed, 15 insertions(+), 53 deletions(-) rename source/retryable-reads/tests/unified/{ExceededTimeLimit.json => exceededTimeLimit.json} (74%) rename source/retryable-reads/tests/unified/{ExceededTimeLimit.yml => exceededTimeLimit.yml} (82%) diff --git a/source/retryable-reads/tests/unified/ExceededTimeLimit.json b/source/retryable-reads/tests/unified/exceededTimeLimit.json similarity index 74% rename from source/retryable-reads/tests/unified/ExceededTimeLimit.json rename to source/retryable-reads/tests/unified/exceededTimeLimit.json index 4ac30c816a..33d1d3ce28 100644 --- a/source/retryable-reads/tests/unified/ExceededTimeLimit.json +++ b/source/retryable-reads/tests/unified/exceededTimeLimit.json @@ -1,6 +1,6 @@ { "description": "ExceededTimeLimit is a retryable read", - "schemaVersion": "1.0", + "schemaVersion": "1.3", "runOnRequirements": [ { "minServerVersion": "4.2", @@ -8,8 +8,7 @@ "replicaset", "sharded", "load-balanced" - ], - "auth": true + ] } ], "createEntities": [ @@ -26,21 +25,21 @@ "database": { "id": "database0", "client": "client0", - "databaseName": "find-tests" + "databaseName": "retryable-reads-tests" } }, { "collection": { "id": "collection0", "database": "database0", - "collectionName": "coll0" + "collectionName": "exceededtimelimit-test" } } ], "initialData": [ { - "collectionName": "coll0", - "databaseName": "find-tests", + "collectionName": "exceededtimelimit-test", + "databaseName": "retryable-reads-tests", "documents": [ { "_id": 1, @@ -53,18 +52,6 @@ { "_id": 3, "x": 33 - }, - { - "_id": 4, - "x": 44 - }, - { - "_id": 5, - "x": 55 - }, - { - "_id": 6, - "x": 66 } ] } @@ -100,8 +87,7 @@ "_id": { "$gt": 1 } - }, - "batchSize": 10 + } }, "object": "collection0", "expectResult": [ @@ -112,18 +98,6 @@ { "_id": 3, "x": 33 - }, - { - "_id": 4, - "x": 44 - }, - { - "_id": 5, - "x": 55 - }, - { - "_id": 6, - "x": 66 } ] } @@ -135,31 +109,29 @@ { "commandStartedEvent": { "command": { - "find": "coll0", + "find": "exceededtimelimit-test", "filter": { "_id": { "$gt": 1 } - }, - "batchSize": 10 + } }, "commandName": "find", - "databaseName": "find-tests" + "databaseName": "retryable-reads-tests" } }, { "commandStartedEvent": { "command": { - "find": "coll0", + "find": "exceededtimelimit-test", "filter": { "_id": { "$gt": 1 } - }, - "batchSize": 10 + } }, "commandName": "find", - "databaseName": "find-tests" + "databaseName": "retryable-reads-tests" } } ] diff --git a/source/retryable-reads/tests/unified/ExceededTimeLimit.yml b/source/retryable-reads/tests/unified/exceededTimeLimit.yml similarity index 82% rename from source/retryable-reads/tests/unified/ExceededTimeLimit.yml rename to source/retryable-reads/tests/unified/exceededTimeLimit.yml index ca1a2d0b0a..65826192ed 100644 --- a/source/retryable-reads/tests/unified/ExceededTimeLimit.yml +++ b/source/retryable-reads/tests/unified/exceededTimeLimit.yml @@ -5,7 +5,6 @@ schemaVersion: "1.3" runOnRequirements: - minServerVersion: "4.2" topologies: [replicaset, sharded, load-balanced] - auth: true createEntities: - client: @@ -15,11 +14,11 @@ createEntities: - database: id: &database0 database0 client: *client0 - databaseName: &database0Name find-tests + databaseName: &database0Name "retryable-reads-tests" - collection: id: &collection0 collection0 database: *database0 - collectionName: &collection0Name coll0 + collectionName: &collection0Name "exceededtimelimit-test" initialData: - collectionName: *collection0Name @@ -28,9 +27,6 @@ initialData: - { _id: 1, x: 11 } - { _id: 2, x: 22 } - { _id: 3, x: 33 } - - { _id: 4, x: 44 } - - { _id: 5, x: 55 } - - { _id: 6, x: 66 } tests: - description: "Find succeeds on second attempt after ExceededTimeLimit" @@ -49,14 +45,10 @@ tests: - name: find arguments: filter: { _id: { $gt: 1 } } - batchSize: 10 object: *collection0 expectResult: - { _id: 2, x: 22 } - { _id: 3, x: 33 } - - { _id: 4, x: 44 } - - { _id: 5, x: 55 } - - { _id: 6, x: 66 } expectEvents: - client: *client0 events: @@ -64,13 +56,11 @@ tests: command: find: *collection0Name filter: { _id: { $gt: 1 } } - batchSize: 10 commandName: find databaseName: *database0Name - commandStartedEvent: command: find: *collection0Name filter: { _id: { $gt: 1 } } - batchSize: 10 commandName: find databaseName: *database0Name From 61faa8f091accca216f95f5b935518a9685a8c71 Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Wed, 29 Nov 2023 14:10:47 -0600 Subject: [PATCH 6/9] use 4.1.7 for sharded and load-balanced clusters, 4.0 otherwise --- .../tests/unified/exceededTimeLimit.json | 10 ++++++++-- .../tests/unified/exceededTimeLimit.yml | 6 ++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/retryable-reads/tests/unified/exceededTimeLimit.json b/source/retryable-reads/tests/unified/exceededTimeLimit.json index 33d1d3ce28..be154570d0 100644 --- a/source/retryable-reads/tests/unified/exceededTimeLimit.json +++ b/source/retryable-reads/tests/unified/exceededTimeLimit.json @@ -3,9 +3,15 @@ "schemaVersion": "1.3", "runOnRequirements": [ { - "minServerVersion": "4.2", + "minServerVersion": "4.0", + "topologies": [ + "single", + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", "topologies": [ - "replicaset", "sharded", "load-balanced" ] diff --git a/source/retryable-reads/tests/unified/exceededTimeLimit.yml b/source/retryable-reads/tests/unified/exceededTimeLimit.yml index 65826192ed..5d6341510a 100644 --- a/source/retryable-reads/tests/unified/exceededTimeLimit.yml +++ b/source/retryable-reads/tests/unified/exceededTimeLimit.yml @@ -3,8 +3,10 @@ description: "ExceededTimeLimit is a retryable read" schemaVersion: "1.3" runOnRequirements: - - minServerVersion: "4.2" - topologies: [replicaset, sharded, load-balanced] + - minServerVersion: "4.0" + topologies: [single, replicaset] + - minServerVersion: "4.1.7" + topologies: [sharded, load-balanced] createEntities: - client: From 820b2d0d667b33d8814168d2ef96f45be9b2b2cb Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Wed, 29 Nov 2023 14:11:29 -0600 Subject: [PATCH 7/9] Update source/retryable-reads/retryable-reads.rst Co-authored-by: Kevin Albertson --- source/retryable-reads/retryable-reads.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/retryable-reads/retryable-reads.rst b/source/retryable-reads/retryable-reads.rst index c5f4febc9b..957aa69b51 100644 --- a/source/retryable-reads/retryable-reads.rst +++ b/source/retryable-reads/retryable-reads.rst @@ -705,7 +705,7 @@ Changelog ========= :2023-11-28: Add ExceededTimeLimit to the list of error codes that should - receive a RetryableReadError label. + be retried. :2023-08-26: Require that in a sharded cluster the server on which the operation failed MUST be provided to the server selection mechanism as a deprioritized server. From 03acef6e2e6da463b0aa4960f9a5431d8300e6b2 Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Wed, 29 Nov 2023 14:46:08 -0600 Subject: [PATCH 8/9] implement Kevin's suggestions --- source/retryable-reads/tests/unified/exceededTimeLimit.json | 1 - source/retryable-reads/tests/unified/exceededTimeLimit.yml | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/retryable-reads/tests/unified/exceededTimeLimit.json b/source/retryable-reads/tests/unified/exceededTimeLimit.json index be154570d0..8d090bbe3f 100644 --- a/source/retryable-reads/tests/unified/exceededTimeLimit.json +++ b/source/retryable-reads/tests/unified/exceededTimeLimit.json @@ -80,7 +80,6 @@ "failCommands": [ "find" ], - "closeConnection": false, "errorCode": 262 } } diff --git a/source/retryable-reads/tests/unified/exceededTimeLimit.yml b/source/retryable-reads/tests/unified/exceededTimeLimit.yml index 5d6341510a..e18ad3dec4 100644 --- a/source/retryable-reads/tests/unified/exceededTimeLimit.yml +++ b/source/retryable-reads/tests/unified/exceededTimeLimit.yml @@ -11,7 +11,7 @@ runOnRequirements: createEntities: - client: id: &client0 client0 - useMultipleMongoses: false + useMultipleMongoses: false # Ensure the `configureFailpoint` and `find` command are run on the same mongos. observeEvents: [ commandStartedEvent ] - database: id: &database0 database0 @@ -42,7 +42,6 @@ tests: mode: { times: 1 } data: failCommands: [ "find" ] - closeConnection: false errorCode: 262 # ExceededTimeLimit - name: find arguments: From 2412f5f87163bf9f9d79a16151a13f3f5e144c3f Mon Sep 17 00:00:00 2001 From: Kyle Kloberdanz Date: Thu, 30 Nov 2023 09:46:12 -0600 Subject: [PATCH 9/9] Update source/retryable-reads/tests/unified/exceededTimeLimit.yml Co-authored-by: Jeremy Mikola --- source/retryable-reads/tests/unified/exceededTimeLimit.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/retryable-reads/tests/unified/exceededTimeLimit.yml b/source/retryable-reads/tests/unified/exceededTimeLimit.yml index e18ad3dec4..1912fa7bbd 100644 --- a/source/retryable-reads/tests/unified/exceededTimeLimit.yml +++ b/source/retryable-reads/tests/unified/exceededTimeLimit.yml @@ -11,7 +11,8 @@ runOnRequirements: createEntities: - client: id: &client0 client0 - useMultipleMongoses: false # Ensure the `configureFailpoint` and `find` command are run on the same mongos. + # Ensure the `configureFailpoint` and `find` commands are run on the same mongos + useMultipleMongoses: false observeEvents: [ commandStartedEvent ] - database: id: &database0 database0