diff --git a/spec/spec_tests/data/load_balancers/cursors.yml b/spec/spec_tests/data/load_balancers/cursors.yml new file mode 100644 index 0000000000..cbd9852c3b --- /dev/null +++ b/spec/spec_tests/data/load_balancers/cursors.yml @@ -0,0 +1,513 @@ +description: cursors are correctly pinned to connections for load-balanced clusters + +schemaVersion: '1.4' + +runOnRequirements: + - topologies: [ load-balanced ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: true + observeEvents: + - commandStartedEvent + - commandSucceededEvent + - commandFailedEvent + - connectionReadyEvent + - connectionClosedEvent + - connectionCheckedOutEvent + - connectionCheckedInEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name database0Name + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name coll1 + - collection: + id: &collection2 collection2 + database: *database0 + collectionName: &collection2Name coll2 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + - { _id: 3 } + - collectionName: *collection1Name + databaseName: *database0Name + documents: [] + - collectionName: *collection2Name + databaseName: *database0Name + documents: [] + +tests: + - description: no connection is pinned if all documents are returned in the initial batch + operations: + - name: createFindCursor + object: *collection0 + arguments: + filter: {} + saveResultAsEntity: &cursor0 cursor0 + - &assertConnectionNotPinned + name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + find: *collection0Name + filter: {} + commandName: find + - commandSucceededEvent: + reply: + cursor: + id: 0 + firstBatch: { $$type: array } + ns: { $$type: string } + commandName: find + - client: *client0 + eventType: cmap + events: + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: pinned connections are returned when the cursor is drained + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - &createAndSaveCursor + name: createFindCursor + object: *collection0 + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: &cursor0 cursor0 + - &assertConnectionPinned + name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 1 + - name: iterateUntilDocumentOrError + object: *cursor0 + expectResult: { _id: 1 } + - name: iterateUntilDocumentOrError + object: *cursor0 + expectResult: { _id: 2 } + - name: iterateUntilDocumentOrError + object: *cursor0 + expectResult: { _id: 3 } + - *assertConnectionNotPinned + - &closeCursor + name: close + object: *cursor0 + expectEvents: + - client: *client0 + events: + - &findWithBatchSizeStarted + commandStartedEvent: + command: + find: *collection0Name + filter: {} + batchSize: 2 + commandName: find + - &findWithBatchSizeSucceeded + commandSucceededEvent: + reply: + cursor: + id: { $$type: [ int, long ] } + firstBatch: { $$type: array } + ns: { $$type: string } + commandName: find + - &getMoreStarted + commandStartedEvent: + command: + getMore: { $$type: [ int, long ] } + collection: *collection0Name + commandName: getMore + - &getMoreSucceeded + commandSucceededEvent: + reply: + cursor: + id: 0 + ns: { $$type: string } + nextBatch: { $$type: array } + commandName: getMore + - client: *client0 + eventType: cmap + events: + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: pinned connections are returned to the pool when the cursor is closed + operations: + - *createAndSaveCursor + - *assertConnectionPinned + - *closeCursor + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *findWithBatchSizeStarted + - *findWithBatchSizeSucceeded + - &killCursorsStarted + commandStartedEvent: + commandName: killCursors + - &killCursorsSucceeded + commandSucceededEvent: + commandName: killCursors + - client: *client0 + eventType: cmap + events: + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + # If a network error occurs during a getMore request, the connection must remain pinned. and drivers must not + # attempt to send a killCursors command when the cursor is closed because the connection is no longer valid. + - description: pinned connections are not returned after an network error during getMore + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ getMore ] + closeConnection: true + - *createAndSaveCursor + - *assertConnectionPinned + - name: iterateUntilDocumentOrError + object: *cursor0 + expectResult: + _id: 1 + - name: iterateUntilDocumentOrError + object: *cursor0 + expectResult: + _id: 2 + # Third next() call should perform a getMore. + - name: iterateUntilDocumentOrError + object: *cursor0 + expectError: + # Network errors are considered client-side errors per the unified test format spec. + isClientError: true + - *assertConnectionPinned + - *closeCursor # Execute a close operation to actually release the connection. + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *findWithBatchSizeStarted + - *findWithBatchSizeSucceeded + - *getMoreStarted + - &getMoreFailed + commandFailedEvent: + commandName: getMore + - client: *client0 + eventType: cmap + events: + # Events to set the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the find command + getMore. + - connectionCheckedOutEvent: {} + # Events for the close() operation. + - connectionCheckedInEvent: {} + - connectionClosedEvent: + reason: error + + - description: pinned connections are returned after a network error during a killCursors request + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ killCursors ] + closeConnection: true + - *createAndSaveCursor + - *assertConnectionPinned + - *closeCursor + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *findWithBatchSizeStarted + - *findWithBatchSizeSucceeded + - *killCursorsStarted + - commandFailedEvent: + commandName: killCursors + - client: *client0 + eventType: cmap + events: + # Events to set the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the find command + killCursors. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + - connectionClosedEvent: + reason: error + + - description: pinned connections are not returned to the pool after a non-network error on getMore + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ getMore ] + errorCode: &hostNotFoundCode 7 # This is not a state change error code, so it should not cause SDAM changes. + - *createAndSaveCursor + - name: iterateUntilDocumentOrError + object: *cursor0 + expectResult: + _id: 1 + - name: iterateUntilDocumentOrError + object: *cursor0 + expectResult: + _id: 2 + - name: iterateUntilDocumentOrError + object: *cursor0 + expectError: + errorCode: *hostNotFoundCode + - *assertConnectionPinned + - *closeCursor + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *findWithBatchSizeStarted + - *findWithBatchSizeSucceeded + - *getMoreStarted + - *getMoreFailed + - *killCursorsStarted + - *killCursorsSucceeded + - client: *client0 + eventType: cmap + events: + # Events to set the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the find command + getMore + killCursors. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + # Basic tests for cursor-creating commands besides "find". We don't need to replicate the full set of tests defined + # above for each such command. Instead, only one test is needed per command to ensure that the pinned connection is + # correctly passed down to the server. + # + # Each test creates a cursor with a small batch size and fully iterates it. Because drivers do not publish CMAP + # events when using pinned connections, each test asserts that only one set of ready/checkout/checkin events are + # published. + + - description: aggregate pins the cursor to a connection + operations: + - name: aggregate + object: *collection0 + arguments: + pipeline: [] + batchSize: 2 + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collection0Name + cursor: + batchSize: 2 + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + - *getMoreStarted + - *getMoreSucceeded + - client: *client0 + eventType: cmap + events: + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: listCollections pins the cursor to a connection + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + runOnRequirements: + - serverless: forbid # CLOUDP-98562 listCollections batchSize is ignored on serverless. + operations: + - name: listCollections + object: *database0 + arguments: + filter: {} + batchSize: 2 + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + listCollections: 1 + cursor: + batchSize: 2 + commandName: listCollections + databaseName: *database0Name + - commandSucceededEvent: + commandName: listCollections + # Write out the event for getMore rather than using the getMoreStarted anchor because the "collection" field + # is not equal to *collection0Name as the command is not executed against a collection. + - commandStartedEvent: + command: + getMore: { $$type: [ int, long ] } + collection: { $$type: string } + commandName: getMore + - *getMoreSucceeded + - client: *client0 + eventType: cmap + events: + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: listIndexes pins the cursor to a connection + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + # There is an automatic index on _id so we create two more indexes to force multiple batches with batchSize=2. + - name: createIndex + object: *collection0 + arguments: + keys: &x1IndexSpec { x: 1 } + name: &x1IndexName x_1 + - name: createIndex + object: *collection0 + arguments: + keys: &y1IndexSpec { y: 1 } + name: &y1IndexName y_1 + - name: listIndexes + object: *collection0 + arguments: + batchSize: 2 + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + createIndexes: *collection0Name + indexes: + - name: *x1IndexName + key: *x1IndexSpec + commandName: createIndexes + - commandSucceededEvent: + commandName: createIndexes + - commandStartedEvent: + command: + createIndexes: *collection0Name + indexes: + - name: *y1IndexName + key: *y1IndexSpec + commandName: createIndexes + - commandSucceededEvent: + commandName: createIndexes + - commandStartedEvent: + command: + listIndexes: *collection0Name + cursor: + batchSize: 2 + commandName: listIndexes + databaseName: *database0Name + - commandSucceededEvent: + commandName: listIndexes + - *getMoreStarted + - *getMoreSucceeded + - client: *client0 + eventType: cmap + events: + # Events for first createIndexes. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for second createIndexes. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for listIndexes and getMore. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: change streams pin to a connection + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + runOnRequirements: + - serverless: forbid # Serverless does not support change streams. + operations: + - name: createChangeStream + object: *collection0 + arguments: + pipeline: [] + saveResultAsEntity: &changeStream0 changeStream0 + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 1 + - name: close + object: *changeStream0 + - name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + - commandStartedEvent: + commandName: killCursors + - commandSucceededEvent: + commandName: killCursors + - client: *client0 + eventType: cmap + events: + # Events for creating the change stream. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + # Events for closing the change stream. + - connectionCheckedInEvent: {} diff --git a/spec/spec_tests/data/load_balancers/sdam-error-handling.yml b/spec/spec_tests/data/load_balancers/sdam-error-handling.yml new file mode 100644 index 0000000000..b237f4c6e1 --- /dev/null +++ b/spec/spec_tests/data/load_balancers/sdam-error-handling.yml @@ -0,0 +1,276 @@ +description: state change errors are correctly handled + +schemaVersion: '1.4' + +runOnRequirements: + - topologies: [ load-balanced ] + +_yamlAnchors: + observedEvents: &observedEvents + - connectionCreatedEvent + - connectionReadyEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent + +createEntities: + - client: + id: &failPointClient failPointClient + useMultipleMongoses: false + - client: + id: &singleClient singleClient + useMultipleMongoses: false + uriOptions: + appname: &singleClientAppName lbSDAMErrorTestClient + retryWrites: false + observeEvents: *observedEvents + - database: + id: &singleDB singleDB + client: *singleClient + databaseName: &singleDBName singleDB + - collection: + id: &singleColl singleColl + database: *singleDB + collectionName: &singleCollName singleColl + - client: + id: &multiClient multiClient + useMultipleMongoses: true + uriOptions: + retryWrites: false + observeEvents: *observedEvents + - database: + id: &multiDB multiDB + client: *multiClient + databaseName: &multiDBName multiDB + - collection: + id: &multiColl multiColl + database: *multiDB + collectionName: &multiCollName multiColl + +initialData: + - collectionName: *singleCollName + databaseName: *singleDBName + documents: + - _id: 1 + - _id: 2 + - _id: 3 + - collectionName: *multiCollName + databaseName: *multiDBName + documents: + - _id: 1 + - _id: 2 + - _id: 3 + +tests: + - description: only connections for a specific serviceId are closed when pools are cleared + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + runOnRequirements: + # This test assumes that two sequential connections receive different serviceIDs. + # Sequential connections to a serverless instance may receive the same serviceID. + - serverless: forbid + operations: + # Create two cursors to force two connections. + - name: createFindCursor + object: *multiColl + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: &cursor0 cursor0 + - name: createFindCursor + object: *multiColl + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: &cursor1 cursor1 + # Close both cursors to return the connections to the pool. + - name: close + object: *cursor0 + - name: close + object: *cursor1 + # Fail an operation with a state change error. + - name: failPoint + object: testRunner + arguments: + client: *multiClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [insert] + errorCode: &errorCode 11600 # InterruptedAtShutdown + - name: insertOne + object: *multiColl + arguments: + document: { x: 1 } + expectError: + errorCode: *errorCode + # Do another operation to ensure the relevant connection has been closed. + - name: insertOne + object: *multiColl + arguments: + document: { x: 1 } + expectEvents: + - client: *multiClient + eventType: cmap + events: + # Create cursors. + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + # Close cursors. + - connectionCheckedInEvent: {} + - connectionCheckedInEvent: {} + # Set failpoint. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # First insertOne. + - connectionCheckedOutEvent: {} + - poolClearedEvent: {} + - connectionCheckedInEvent: {} + - connectionClosedEvent: + reason: stale + # Second insertOne. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + # This test uses singleClient to ensure that connection attempts are routed + # to the same mongos on which the failpoint is set. + - description: errors during the initial connection hello are ignored + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + runOnRequirements: + # Require SERVER-49336 for failCommand + appName on the initial handshake. + - minServerVersion: '4.4.7' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [isMaster, hello] + closeConnection: true + appName: *singleClientAppName + - name: insertOne + object: *singleColl + arguments: + document: { x: 1 } + expectError: + isClientError: true + expectEvents: + - client: *singleClient + eventType: cmap + events: + - connectionCreatedEvent: {} + - connectionClosedEvent: + reason: error + - connectionCheckOutFailedEvent: + reason: connectionError + + - description: errors during authentication are processed + runOnRequirements: + - auth: true + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [saslContinue] + closeConnection: true + appName: *singleClientAppName + - name: insertOne + object: *singleColl + arguments: + document: { x: 1 } + expectError: + isClientError: true + expectEvents: + - client: *singleClient + eventType: cmap + events: + - connectionCreatedEvent: {} + - poolClearedEvent: {} + - connectionClosedEvent: + reason: error + - connectionCheckOutFailedEvent: + reason: connectionError + + - description: stale errors are ignored + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [getMore] + closeConnection: true + # Force two connections to be checked out from the pool. + - name: createFindCursor + object: *singleColl + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: &cursor0 cursor0 + - name: createFindCursor + object: *singleColl + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: &cursor1 cursor1 + # Iterate cursor0 three times to force a network error. + - name: iterateUntilDocumentOrError + object: *cursor0 + - name: iterateUntilDocumentOrError + object: *cursor0 + - name: iterateUntilDocumentOrError + object: *cursor0 + expectError: + isClientError: true + - name: close + object: *cursor0 + # Iterate cursor1 three times to force a network error. + - name: iterateUntilDocumentOrError + object: *cursor1 + - name: iterateUntilDocumentOrError + object: *cursor1 + - name: iterateUntilDocumentOrError + object: *cursor1 + expectError: + isClientError: true + - name: close + object: *cursor1 + expectEvents: + - client: *singleClient + eventType: cmap + events: + # Events for creating both cursors. + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + # Events for iterating and closing the first cursor. The failed + # getMore should cause a poolClearedEvent to be published. + - poolClearedEvent: {} + - connectionCheckedInEvent: {} + - connectionClosedEvent: {} + # Events for iterating and closing the second cursor. The failed + # getMore should not clear the pool because the connection's + # generation number is stale. + - connectionCheckedInEvent: {} + - connectionClosedEvent: {} diff --git a/spec/spec_tests/data/load_balancers/transactions.yml b/spec/spec_tests/data/load_balancers/transactions.yml new file mode 100644 index 0000000000..26c18ddbe2 --- /dev/null +++ b/spec/spec_tests/data/load_balancers/transactions.yml @@ -0,0 +1,613 @@ +description: transactions are correctly pinned to connections for load-balanced clusters + +schemaVersion: '1.4' + +runOnRequirements: + - topologies: [ load-balanced ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: true + observeEvents: + # Do not observe commandSucceededEvent or commandFailedEvent because we cannot guarantee success or failure of + # commands like commitTransaction and abortTransaction in a multi-mongos load-balanced setup. + - commandStartedEvent + - connectionReadyEvent + - connectionClosedEvent + - connectionCheckedOutEvent + - connectionCheckedInEvent + - session: + id: &session0 session0 + client: *client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name database0Name + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + - { _id: 3 } + +_yamlAnchors: + documents: + - &insertDocument + _id: 4 + +tests: + - description: sessions are reused in LB mode + operations: + - &nonTransactionalInsert + name: insertOne + object: *collection0 + arguments: + document: { x: 1 } + - *nonTransactionalInsert + - name: assertSameLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client0 + + - description: all operations go to the same mongos + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - &startTransaction + name: startTransaction + object: *session0 + - &transactionalInsert + name: insertOne + object: *collection0 + arguments: + document: { x: 1 } + session: *session0 + - &assertConnectionPinned + name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 1 + - *transactionalInsert + - *transactionalInsert + - *transactionalInsert + - *transactionalInsert + - *transactionalInsert + - *assertConnectionPinned + - &commitTransaction + name: commitTransaction + object: *session0 + expectEvents: + - client: *client0 + events: + - &insertStarted + commandStartedEvent: + commandName: insert + - *insertStarted + - *insertStarted + - *insertStarted + - *insertStarted + - *insertStarted + - &commitStarted + commandStartedEvent: + commandName: commitTransaction + - client: *client0 + eventType: cmap + events: + # The connection is never checked back in. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + + - description: transaction can be committed multiple times + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - *startTransaction + - *transactionalInsert + - *assertConnectionPinned + - *commitTransaction + - *assertConnectionPinned + - *commitTransaction + - *commitTransaction + - *commitTransaction + - *assertConnectionPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *commitStarted + - *commitStarted + - *commitStarted + - *commitStarted + - client: *client0 + eventType: cmap + events: + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + + - description: pinned connection is not released after a non-transient CRUD error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + errorCode: &nonTransientErrorCode 51 # ManualInterventionRequired + - *startTransaction + - name: insertOne + object: *collection0 + arguments: + document: { x: 1 } + session: *session0 + expectError: &nonTransientExpectedError + errorCode: *nonTransientErrorCode + errorLabelsOmit: [ TransientTransactionError ] + - *assertConnectionPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - client: *client0 + eventType: cmap + events: + # Events for setting the fail point. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the transactional insert. + - connectionCheckedOutEvent: {} + + - description: pinned connection is not released after a non-transient commit error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ commitTransaction ] + errorCode: *nonTransientErrorCode + - *startTransaction + - *transactionalInsert + - name: commitTransaction + object: *session0 + expectError: *nonTransientExpectedError + - *assertConnectionPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *commitStarted + - client: *client0 + eventType: cmap + events: + # Events for setting the fail point. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the transactional insert and commit. + - connectionCheckedOutEvent: {} + + # Errors during abort are different than errors during commit and CRUD operations because the pinned connection is + # always released after abort. + - description: pinned connection is released after a non-transient abort error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ abortTransaction ] + errorCode: &nonTransientErrorCode 51 # ManualInterventionRequired + - *startTransaction + - *transactionalInsert + - name: abortTransaction + object: *session0 + - &assertConnectionNotPinned + name: assertNumberConnectionsCheckedOut + object: testRunner + arguments: + client: *client0 + connections: 0 + expectEvents: + - client: *client0 + events: + - *insertStarted + - &abortStarted + commandStartedEvent: + commandName: abortTransaction + - client: *client0 + eventType: cmap + events: + # Events for setting the fail point. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the transactional insert and abort. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: pinned connection is released after a transient non-network CRUD error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + runOnRequirements: + - serverless: forbid # (CLOUDP-88216) Serverless does not append error labels to errors triggered by failpoints. + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + errorCode: &transientErrorCode 24 # LockTimeout + - *startTransaction + - <<: *transactionalInsert + expectError: &transientExpectedServerError + errorCode: *transientErrorCode + errorLabelsContain: [ TransientTransactionError ] + - *assertConnectionNotPinned + - name: abortTransaction + object: *session0 + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *abortStarted + - client: *client0 + eventType: cmap + events: + # Events for setting the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the insert. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for abortTransction. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: pinned connection is released after a transient network CRUD error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + runOnRequirements: + - serverless: forbid # (CLOUDP-88216) Serverless does not append error labels to errors triggered by failpoints. + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + closeConnection: true + - *startTransaction + - <<: *transactionalInsert + expectError: &transientExpectedNetworkError + isClientError: true + errorLabelsContain: [ TransientTransactionError ] + - *assertConnectionNotPinned + - name: abortTransaction + object: *session0 + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *abortStarted + - client: *client0 + eventType: cmap + events: + # Events for setting the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the insert. + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + - connectionClosedEvent: + reason: error + # Events for abortTransaction + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: pinned connection is released after a transient non-network commit error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + runOnRequirements: + - serverless: forbid # (CLOUDP-88216) Serverless does not append error labels to errors triggered by failpoints. + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ commitTransaction ] + errorCode: *transientErrorCode + - *startTransaction + - *transactionalInsert + - <<: *commitTransaction + expectError: *transientExpectedServerError + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *commitStarted + - client: *client0 + eventType: cmap + events: + # Events for setting the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the insert. + - connectionCheckedOutEvent: {} + # Events for commitTransaction. + - connectionCheckedInEvent: {} + + - description: pinned connection is released after a transient network commit error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ commitTransaction ] + closeConnection: true + - *startTransaction + - *transactionalInsert + - <<: *commitTransaction + # Ignore the result and error because the operation might fail if it targets a new mongos that isn't aware of + # the transaction or the server-side reaper thread closes the transaction first. We only want to assert that + # the operation is retried, which is done via monitoring expectations, so the exact result/error is not + # necessary. + ignoreResultAndError: true + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *commitStarted + # The commit will be automatically retried. + - *commitStarted + - client: *client0 + eventType: cmap + events: + # Events for setting the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the insert. + - connectionCheckedOutEvent: {} + # Events for the first commitTransaction. + - connectionCheckedInEvent: {} + - connectionClosedEvent: + reason: error + # Events for the commitTransaction retry. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: pinned connection is released after a transient non-network abort error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ abortTransaction ] + errorCode: *transientErrorCode + - *startTransaction + - *transactionalInsert + - name: abortTransaction + object: *session0 + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *abortStarted + - client: *client0 + eventType: cmap + events: + # Events for setting the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the insert. + - connectionCheckedOutEvent: {} + # Events for abortTransaction. + - connectionCheckedInEvent: {} + + - description: pinned connection is released after a transient network abort error + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ abortTransaction ] + closeConnection: true + - *startTransaction + - *transactionalInsert + - name: abortTransaction + object: *session0 + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *abortStarted + # The abort will be automatically retried. + - *abortStarted + - client: *client0 + eventType: cmap + events: + # Events for setting the failpoint. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + # Events for the insert. + - connectionCheckedOutEvent: {} + # Events for the first abortTransaction. + - connectionCheckedInEvent: {} + - connectionClosedEvent: + reason: error + # Events for the abortTransaction retry. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: pinned connection is released on successful abort + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - *startTransaction + - *transactionalInsert + - name: abortTransaction + object: *session0 + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *abortStarted + - client: *client0 + eventType: cmap + events: + # The insert will create and pin a connection. The abort will use it and then unpin. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: pinned connection is returned when a new transaction is started + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - *startTransaction + - *transactionalInsert + - *commitTransaction + - *assertConnectionPinned + - *startTransaction + - *assertConnectionNotPinned # startTransaction will unpin the connection. + - *transactionalInsert + - *assertConnectionPinned # The first operation in the new transaction will pin the connection again. + - *commitTransaction + expectEvents: + - client: *client0 + events: + - *insertStarted + - *commitStarted + - *insertStarted + - *commitStarted + - client: *client0 + eventType: cmap + events: + # Events for the first insert and commit. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + # Events for startTransaction. + - connectionCheckedInEvent: {} + # Events for the second insert and commit. + - connectionCheckedOutEvent: {} + + - description: pinned connection is returned when a non-transaction operation uses the session + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - *startTransaction + - *transactionalInsert + - *commitTransaction + - *assertConnectionPinned + - *transactionalInsert + # The insert is a non-transactional operation that uses the session, so it unpins the connection. + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - *commitStarted + - *insertStarted + - client: *client0 + eventType: cmap + events: + # Events for the first insert and commit. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + # Events for the second insert. + - connectionCheckedInEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + + - description: a connection can be shared by a transaction and a cursor + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - *startTransaction + - *transactionalInsert + - *assertConnectionPinned + - name: createFindCursor + object: *collection0 + arguments: + filter: {} + batchSize: 2 + session: *session0 + saveResultAsEntity: &cursor0 cursor0 + - *assertConnectionPinned + - name: close + object: *cursor0 + - *assertConnectionPinned + # Abort the transaction to ensure that the connection is unpinned. + - name: abortTransaction + object: *session0 + - *assertConnectionNotPinned + expectEvents: + - client: *client0 + events: + - *insertStarted + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: killCursors + - *abortStarted + - client: *client0 + eventType: cmap + events: + # Events for the insert, find, and killCursors. + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + # Events for abortTransaction. + - connectionCheckedInEvent: {} diff --git a/spec/spec_tests/data/load_balancers/wait-queue-timeouts.yml b/spec/spec_tests/data/load_balancers/wait-queue-timeouts.yml new file mode 100644 index 0000000000..33f9361a43 --- /dev/null +++ b/spec/spec_tests/data/load_balancers/wait-queue-timeouts.yml @@ -0,0 +1,84 @@ +description: wait queue timeout errors include details about checked out connections + +schemaVersion: '1.3' + +runOnRequirements: + - topologies: [ load-balanced ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: true + uriOptions: + maxPoolSize: 1 + waitQueueTimeoutMS: 50 + observeEvents: + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - session: + id: &session0 session0 + client: *client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name database0Name + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - _id: 1 + - _id: 2 + - _id: 3 + +tests: + - description: wait queue timeout errors include cursor statistics + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: createFindCursor + object: *collection0 + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: &cursor0 cursor0 + - name: insertOne + object: *collection0 + arguments: + document: { x: 1 } + expectError: + isClientError: true + errorContains: 'maxPoolSize: 1, connections in use by cursors: 1, connections in use by transactions: 0, connections in use by other operations: 0' + expectEvents: + - client: *client0 + eventType: cmap + events: + - connectionCheckedOutEvent: {} + - connectionCheckOutFailedEvent: {} + + - description: wait queue timeout errors include transaction statistics + skipReason: "RUBY-2881: ruby driver LB is not spec compliant" + operations: + - name: startTransaction + object: *session0 + - name: insertOne + object: *collection0 + arguments: + document: { x: 1 } + session: *session0 + - name: insertOne + object: *collection0 + arguments: + document: { x: 1 } + expectError: + isClientError: true + errorContains: 'maxPoolSize: 1, connections in use by cursors: 0, connections in use by transactions: 1, connections in use by other operations: 0' + expectEvents: + - client: *client0 + eventType: cmap + events: + - connectionCheckedOutEvent: {} + - connectionCheckOutFailedEvent: {}