# Tests for resume behavior on server versions that support the ResumableChangeStreamError label description: "change-streams-resume-errorlabels" schemaVersion: "1.7" runOnRequirements: - minServerVersion: "4.3.1" topologies: [ replicaset, sharded, load-balanced ] serverless: forbid createEntities: - client: id: &client0 client0 observeEvents: [ commandStartedEvent ] ignoreCommandMonitoringEvents: [ killCursors ] useMultipleMongoses: false - client: id: &globalClient globalClient useMultipleMongoses: false - database: id: &database0 database0 client: *client0 databaseName: *database0 - collection: id: &collection0 collection0 database: *database0 collectionName: *collection0 - database: id: &globalDatabase0 globalDatabase0 client: *globalClient databaseName: *database0 - collection: id: &globalCollection0 globalCollection0 database: *globalDatabase0 collectionName: *collection0 tests: - description: change stream resumes after HostUnreachable operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout # SERVER-46091 explains why a new failpoint was needed mode: { times: 1 } data: errorCode: 6 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after HostNotFound operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 7 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after NetworkTimeout operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 89 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate - description: change stream resumes after ShutdownInProgress operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 91 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after PrimarySteppedDown operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 189 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after ExceededTimeLimit operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 262 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after SocketException operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 9001 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after NotWritablePrimary operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 10107 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after InterruptedAtShutdown operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 11600 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after InterruptedDueToReplStateChange operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 11602 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after NotPrimaryNoSecondaryOk operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 13435 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after NotPrimaryOrSecondary operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 13436 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after StaleShardVersion runOnRequirements: # StaleShardVersion is obsolete as of 6.1 and is no longer marked as resumable. - maxServerVersion: "6.0.99" operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 63 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after StaleEpoch operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 150 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after RetryChangeStream operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 234 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream resumes after FailedToSatisfyReadPreference operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failGetMoreAfterCursorCheckout mode: { times: 1 } data: errorCode: 133 closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 # The next two tests ensure that the driver only uses the error label, not the allow list. - description: change stream resumes if error contains ResumableChangeStreamError operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failCommand mode: { times: 1 } data: failCommands: [ getMore ] errorCode: 50 # Use an error code that does not have the allow list label by default closeConnection: false errorLabels: [ ResumableChangeStreamError ] - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: _id: { $$exists: true } documentKey: { $$exists: true } operationType: insert ns: db: *database0 coll: *collection0 fullDocument: x: 1 _id: { $$exists: true } expectEvents: - client: *client0 ignoreExtraEvents: true events: - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: [ { $changeStream: {} } ] commandName: aggregate databaseName: *database0 - commandStartedEvent: command: getMore: { $$exists: true } collection: *collection0 commandName: getMore databaseName: *database0 - commandStartedEvent: command: aggregate: *collection0 cursor: {} pipeline: - $changeStream: resumeAfter: { $$unsetOrMatches: { $$exists: true } } commandName: aggregate databaseName: *database0 - description: change stream does not resume if error does not contain ResumableChangeStreamError operations: - name: failPoint object: testRunner arguments: client: *globalClient failPoint: configureFailPoint: failCommand # failCommand will not add the allow list error label mode: { times: 1 } data: failCommands: [ getMore ] errorCode: 6 # Use an error code that is on the allow list closeConnection: false - name: createChangeStream object: *collection0 arguments: { pipeline: [] } saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *globalCollection0 arguments: document: { x: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectError: { errorCode: 6 }