From ae76c38146777f23c274da323d6163c277edcc4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Mon, 30 Sep 2024 11:36:07 +0200 Subject: [PATCH] [test visibility] Support early flake detection for cucumber (#4733) --- integration-tests/cucumber/cucumber.spec.js | 2325 ++++++++++------- .../datadog-instrumentations/src/cucumber.js | 112 +- packages/datadog-plugin-cucumber/src/index.js | 5 + 3 files changed, 1410 insertions(+), 1032 deletions(-) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index bcf768883e8..35c4b3b2060 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -29,6 +29,7 @@ const { TEST_SOURCE_FILE, TEST_SOURCE_START, TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, TEST_IS_NEW, TEST_IS_RETRY, TEST_NAME, @@ -43,357 +44,415 @@ const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/ const isOldNode = semver.satisfies(process.version, '<=16') const versions = ['7.0.0', isOldNode ? '9' : 'latest'] -const moduleType = [ - { - type: 'commonJS', - runTestsCommand: './node_modules/.bin/cucumber-js ci-visibility/features/*.feature', - runTestsWithCoverageCommand: - './node_modules/nyc/bin/nyc.js -r=text-summary ' + - 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', - parallelModeCommand: './node_modules/.bin/cucumber-js ' + - 'ci-visibility/features/*.feature --parallel 2', - featuresPath: 'ci-visibility/features/', - fileExtension: 'js' - } -] +const runTestsCommand = './node_modules/.bin/cucumber-js ci-visibility/features/*.feature' +const runTestsWithCoverageCommand = './node_modules/nyc/bin/nyc.js -r=text-summary ' + + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature' +const parallelModeCommand = './node_modules/.bin/cucumber-js ci-visibility/features/*.feature --parallel 2' +const featuresPath = 'ci-visibility/features/' +const fileExtension = 'js' versions.forEach(version => { - moduleType.forEach(({ - type, - runTestsCommand, - runTestsWithCoverageCommand, - parallelModeCommand, - featuresPath, - fileExtension - }) => { - // TODO: add esm tests - describe(`cucumber@${version} ${type}`, () => { - let sandbox, cwd, receiver, childProcess, testOutput - - before(async function () { - // add an explicit timeout to make tests less flaky - this.timeout(50000) - - sandbox = await createSandbox([`@cucumber/cucumber@${version}`, 'assert', 'nyc'], true) - cwd = sandbox.folder - }) + // TODO: add esm tests + describe(`cucumber@${version} commonJS`, () => { + let sandbox, cwd, receiver, childProcess, testOutput - after(async function () { - // add an explicit timeout to make tests less flaky - this.timeout(50000) + before(async function () { + // add an explicit timeout to make tests less flaky + this.timeout(50000) - await sandbox.remove() - }) + sandbox = await createSandbox([`@cucumber/cucumber@${version}`, 'assert', 'nyc'], true) + cwd = sandbox.folder + }) - beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() - }) + after(async function () { + // add an explicit timeout to make tests less flaky + this.timeout(50000) - afterEach(async () => { - testOutput = '' - childProcess.kill() - await receiver.stop() - }) + await sandbox.remove() + }) - const reportMethods = ['agentless', 'evp proxy'] + beforeEach(async function () { + const port = await getPort() + receiver = await new FakeCiVisIntake(port).start() + }) - reportMethods.forEach((reportMethod) => { - context(`reporting via ${reportMethod}`, () => { - let envVars, isAgentless - beforeEach(() => { - isAgentless = reportMethod === 'agentless' - envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port) - }) - const runModes = ['serial'] + afterEach(async () => { + testOutput = '' + childProcess.kill() + await receiver.stop() + }) - if (version !== '7.0.0') { // only on latest or 9 if node is old - runModes.push('parallel') - } + const reportMethods = ['agentless', 'evp proxy'] - runModes.forEach((runMode) => { - it(`(${runMode}) can run and report tests`, (done) => { - const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand + reportMethods.forEach((reportMethod) => { + context(`reporting via ${reportMethod}`, () => { + let envVars, isAgentless + beforeEach(() => { + isAgentless = reportMethod === 'agentless' + envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port) + }) + const runModes = ['serial'] + + if (version !== '7.0.0') { // only on latest or 9 if node is old + runModes.push('parallel') + } + + runModes.forEach((runMode) => { + it(`(${runMode}) can run and report tests`, (done) => { + const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) - const receiverPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) - metadataDicts.forEach(metadata => { - for (const testLevel of TEST_LEVEL_EVENT_TYPES) { - assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') - } - }) + const events = payloads.flatMap(({ payload }) => payload.events) - const events = payloads.flatMap(({ payload }) => payload.events) + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') - const testSessionEvent = events.find(event => event.type === 'test_session_end') - const testModuleEvent = events.find(event => event.type === 'test_module_end') - const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') - const testEvents = events.filter(event => event.type === 'test') + const stepEvents = events.filter(event => event.type === 'span') - const stepEvents = events.filter(event => event.type === 'span') + const { content: testSessionEventContent } = testSessionEvent + const { content: testModuleEventContent } = testModuleEvent - const { content: testSessionEventContent } = testSessionEvent - const { content: testModuleEventContent } = testModuleEvent + if (runMode === 'parallel') { + assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true') + } - if (runMode === 'parallel') { - assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true') - } + assert.exists(testSessionEventContent.test_session_id) + assert.exists(testSessionEventContent.meta[TEST_COMMAND]) + assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN]) + assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true) + assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail') + + assert.exists(testModuleEventContent.test_session_id) + assert.exists(testModuleEventContent.test_module_id) + assert.exists(testModuleEventContent.meta[TEST_COMMAND]) + assert.exists(testModuleEventContent.meta[TEST_MODULE]) + assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true) + assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail') + assert.equal( + testModuleEventContent.test_session_id.toString(10), + testSessionEventContent.test_session_id.toString(10) + ) - assert.exists(testSessionEventContent.test_session_id) - assert.exists(testSessionEventContent.meta[TEST_COMMAND]) - assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN]) - assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true) - assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail') - - assert.exists(testModuleEventContent.test_session_id) - assert.exists(testModuleEventContent.test_module_id) - assert.exists(testModuleEventContent.meta[TEST_COMMAND]) - assert.exists(testModuleEventContent.meta[TEST_MODULE]) - assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true) - assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail') - assert.equal( - testModuleEventContent.test_session_id.toString(10), - testSessionEventContent.test_session_id.toString(10) - ) + assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ + `test_suite.${featuresPath}farewell.feature`, + `test_suite.${featuresPath}greetings.feature` + ]) + assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [ + 'pass', + 'fail' + ]) - assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ - `test_suite.${featuresPath}farewell.feature`, - `test_suite.${featuresPath}greetings.feature` - ]) - assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [ - 'pass', - 'fail' - ]) - - testSuiteEvents.forEach(({ - content: { - meta, - metrics, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - } - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) - assert.isTrue(meta[TEST_SOURCE_FILE].startsWith(featuresPath)) - assert.equal(metrics[TEST_SOURCE_START], 1) - assert.exists(metrics[DD_HOST_CPU_COUNT]) - }) + testSuiteEvents.forEach(({ + content: { + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + } + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.isTrue(meta[TEST_SOURCE_FILE].startsWith(featuresPath)) + assert.equal(metrics[TEST_SOURCE_START], 1) + assert.exists(metrics[DD_HOST_CPU_COUNT]) + }) - assert.includeMembers(testEvents.map(test => test.content.resource), [ - `${featuresPath}farewell.feature.Say farewell`, - `${featuresPath}greetings.feature.Say greetings`, - `${featuresPath}greetings.feature.Say yeah`, - `${featuresPath}greetings.feature.Say yo`, - `${featuresPath}greetings.feature.Say skip` - ]) - assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ - 'pass', - 'pass', - 'pass', - 'fail', - 'skip' - ]) - - testEvents.forEach(({ - content: { - meta, - metrics, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - } - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) - assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true) - // Can read DD_TAGS - assert.propertyVal(meta, 'test.customtag', 'customvalue') - assert.propertyVal(meta, 'test.customtag2', 'customvalue2') - if (runMode === 'parallel') { - assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true') - } - assert.exists(metrics[DD_HOST_CPU_COUNT]) - }) + assert.includeMembers(testEvents.map(test => test.content.resource), [ + `${featuresPath}farewell.feature.Say farewell`, + `${featuresPath}greetings.feature.Say greetings`, + `${featuresPath}greetings.feature.Say yeah`, + `${featuresPath}greetings.feature.Say yo`, + `${featuresPath}greetings.feature.Say skip` + ]) + assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ + 'pass', + 'pass', + 'pass', + 'fail', + 'skip' + ]) - stepEvents.forEach(stepEvent => { - assert.equal(stepEvent.content.name, 'cucumber.step') - assert.property(stepEvent.content.meta, 'cucumber.step') - }) - }, 5000) + testEvents.forEach(({ + content: { + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + } + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true) + // Can read DD_TAGS + assert.propertyVal(meta, 'test.customtag', 'customvalue') + assert.propertyVal(meta, 'test.customtag2', 'customvalue2') + if (runMode === 'parallel') { + assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true') + } + assert.exists(metrics[DD_HOST_CPU_COUNT]) + }) - childProcess = exec( - runCommand, - { - cwd, - env: { - ...envVars, - DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', - DD_TEST_SESSION_NAME: 'my-test-session' - }, - stdio: 'pipe' - } - ) + stepEvents.forEach(stepEvent => { + assert.equal(stepEvent.content.name, 'cucumber.step') + assert.property(stepEvent.content.meta, 'cucumber.step') + }) + }, 5000) + + childProcess = exec( + runCommand, + { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', + DD_TEST_SESSION_NAME: 'my-test-session' + }, + stdio: 'pipe' + } + ) - childProcess.on('exit', () => { - receiverPromise.then(() => done()).catch(done) - }) + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) }) }) - context('intelligent test runner', () => { - it('can report git metadata', (done) => { - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url.endsWith('/api/v2/git/repository/search_commits') + }) + context('intelligent test runner', () => { + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url.endsWith('/api/v2/git/repository/search_commits') + ) + const packfileRequestPromise = receiver + .payloadReceived(({ url }) => url.endsWith('/api/v2/git/repository/packfile')) + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + if (isAgentless) { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + } else { + assert.notProperty(searchCommitRequest.headers, 'dd-api-key') + assert.notProperty(packfileRequest.headers, 'dd-api-key') + } + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) - const packfileRequestPromise = receiver - .payloadReceived(({ url }) => url.endsWith('/api/v2/git/repository/packfile')) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + assert.equal(numSuites, 2) - Promise.all([ - searchCommitsRequestPromise, - packfileRequestPromise, - eventsRequestPromise - ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { - if (isAgentless) { - assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') - assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') - } else { - assert.notProperty(searchCommitRequest.headers, 'dd-api-key') - assert.notProperty(packfileRequest.headers, 'dd-api-key') - } + done() + }).catch(done) - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + }) + it('can report code coverage', (done) => { + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url.endsWith('/api/v2/libraries/tests/services/setting') + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + const [coveragePayload] = codeCovRequest.payload + if (isAgentless) { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + } else { + assert.notProperty(libraryConfigRequest.headers, 'dd-api-key') + assert.notProperty(codeCovRequest.headers, 'dd-api-key', '1') + } + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, [ + `${featuresPath}support/steps.${fileExtension}`, + `${featuresPath}farewell.feature`, + `${featuresPath}greetings.feature` + ]) + // steps is twice because there are two suites using it + assert.equal( + allCoverageFiles.filter(file => file === `${featuresPath}support/steps.${fileExtension}`).length, + 2 + ) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest + .payload + .events + .find(event => event.type === 'test_session_end') + .content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) - done() - }).catch(done) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + // check that reported coverage is still the same + assert.include(testOutput, 'Lines : 100%') + done() + }) + }) + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) - childProcess = exec( - runTestsCommand, - { - cwd, - env: envVars, - stdio: 'pipe' + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url.endsWith('/api/v2/citestcov')).catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + }) + it('can skip suites received by the intelligent test runner API and still reports code coverage', + (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` } - ) - }) - it('can report code coverage', (done) => { - const libraryConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url.endsWith('/api/v2/libraries/tests/services/setting') - ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) + }]) + + const skippableRequestPromise = receiver + .payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) Promise.all([ - libraryConfigRequestPromise, - codeCovRequestPromise, + skippableRequestPromise, + coverageRequestPromise, eventsRequestPromise - ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { - const [coveragePayload] = codeCovRequest.payload + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + const [coveragePayload] = coverageRequest.payload if (isAgentless) { - assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') - assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') } else { - assert.notProperty(libraryConfigRequest.headers, 'dd-api-key') - assert.notProperty(codeCovRequest.headers, 'dd-api-key', '1') + assert.notProperty(skippableRequest.headers, 'dd-api-key', '1') + assert.notProperty(coverageRequest.headers, 'dd-api-key', '1') + assert.notProperty(eventsRequest.headers, 'dd-api-key', '1') } - assert.propertyVal(coveragePayload, 'name', 'coverage1') assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 - }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, [ - `${featuresPath}support/steps.${fileExtension}`, - `${featuresPath}farewell.feature`, - `${featuresPath}greetings.feature` - ]) - // steps is twice because there are two suites using it - assert.equal( - allCoverageFiles.filter(file => file === `${featuresPath}support/steps.${fileExtension}`).length, - 2 - ) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) - - const testSession = eventsRequest - .payload - .events - .find(event => event.type === 'test_session_end') - .content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) const eventTypes = eventsRequest.payload.events.map(event => event.type) + + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === `test_suite.${featuresPath}farewell.feature` + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) assert.equal(numSuites, 2) - }).catch(done) + const testSession = eventsRequest + .payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'pipe' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - // check that reported coverage is still the same - assert.include(testOutput, 'Lines : 100%') + const testModule = eventsRequest + .payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) done() - }) - }) - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url.endsWith('/api/v2/citestcov')).catch(() => {}) - - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + }).catch(done) childProcess = exec( runTestsWithCoverageCommand, @@ -404,692 +463,763 @@ versions.forEach(version => { } ) }) - it('can skip suites received by the intelligent test runner API and still reports code coverage', - (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } - }]) - - const skippableRequestPromise = receiver - .payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - const [coveragePayload] = coverageRequest.payload - if (isAgentless) { - assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') - assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') - assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') - } else { - assert.notProperty(skippableRequest.headers, 'dd-api-key', '1') - assert.notProperty(coverageRequest.headers, 'dd-api-key', '1') - assert.notProperty(eventsRequest.headers, 'dd-api-key', '1') - } - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + }) + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) - const eventTypes = eventsRequest.payload.events.map(event => event.type) + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === `test_suite.${featuresPath}farewell.feature` - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }, + { + type: 'suite', + attributes: { + suite: `${featuresPath}greetings.feature` + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 2) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' + ).content + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' + ).content + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.meta, TEST_STATUS, 'fail') + assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = eventsRequest - .payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) - - const testModule = eventsRequest - .payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) - done() - }).catch(done) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) - }) - it('does not skip tests if git metadata upload fails', (done) => { - receiver.setSuitesToSkip([{ + receiver.setSuitesToSkip([ + { type: 'suite', attributes: { suite: `${featuresPath}farewell.feature` } - }]) + } + ]) - receiver.setGitUploadStatus(404) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + assert.equal(suites.length, 2) - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' ) - assert.equal(numSuites, 2) - const testSession = payload.events.find(event => event.type === 'test_session_end').content + const failedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' + ) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(failedSuite.content.meta, TEST_STATUS, 'fail') + assert.propertyVal(failedSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(failedSuite.content.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}not-existing.feature` + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 0) + const testModule = events.find(event => event.type === 'test_module_end').content assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 0) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } - }]) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + }) + if (!isAgentless) { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting') + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + `${featuresPath}farewell.feature.Say farewell`, + `${featuresPath}greetings.feature.Say greetings`, + `${featuresPath}greetings.feature.Say yeah`, + `${featuresPath}greetings.feature.Say yo`, + `${featuresPath}greetings.feature.Say skip` + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'inherit' + } ) - assert.equal(numSuites, 2) - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + }) + }) + } + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' + context('early flake detection', () => { + it('retries new tests', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ) + } }) - it('does not skip suites if suite is marked as unskippable', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) - - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } - }, - { - type: 'suite', - attributes: { - suite: `${featuresPath}greetings.feature` - } + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 2) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_session_end').content - - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') - - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' - ).content - const forcedToRunSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' - ).content - - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(forcedToRunSuite.meta, TEST_STATUS, 'fail') - assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_FORCED_RUN, 'true') - }, 25000) + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + const newTests = tests.filter(test => + test.resource === 'ci-visibility/features/farewell.feature.Say whatever' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'Say whatever') + }) }) + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - it('only sets forced to run if suite was going to be skipped by ITR', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) + }) - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 2) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_session_end').content - - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) - - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' - ) - const failedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' - ) - - assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + } + }) - assert.propertyVal(failedSuite.content.meta, TEST_STATUS, 'fail') - assert.propertyVal(failedSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(failedSuite.content.meta, TEST_ITR_FORCED_RUN) - }, 25000) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + // new tests are not detected + assert.equal(newTests.length, 0) }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } }) - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}not-existing.feature` - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 0) - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 0) - }, 25000) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) + childProcess = exec( + runTestsCommand, + { + cwd, + env: { ...envVars, DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - if (!isAgentless) { - context('if the agent is not event platform proxy compatible', () => { - it('does not do any intelligent test runner request', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/api/v2/git/repository/search_commits') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/api/v2/libraries/tests/services/setting') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting') - - receiver.assertPayloadReceived(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - `${featuresPath}farewell.feature.Say farewell`, - `${featuresPath}greetings.feature.Say greetings`, - `${featuresPath}greetings.feature.Say yeah`, - `${featuresPath}greetings.feature.Say yo`, - `${featuresPath}greetings.feature.Say skip` - ] - ) - }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - }) - } - it('reports itr_correlation_id in test suites', (done) => { - const itrCorrelationId = '4321' - receiver.setItrCorrelationId(itrCorrelationId) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - testSuites.forEach(testSuite => { - assert.equal(testSuite.itr_correlation_id, itrCorrelationId) - }) - }, 25000) + }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) + } }) - }) + // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new + receiver.setKnownTests({}) - context('early flake detection', () => { - it('retries new tests', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - } - } - }) - // "cucumber.ci-visibility/features/farewell.feature.Say" whatever will be considered new - receiver.setKnownTests( - { - cucumber: { - 'ci-visibility/features/farewell.feature': ['Say farewell'], - 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] - } - } - ) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - const newTests = tests.filter(test => - test.resource === 'ci-visibility/features/farewell.feature.Say whatever' - ) - newTests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) - const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - newTests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Test name does not change - newTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'Say whatever') - }) + tests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // All test suites pass, even though there are failed tests + testSuites.forEach(testSuite => { + assert.propertyVal(testSuite.meta, TEST_STATUS, 'pass') }) - childProcess = exec( - runTestsCommand, - { - cwd, - env: envVars, - stdio: 'pipe' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - } - } - }) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + const passedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - const tests = events.filter(event => event.type === 'test').map(event => event.content) - const newTests = tests.filter(test => - test.meta[TEST_IS_NEW] === 'true' - ) - // new tests are not detected - assert.equal(newTests.length, 0) - }) - // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new - receiver.setKnownTests({ - cucumber: { - 'ci-visibility/features/farewell.feature': ['Say farewell'], - 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] - } + // (1 original run + 3 retries) / 2 + assert.equal(failedAttempts.length, 2) + assert.equal(passedAttempts.length, 2) }) - childProcess = exec( - runTestsCommand, - { - cwd, - env: { ...envVars, DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' }, - stdio: 'pipe' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-flaky/*.feature', + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', (exitCode) => { + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) }) - it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - } - } - }) - // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new - receiver.setKnownTests({}) + }) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) + it('does not retry tests that are skipped', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new + // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo'] + } + }) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - const tests = events.filter(event => event.type === 'test').map(event => event.content) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - tests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) - const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - const passedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const skippedNewTest = tests.filter(test => + test.resource === 'ci-visibility/features/greetings.feature.Say skip' + ) + // not retried + assert.equal(skippedNewTest.length, 1) + }) - // (1 original run + 3 retries) / 2 - assert.equal(failedAttempts.length, 2) - assert.equal(passedAttempts.length, 2) - }) + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) - childProcess = exec( - './node_modules/.bin/cucumber-js ci-visibility/features-flaky/*.feature', - { - cwd, - env: envVars, - stdio: 'pipe' + it('does not run EFD if the known tests request fails', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ) - childProcess.on('exit', (exitCode) => { - assert.equal(exitCode, 0) - eventsPromise.then(() => { - done() - }).catch(done) - }) + } }) - it('does not retry tests that are skipped', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - } - } + receiver.setKnownTestsResponseCode(500) + receiver.setKnownTests({}) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 6) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) }) - // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new - // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new - receiver.setKnownTests({ + + childProcess = exec( + runTestsCommand, + { cwd, env: envVars, stdio: 'pipe' } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('bails out of EFD if the percentage of new tests is too high', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 + } + }) + // tests in cucumber.ci-visibility/features/farewell.feature will be considered new + receiver.setKnownTests( + { cucumber: { - 'ci-visibility/features/farewell.feature': ['Say farewell'], - 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo'] + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] } - }) + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - const tests = events.filter(event => event.type === 'test').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) - const skippedNewTest = tests.filter(test => - test.resource === 'ci-visibility/features/greetings.feature.Say skip' - ) - // not retried - assert.equal(skippedNewTest.length, 1) - }) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) - childProcess = exec( - runTestsCommand, - { - cwd, - env: envVars, - stdio: 'pipe' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) }) + + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - it('does not run EFD if the known tests request fails', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD + }) + + if (version !== '7.0.0') { // EFD in parallel mode only supported from cucumber>=11 + context('parallel mode', () => { + it('retries new tests', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } } - } - }) - receiver.setKnownTestsResponseCode(500) - receiver.setKnownTests({}) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) - const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') - assert.equal(tests.length, 6) - const newTests = tests.filter(test => - test.meta[TEST_IS_NEW] === 'true' - ) - assert.equal(newTests.length, 0) - }) + const tests = events.filter(event => event.type === 'test').map(event => event.content) - childProcess = exec( - runTestsCommand, - { cwd, env: envVars, stdio: 'pipe' } - ) + const newTests = tests.filter(test => + test.resource === 'ci-visibility/features/farewell.feature.Say whatever' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + // Test name does not change + assert.propertyVal(test.meta, TEST_NAME, 'Say whatever') + assert.propertyVal(test.meta, CUCUMBER_IS_PARALLEL, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + childProcess = exec( + parallelModeCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) }) - }) - }) - if (version === 'latest') { // flaky test retries only supported from >=8.0.0 - context('flaky test retries', () => { - it('can retry failed tests', (done) => { + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + const NUM_RETRIES_EFD = 3 receiver.setSettings({ itr_enabled: false, code_coverage: false, tests_skipping: false, - flaky_test_retries_enabled: true, early_flake_detection: { - enabled: false + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } } }) + // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new + receiver.setKnownTests({}) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSuites = events + .filter(event => event.type === 'test_suite_end').map(event => event.content) - // 2 failures and 1 passed attempt - assert.equal(tests.length, 3) + tests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + assert.propertyVal(test.meta, CUCUMBER_IS_PARALLEL, 'true') + }) - const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - assert.equal(failedTests.length, 2) - const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - assert.equal(passedTests.length, 1) + // All test suites pass, even though there are failed tests + testSuites.forEach(testSuite => { + assert.propertyVal(testSuite.meta, TEST_STATUS, 'pass') + }) - // All but the first one are retries - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 2) + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + const passedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + + // (1 original run + 3 retries) / 2 + assert.equal(failedAttempts.length, 2) + assert.equal(passedAttempts.length, 2) }) childProcess = exec( - './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + './node_modules/.bin/cucumber-js ci-visibility/features-flaky/*.feature --parallel 2', { cwd, env: envVars, @@ -1097,44 +1227,60 @@ versions.forEach(version => { } ) - childProcess.on('exit', () => { + childProcess.on('exit', (exitCode) => { + assert.equal(exitCode, 0) eventsPromise.then(() => { done() }).catch(done) }) }) - it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + it('bails out of EFD if the percentage of new tests is too high', (done) => { + const NUM_RETRIES_EFD = 3 receiver.setSettings({ itr_enabled: false, code_coverage: false, tests_skipping: false, - flaky_test_retries_enabled: true, early_flake_detection: { - enabled: false + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 } }) + // tests in cucumber.ci-visibility/features/farewell.feature will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.equal(tests.length, 1) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedTests.length, 0) }) childProcess = exec( - './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + parallelModeCommand, { cwd, - env: { - ...envVars, - DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false' - }, + env: envVars, stdio: 'pipe' } ) @@ -1146,14 +1292,25 @@ versions.forEach(version => { }) }) - it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + it('does not retry tests that are skipped', (done) => { + const NUM_RETRIES_EFD = 3 receiver.setSettings({ itr_enabled: false, code_coverage: false, tests_skipping: false, - flaky_test_retries_enabled: true, early_flake_detection: { - enabled: false + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new + // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo'] } }) @@ -1161,33 +1318,26 @@ versions.forEach(version => { .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') const tests = events.filter(event => event.type === 'test').map(event => event.content) - // 2 failures - assert.equal(tests.length, 2) - - const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - assert.equal(failedTests.length, 2) - const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - assert.equal(passedTests.length, 0) - - // All but the first one are retries - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 1) + const skippedNewTest = tests.filter(test => + test.resource === 'ci-visibility/features/greetings.feature.Say skip' + ) + // not retried + assert.equal(skippedNewTest.length, 1) }) childProcess = exec( - './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + parallelModeCommand, { cwd, - env: { - ...envVars, - DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1 - }, + env: envVars, stdio: 'pipe' } ) - childProcess.on('exit', () => { eventsPromise.then(() => { done() @@ -1197,55 +1347,235 @@ versions.forEach(version => { }) } }) - }) - it('correctly calculates test code owners when working directory is not repository root', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) + if (version === 'latest') { // flaky test retries only supported from >=8.0.0 + context('flaky test retries', () => { + it('can retry failed tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // 2 failures and 1 passed attempt + assert.equal(tests.length, 3) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedTests.length, 1) + + // All but the first one are retries + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 2) + }) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + { + cwd, + env: { + ...envVars, + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // 2 failures + assert.equal(tests.length, 2) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedTests.length, 0) + + // All but the first one are retries + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 1) + }) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + { + cwd, + env: { + ...envVars, + DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1 + }, + stdio: 'pipe' + } + ) - const test = events.find(event => event.type === 'test').content - const testSuite = events.find(event => event.type === 'test_suite_end').content - // The test is in a subproject - assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) - assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) - assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) }) + } + }) + }) - childProcess = exec( - 'node ../../node_modules/.bin/cucumber-js features/*.feature', - { - cwd: `${cwd}/ci-visibility/subproject`, - env: { - ...getCiVisAgentlessConfig(receiver.port) - }, - stdio: 'inherit' - } - ) + it('correctly calculates test code owners when working directory is not repository root', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const test = events.find(event => event.type === 'test').content + const testSuite = events.find(event => event.type === 'test_suite_end').content + // The test is in a subproject + assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) + assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + childProcess = exec( + 'node ../../node_modules/.bin/cucumber-js features/*.feature', + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...getCiVisAgentlessConfig(receiver.port) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('takes into account untested files if "all" is passed to nyc', (done) => { + const linesPctMatchRegex = /Lines\s*:\s*([\d.]+)%/ + let linesPctMatch + let linesPctFromNyc = 0 + let codeCoverageWithUntestedFiles = 0 + let codeCoverageWithoutUntestedFiles = 0 + + let eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + codeCoverageWithUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] }) + + childProcess = exec( + './node_modules/nyc/bin/nyc.js --all -r=text-summary --nycrc-path ./my-nyc.config.js ' + + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NYC_INCLUDE: JSON.stringify( + [ + 'ci-visibility/features/**', + 'ci-visibility/features-esm/**' + ] + ) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() }) - it('takes into account untested files if "all" is passed to nyc', (done) => { - const linesPctMatchRegex = /Lines\s*:\s*([\d.]+)%/ - let linesPctMatch - let linesPctFromNyc = 0 - let codeCoverageWithUntestedFiles = 0 - let codeCoverageWithoutUntestedFiles = 0 + childProcess.on('exit', () => { + linesPctMatch = testOutput.match(linesPctMatchRegex) + linesPctFromNyc = linesPctMatch ? Number(linesPctMatch[1]) : null - let eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - codeCoverageWithUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] - }) + assert.equal( + linesPctFromNyc, + codeCoverageWithUntestedFiles, + 'nyc --all output does not match the reported coverage' + ) + // reset test output for next test session + testOutput = '' + // we run the same tests without the all flag childProcess = exec( - './node_modules/nyc/bin/nyc.js --all -r=text-summary --nycrc-path ./my-nyc.config.js ' + + './node_modules/nyc/bin/nyc.js -r=text-summary --nycrc-path ./my-nyc.config.js ' + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', { cwd, @@ -1262,6 +1592,13 @@ versions.forEach(version => { } ) + eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + codeCoverageWithoutUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] + }) + childProcess.stdout.on('data', (chunk) => { testOutput += chunk.toString() }) @@ -1275,60 +1612,14 @@ versions.forEach(version => { assert.equal( linesPctFromNyc, - codeCoverageWithUntestedFiles, - 'nyc --all output does not match the reported coverage' - ) - - // reset test output for next test session - testOutput = '' - // we run the same tests without the all flag - childProcess = exec( - './node_modules/nyc/bin/nyc.js -r=text-summary --nycrc-path ./my-nyc.config.js ' + - 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - NYC_INCLUDE: JSON.stringify( - [ - 'ci-visibility/features/**', - 'ci-visibility/features-esm/**' - ] - ) - }, - stdio: 'inherit' - } + codeCoverageWithoutUntestedFiles, + 'nyc output does not match the reported coverage (no --all flag)' ) - eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - codeCoverageWithoutUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] - }) - - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - - childProcess.on('exit', () => { - linesPctMatch = testOutput.match(linesPctMatchRegex) - linesPctFromNyc = linesPctMatch ? Number(linesPctMatch[1]) : null - - assert.equal( - linesPctFromNyc, - codeCoverageWithoutUntestedFiles, - 'nyc output does not match the reported coverage (no --all flag)' - ) - - eventsPromise.then(() => { - assert.isAbove(codeCoverageWithoutUntestedFiles, codeCoverageWithUntestedFiles) - done() - }).catch(done) - }) + eventsPromise.then(() => { + assert.isAbove(codeCoverageWithoutUntestedFiles, codeCoverageWithUntestedFiles) + done() + }).catch(done) }) }) }) diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index abe46f21aaf..0f84d717381 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -35,7 +35,8 @@ const { mergeCoverage, fromCoverageMapToCoverage, getTestSuitePath, - CUCUMBER_WORKER_TRACE_PAYLOAD_CODE + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + getIsFaultyEarlyFlakeDetection } = require('../../dd-trace/src/plugins/util/test') const isMarkedAsUnskippable = (pickle) => { @@ -51,6 +52,7 @@ const patched = new WeakSet() const lastStatusByPickleId = new Map() const numRetriesByPickleId = new Map() const numAttemptToAsyncResource = new Map() +const newTestsByTestFullname = new Map() let eventDataCollector = null let pickleByFile = {} @@ -65,6 +67,8 @@ let isUnskippable = false let isSuitesSkippingEnabled = false let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 +let earlyFlakeDetectionFaultyThreshold = 0 +let isEarlyFlakeDetectionFaulty = false let isFlakyTestRetriesEnabled = false let numTestRetries = 0 let knownTests = [] @@ -351,6 +355,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin isEarlyFlakeDetectionEnabled = configurationResponse.libraryConfig?.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = configurationResponse.libraryConfig?.earlyFlakeDetectionNumRetries + earlyFlakeDetectionFaultyThreshold = configurationResponse.libraryConfig?.earlyFlakeDetectionFaultyThreshold isSuitesSkippingEnabled = configurationResponse.libraryConfig?.isSuitesSkippingEnabled isFlakyTestRetriesEnabled = configurationResponse.libraryConfig?.isFlakyTestRetriesEnabled numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount @@ -397,6 +402,18 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin pickleByFile = isCoordinator ? getPickleByFileNew(this) : getPickleByFile(this) + if (isEarlyFlakeDetectionEnabled) { + const isFaulty = getIsFaultyEarlyFlakeDetection( + Object.keys(pickleByFile), + knownTests.cucumber || {}, + earlyFlakeDetectionFaultyThreshold + ) + if (isFaulty) { + isEarlyFlakeDetectionEnabled = false + isEarlyFlakeDetectionFaulty = true + } + } + const processArgv = process.argv.slice(2).join(' ') const command = process.env.npm_lifecycle_script || `cucumber-js ${processArgv}` @@ -443,6 +460,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin hasUnskippableSuites: isUnskippable, hasForcedToRunSuites: isForcedToRun, isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, isParallel }) }) @@ -451,7 +469,9 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin } } -function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) { +// Generates suite start and finish events in the main process. +// Handles EFD in both the main process and the worker process. +function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = false, isWorker = false) { return async function () { let pickle if (isNewerCucumberVersion) { @@ -463,7 +483,8 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) { const testFileAbsolutePath = pickle.uri const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd()) - if (!pickleResultByFile[testFileAbsolutePath]) { // first test in suite + // If it's a worker, suite events are handled in `getWrappedParseWorkerMessage` + if (!isWorker && !pickleResultByFile[testFileAbsolutePath]) { // first test in suite isUnskippable = isMarkedAsUnskippable(pickle) isForcedToRun = isUnskippable && skippableSuites.includes(testSuitePath) @@ -519,8 +540,9 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) { pickleResultByFile[testFileAbsolutePath].push(testStatus) } - // last test in suite - if (pickleResultByFile[testFileAbsolutePath].length === pickleByFile[testFileAbsolutePath].length) { + // If it's a worker, suite events are handled in `getWrappedParseWorkerMessage` + if (!isWorker && pickleResultByFile[testFileAbsolutePath].length === pickleByFile[testFileAbsolutePath].length) { + // last test in suite const testSuiteStatus = getSuiteStatusFromTestStatuses(pickleResultByFile[testFileAbsolutePath]) if (global.__coverage__) { const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) @@ -539,7 +561,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) { testSuiteFinishCh.publish({ status: testSuiteStatus, testSuitePath }) } - if (isNewerCucumberVersion && isNew && isEarlyFlakeDetectionEnabled) { + if (isNewerCucumberVersion && isEarlyFlakeDetectionEnabled && isNew) { return shouldBePassedByEFD } @@ -618,16 +640,40 @@ function getWrappedParseWorkerMessage (parseWorkerMessageFunction, isNewVersion) pickle = testCase.pickle } - // TODO: can we get error message? const { status } = getStatusFromResultLatest(worstTestStepResult) + let isNew = false + + if (isEarlyFlakeDetectionEnabled) { + isNew = isNewTest(pickle.uri, pickle.name) + } const testFileAbsolutePath = pickle.uri const finished = pickleResultByFile[testFileAbsolutePath] - finished.push(status) + + if (isNew) { + const testFullname = `${pickle.uri}:${pickle.name}` + let testStatuses = newTestsByTestFullname.get(testFullname) + if (!testStatuses) { + testStatuses = [status] + newTestsByTestFullname.set(testFullname, testStatuses) + } else { + testStatuses.push(status) + } + // We have finished all retries + if (testStatuses.length === earlyFlakeDetectionNumRetries + 1) { + const newTestFinalStatus = getTestStatusFromRetries(testStatuses) + // we only push to `finished` if the retries have finished + finished.push(newTestFinalStatus) + } + } else { + // TODO: can we get error message? + const finished = pickleResultByFile[testFileAbsolutePath] + finished.push(status) + } if (finished.length === pickleByFile[testFileAbsolutePath].length) { testSuiteFinishCh.publish({ - status: getSuiteStatusFromTestStatuses(finished), // maybe tests themselves can add to this list + status: getSuiteStatusFromTestStatuses(finished), testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd()) }) } @@ -645,7 +691,6 @@ addHook({ }, pickleHook) // Test start / finish for newer versions. The only hook executed in workers when in parallel mode - addHook({ name: '@cucumber/cucumber', versions: ['>=7.3.0'], @@ -701,15 +746,19 @@ addHook({ }) // >=11.0.0 hooks -// `getWrappedRunTestCase` generates suite start and finish events and handles EFD. +// `getWrappedRunTestCase` does two things: +// - generates suite start and finish events in the main process, +// - handles EFD in both the main process and the worker process. addHook({ name: '@cucumber/cucumber', versions: ['>=11.0.0'], file: 'lib/runtime/worker.js' }, (workerPackage) => { - if (!process.env.CUCUMBER_WORKER_ID) { - shimmer.wrap(workerPackage.Worker.prototype, 'runTestCase', runTestCase => getWrappedRunTestCase(runTestCase, true)) - } + shimmer.wrap( + workerPackage.Worker.prototype, + 'runTestCase', + runTestCase => getWrappedRunTestCase(runTestCase, true, !!process.env.CUCUMBER_WORKER_ID) + ) return workerPackage }) @@ -740,8 +789,9 @@ addHook({ return eventDataCollectorPackage }) -// Only executed in parallel mode for >=11. +// Only executed in parallel mode for >=11, in the main process. // `getWrappedParseWorkerMessage` generates suite start and finish events +// In `startWorker` we pass early flake detection info to the worker. addHook({ name: '@cucumber/cucumber', versions: ['>=11.0.0'], @@ -752,5 +802,37 @@ addHook({ 'parseWorkerMessage', parseWorkerMessage => getWrappedParseWorkerMessage(parseWorkerMessage, true) ) + // EFD in parallel mode only supported in >=11.0.0 + shimmer.wrap(adapterPackage.ChildProcessAdapter.prototype, 'startWorker', startWorker => function () { + if (isEarlyFlakeDetectionEnabled) { + this.options.worldParameters._ddKnownTests = knownTests + this.options.worldParameters._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + } + + return startWorker.apply(this, arguments) + }) return adapterPackage }) + +// Hook executed in the worker process when in parallel mode. +// In this hook we read the information passed in `worldParameters` and make it available for +// `getWrappedRunTestCase`. +addHook({ + name: '@cucumber/cucumber', + versions: ['>=11.0.0'], + file: 'lib/runtime/parallel/worker.js' +}, (workerPackage) => { + shimmer.wrap( + workerPackage.ChildProcessWorker.prototype, + 'initialize', + initialize => async function () { + await initialize.apply(this, arguments) + isEarlyFlakeDetectionEnabled = !!this.options.worldParameters._ddKnownTests + if (isEarlyFlakeDetectionEnabled) { + knownTests = this.options.worldParameters._ddKnownTests + earlyFlakeDetectionNumRetries = this.options.worldParameters._ddEarlyFlakeDetectionNumRetries + } + } + ) + return workerPackage +}) diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 7aaf264d763..d24f97c33e6 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -17,6 +17,7 @@ const { ITR_CORRELATION_ID, TEST_SOURCE_FILE, TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, TEST_IS_NEW, TEST_IS_RETRY, TEST_SUITE_ID, @@ -79,6 +80,7 @@ class CucumberPlugin extends CiPlugin { hasUnskippableSuites, hasForcedToRunSuites, isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, isParallel }) => { const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {} @@ -99,6 +101,9 @@ class CucumberPlugin extends CiPlugin { if (isEarlyFlakeDetectionEnabled) { this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') } + if (isEarlyFlakeDetectionFaulty) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + } if (isParallel) { this.testSessionSpan.setTag(CUCUMBER_IS_PARALLEL, 'true') }