Skip to content

Commit

Permalink
[test visibility] Add test session name (#4621)
Browse files Browse the repository at this point in the history
  • Loading branch information
@juan-fernandez
juan-fernandez authored Sep 2, 2024
1 parent 34a651e commit 9949796
Show file tree
Hide file tree
Showing 18 changed files with 182 additions and 48 deletions.
10 changes: 8 additions & 2 deletions integration-tests/cucumber/cucumber.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ const {
TEST_NAME,
CUCUMBER_IS_PARALLEL,
TEST_SUITE,
TEST_CODE_OWNERS
TEST_CODE_OWNERS,
TEST_SESSION_NAME
} = require('../../packages/dd-trace/src/plugins/util/test')

const isOldNode = semver.satisfies(process.version, '<=16')
Expand Down Expand Up @@ -129,12 +130,14 @@ versions.forEach(version => {
assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true')
}

assert.equal(testSessionEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testSessionEventContent.test_session_id)
assert.exists(testSessionEventContent.meta[TEST_COMMAND])
assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN])
assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true)
assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail')

assert.equal(testModuleEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testModuleEventContent.test_session_id)
assert.exists(testModuleEventContent.test_module_id)
assert.exists(testModuleEventContent.meta[TEST_COMMAND])
Expand Down Expand Up @@ -163,6 +166,7 @@ versions.forEach(version => {
test_session_id: testSessionId
}
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand Down Expand Up @@ -193,6 +197,7 @@ versions.forEach(version => {
test_session_id: testSessionId
}
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand All @@ -219,7 +224,8 @@ versions.forEach(version => {
cwd,
env: {
...envVars,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2',
DD_SESSION_NAME: 'my-test-session'
},
stdio: 'pipe'
}
Expand Down
10 changes: 8 additions & 2 deletions integration-tests/cypress/cypress.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ const {
TEST_IS_RETRY,
TEST_EARLY_FLAKE_ENABLED,
TEST_SUITE,
TEST_CODE_OWNERS
TEST_CODE_OWNERS,
TEST_SESSION_NAME
} = require('../../packages/dd-trace/src/plugins/util/test')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
const { NODE_MAJOR } = require('../../version')
Expand Down Expand Up @@ -235,12 +236,14 @@ moduleTypes.forEach(({
const { content: testSessionEventContent } = testSessionEvent
const { content: testModuleEventContent } = testModuleEvent

assert.equal(testSessionEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testSessionEventContent.test_session_id)
assert.exists(testSessionEventContent.meta[TEST_COMMAND])
assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN])
assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true)
assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail')

assert.equal(testModuleEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testModuleEventContent.test_session_id)
assert.exists(testModuleEventContent.test_module_id)
assert.exists(testModuleEventContent.meta[TEST_COMMAND])
Expand Down Expand Up @@ -271,6 +274,7 @@ moduleTypes.forEach(({
test_session_id: testSessionId
}
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand Down Expand Up @@ -298,6 +302,7 @@ moduleTypes.forEach(({
test_session_id: testSessionId
}
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand All @@ -322,7 +327,8 @@ moduleTypes.forEach(({
env: {
...restEnvVars,
CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2',
DD_SESSION_NAME: 'my-test-session'
},
stdio: 'pipe'
}
Expand Down
36 changes: 27 additions & 9 deletions integration-tests/jest/jest.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ const {
JEST_DISPLAY_NAME,
TEST_EARLY_FLAKE_ABORT_REASON,
TEST_SOURCE_START,
TEST_CODE_OWNERS
TEST_CODE_OWNERS,
TEST_SESSION_NAME
} = require('../../packages/dd-trace/src/plugins/util/test')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')

Expand Down Expand Up @@ -149,19 +150,23 @@ describe('jest CommonJS', () => {
)
assert.equal(suites.length, 2)
assert.exists(sessionEventContent)
assert.equal(sessionEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(moduleEventContent)
assert.equal(moduleEventContent.meta[TEST_SESSION_NAME], 'my-test-session')

assert.include(testOutput, expectedStdout)

// Can read DD_TAGS
tests.forEach(testEvent => {
assert.equal(testEvent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true)
assert.exists(testEvent.metrics[TEST_SOURCE_START])
// Can read DD_TAGS
assert.propertyVal(testEvent.meta, 'test.customtag', 'customvalue')
assert.propertyVal(testEvent.meta, 'test.customtag2', 'customvalue2')
})

tests.forEach(testEvent => {
assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true)
assert.exists(testEvent.metrics[TEST_SOURCE_START])
suites.forEach(testSuite => {
assert.equal(testSuite.meta[TEST_SESSION_NAME], 'my-test-session')
})

done()
Expand All @@ -171,7 +176,8 @@ describe('jest CommonJS', () => {
cwd,
env: {
...envVars,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2',
DD_SESSION_NAME: 'my-test-session'
},
stdio: 'pipe'
})
Expand Down Expand Up @@ -428,17 +434,29 @@ describe('jest CommonJS', () => {
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
RUN_IN_PARALLEL: true
RUN_IN_PARALLEL: true,
DD_SESSION_NAME: 'my-test-session'
},
stdio: 'pipe'
})

receiver.gatherPayloads(({ url }) => url === '/api/v2/citestcycle', 5000).then(eventsRequests => {
const eventTypes = eventsRequests.map(({ payload }) => payload)
const events = eventsRequests.map(({ payload }) => payload)
.flatMap(({ events }) => events)
.map(event => event.type)
const eventTypes = events.map(event => event.type)

assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end'])
const tests = events.filter(event => event.type === 'test').map(event => event.content)
const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content)

// it propagates test session name to the test and test suite events in parallel mode
tests.forEach(testEvent => {
assert.equal(testEvent.meta[TEST_SESSION_NAME], 'my-test-session')
})
testSuites.forEach(testSuite => {
assert.equal(testSuite.meta[TEST_SESSION_NAME], 'my-test-session')
})

done()
}).catch(done)
})
Expand Down
23 changes: 16 additions & 7 deletions integration-tests/mocha/mocha.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ const {
TEST_MODULE,
MOCHA_IS_PARALLEL,
TEST_SOURCE_START,
TEST_CODE_OWNERS
TEST_CODE_OWNERS,
TEST_SESSION_NAME
} = require('../../packages/dd-trace/src/plugins/util/test')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')

Expand Down Expand Up @@ -148,20 +149,24 @@ describe('mocha CommonJS', function () {
)
assert.equal(suites.length, 2)
assert.exists(sessionEventContent)
assert.equal(sessionEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(moduleEventContent)
assert.equal(moduleEventContent.meta[TEST_SESSION_NAME], 'my-test-session')

assert.include(testOutput, expectedStdout)
assert.include(testOutput, extraStdout)

// Can read DD_TAGS
tests.forEach(testEvent => {
assert.equal(testEvent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true)
assert.exists(testEvent.metrics[TEST_SOURCE_START])
// Can read DD_TAGS
assert.propertyVal(testEvent.meta, 'test.customtag', 'customvalue')
assert.propertyVal(testEvent.meta, 'test.customtag2', 'customvalue2')
})

tests.forEach(testEvent => {
assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true)
assert.exists(testEvent.metrics[TEST_SOURCE_START])
suites.forEach(testSuite => {
assert.equal(testSuite.meta[TEST_SESSION_NAME], 'my-test-session')
})

done()
Expand All @@ -171,7 +176,8 @@ describe('mocha CommonJS', function () {
cwd,
env: {
...envVars,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2',
DD_SESSION_NAME: 'my-test-session'
},
stdio: 'pipe'
})
Expand Down Expand Up @@ -324,6 +330,7 @@ describe('mocha CommonJS', function () {
test_module_id: testModuleId,
test_session_id: testSessionId
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand All @@ -338,6 +345,7 @@ describe('mocha CommonJS', function () {
test_module_id: testModuleId,
test_session_id: testSessionId
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand All @@ -354,7 +362,8 @@ describe('mocha CommonJS', function () {
...getCiVisAgentlessConfig(receiver.port),
RUN_IN_PARALLEL: true,
DD_TRACE_DEBUG: 1,
DD_TRACE_LOG_LEVEL: 'warn'
DD_TRACE_LOG_LEVEL: 'warn',
DD_SESSION_NAME: 'my-test-session'
},
stdio: 'pipe'
})
Expand Down
10 changes: 8 additions & 2 deletions integration-tests/playwright/playwright.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ const {
TEST_IS_RETRY,
TEST_EARLY_FLAKE_ENABLED,
TEST_SUITE,
TEST_CODE_OWNERS
TEST_CODE_OWNERS,
TEST_SESSION_NAME
} = require('../../packages/dd-trace/src/plugins/util/test')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')

Expand Down Expand Up @@ -80,8 +81,10 @@ versions.forEach((version) => {

const stepEvents = events.filter(event => event.type === 'span')

assert.equal(testSessionEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.include(testSessionEvent.content.resource, 'test_session.playwright test')
assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testModuleEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.include(testModuleEvent.content.resource, 'test_module.playwright test')
assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'browser')
Expand All @@ -103,6 +106,7 @@ versions.forEach((version) => {
])

testSuiteEvents.forEach(testSuiteEvent => {
assert.equal(testSuiteEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
if (testSuiteEvent.content.meta[TEST_STATUS] === 'fail') {
assert.exists(testSuiteEvent.content.meta[ERROR_MESSAGE])
}
Expand All @@ -124,6 +128,7 @@ versions.forEach((version) => {
])

testEvents.forEach(testEvent => {
assert.equal(testEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testEvent.content.metrics[TEST_SOURCE_START])
assert.equal(
testEvent.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/playwright-tests/'), true
Expand Down Expand Up @@ -155,7 +160,8 @@ versions.forEach((version) => {
env: {
...envVars,
PW_BASE_URL: `http://localhost:${webAppPort}`,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2',
DD_SESSION_NAME: 'my-test-session'
},
stdio: 'pipe'
}
Expand Down
19 changes: 17 additions & 2 deletions integration-tests/vitest/vitest.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ const {
TEST_TYPE,
TEST_IS_RETRY,
TEST_CODE_OWNERS,
TEST_CODE_COVERAGE_LINES_PCT
TEST_CODE_COVERAGE_LINES_PCT,
TEST_SESSION_NAME,
TEST_COMMAND
} = require('../../packages/dd-trace/src/plugins/util/test')

const versions = ['1.6.0', 'latest']
Expand Down Expand Up @@ -57,8 +59,10 @@ versions.forEach((version) => {
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')

assert.equal(testSessionEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.include(testSessionEvent.content.resource, 'test_session.vitest run')
assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testModuleEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.include(testModuleEvent.content.resource, 'test_module.vitest run')
assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test')
Expand Down Expand Up @@ -129,6 +133,16 @@ versions.forEach((version) => {
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip'
]
)

testEvents.forEach(test => {
assert.equal(test.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.equal(test.content.meta[TEST_COMMAND], 'vitest run')
})

testSuiteEvents.forEach(testSuite => {
assert.equal(testSuite.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.equal(testSuite.content.meta[TEST_COMMAND], 'vitest run')
})
// TODO: check error messages
}).then(() => done()).catch(done)

Expand All @@ -138,7 +152,8 @@ versions.forEach((version) => {
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags
DD_SESSION_NAME: 'my-test-session'
},
stdio: 'pipe'
}
Expand Down
1 change: 1 addition & 0 deletions packages/datadog-instrumentations/src/jest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -765,6 +765,7 @@ addHook({
_ddTestModuleId,
_ddTestSessionId,
_ddTestCommand,
_ddTestSessionName,
_ddForcedToRun,
_ddUnskippable,
_ddItrCorrelationId,
Expand Down
9 changes: 7 additions & 2 deletions packages/datadog-plugin-cucumber/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ const {
TEST_MODULE,
TEST_MODULE_ID,
TEST_SUITE,
CUCUMBER_IS_PARALLEL
CUCUMBER_IS_PARALLEL,
TEST_SESSION_NAME
} = require('../../dd-trace/src/plugins/util/test')
const { RESOURCE_NAME } = require('../../../ext/tags')
const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants')
Expand All @@ -51,7 +52,8 @@ function getTestSuiteTags (testSuiteSpan) {
[TEST_SUITE_ID]: testSuiteSpan.context().toSpanId(),
[TEST_SESSION_ID]: testSuiteSpan.context().toTraceId(),
[TEST_COMMAND]: testSuiteSpan.context()._tags[TEST_COMMAND],
[TEST_MODULE]: 'cucumber'
[TEST_MODULE]: 'cucumber',
[TEST_SESSION_NAME]: testSuiteSpan.context()._tags[TEST_SESSION_NAME]
}
if (testSuiteSpan.context()._parentId) {
suiteTags[TEST_MODULE_ID] = testSuiteSpan.context()._parentId.toString(10)
Expand Down Expand Up @@ -134,6 +136,9 @@ class CucumberPlugin extends CiPlugin {
if (itrCorrelationId) {
testSuiteMetadata[ITR_CORRELATION_ID] = itrCorrelationId
}
if (this.testSessionName) {
testSuiteMetadata[TEST_SESSION_NAME] = this.testSessionName
}
const testSuiteSpan = this.tracer.startSpan('cucumber.test_suite', {
childOf: this.testModuleSpan,
tags: {
Expand Down
Loading

0 comments on commit 9949796

Please sign in to comment.