Skip to content

Commit

Permalink
Revert "Revert "[test visibility] Use new metadata field in citestcyc…
Browse files Browse the repository at this point in the history 
…le (#46…" (#4673)

This reverts commit fab4824.
  • Loading branch information
@juan-fernandez
juan-fernandez committed Oct 1, 2024
1 parent 6c594d4 commit f02900e
Show file tree
Hide file tree
Showing 17 changed files with 179 additions and 91 deletions.
14 changes: 9 additions & 5 deletions integration-tests/cucumber/cucumber.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ const {
CUCUMBER_IS_PARALLEL,
TEST_SUITE,
TEST_CODE_OWNERS,
TEST_SESSION_NAME
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES
} = require('../../packages/dd-trace/src/plugins/util/test')

const isOldNode = semver.satisfies(process.version, '<=16')
Expand Down Expand Up @@ -115,6 +116,13 @@ versions.forEach(version => {

const receiverPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)
metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})

const events = payloads.flatMap(({ payload }) => payload.events)

const testSessionEvent = events.find(event => event.type === 'test_session_end')
Expand All @@ -131,14 +139,12 @@ versions.forEach(version => {
assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true')
}

assert.equal(testSessionEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testSessionEventContent.test_session_id)
assert.exists(testSessionEventContent.meta[TEST_COMMAND])
assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN])
assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true)
assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail')

assert.equal(testModuleEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testModuleEventContent.test_session_id)
assert.exists(testModuleEventContent.test_module_id)
assert.exists(testModuleEventContent.meta[TEST_COMMAND])
Expand Down Expand Up @@ -168,7 +174,6 @@ versions.forEach(version => {
test_session_id: testSessionId
}
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand Down Expand Up @@ -201,7 +206,6 @@ versions.forEach(version => {
test_session_id: testSessionId
}
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand Down
14 changes: 9 additions & 5 deletions integration-tests/cypress/cypress.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ const {
TEST_EARLY_FLAKE_ENABLED,
TEST_SUITE,
TEST_CODE_OWNERS,
TEST_SESSION_NAME
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES
} = require('../../packages/dd-trace/src/plugins/util/test')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
const { NODE_MAJOR } = require('../../version')
Expand Down Expand Up @@ -227,6 +228,13 @@ moduleTypes.forEach(({
it('can run and report tests', (done) => {
const receiverPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)

metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})
const events = payloads.flatMap(({ payload }) => payload.events)

const testSessionEvent = events.find(event => event.type === 'test_session_end')
Expand All @@ -237,14 +245,12 @@ moduleTypes.forEach(({
const { content: testSessionEventContent } = testSessionEvent
const { content: testModuleEventContent } = testModuleEvent

assert.equal(testSessionEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testSessionEventContent.test_session_id)
assert.exists(testSessionEventContent.meta[TEST_COMMAND])
assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN])
assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true)
assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail')

assert.equal(testModuleEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testModuleEventContent.test_session_id)
assert.exists(testModuleEventContent.test_module_id)
assert.exists(testModuleEventContent.meta[TEST_COMMAND])
Expand Down Expand Up @@ -276,7 +282,6 @@ moduleTypes.forEach(({
test_session_id: testSessionId
}
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand Down Expand Up @@ -306,7 +311,6 @@ moduleTypes.forEach(({
test_session_id: testSessionId
}
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand Down
35 changes: 19 additions & 16 deletions integration-tests/jest/jest.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ const {
TEST_EARLY_FLAKE_ABORT_REASON,
TEST_SOURCE_START,
TEST_CODE_OWNERS,
TEST_SESSION_NAME
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES
} = require('../../packages/dd-trace/src/plugins/util/test')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')

Expand Down Expand Up @@ -134,6 +135,14 @@ describe('jest CommonJS', () => {
receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
}
receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)

metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})

const events = payloads.flatMap(({ payload }) => payload.events)
const sessionEventContent = events.find(event => event.type === 'test_session_end').content
const moduleEventContent = events.find(event => event.type === 'test_module_end').content
Expand All @@ -150,14 +159,11 @@ describe('jest CommonJS', () => {
)
assert.equal(suites.length, 2)
assert.exists(sessionEventContent)
assert.equal(sessionEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(moduleEventContent)
assert.equal(moduleEventContent.meta[TEST_SESSION_NAME], 'my-test-session')

assert.include(testOutput, expectedStdout)

tests.forEach(testEvent => {
assert.equal(testEvent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true)
assert.exists(testEvent.metrics[TEST_SOURCE_START])
// Can read DD_TAGS
Expand All @@ -166,7 +172,6 @@ describe('jest CommonJS', () => {
})

suites.forEach(testSuite => {
assert.equal(testSuite.meta[TEST_SESSION_NAME], 'my-test-session')
assert.isTrue(testSuite.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'))
assert.equal(testSuite.metrics[TEST_SOURCE_START], 1)
})
Expand Down Expand Up @@ -445,21 +450,19 @@ describe('jest CommonJS', () => {
})

receiver.gatherPayloads(({ url }) => url === '/api/v2/citestcycle', 5000).then(eventsRequests => {
const metadataDicts = eventsRequests.flatMap(({ payload }) => payload.metadata)

// it propagates test session name to the test and test suite events in parallel mode
metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})

const events = eventsRequests.map(({ payload }) => payload)
.flatMap(({ events }) => events)
const eventTypes = events.map(event => event.type)

assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end'])
const tests = events.filter(event => event.type === 'test').map(event => event.content)
const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content)

// it propagates test session name to the test and test suite events in parallel mode
tests.forEach(testEvent => {
assert.equal(testEvent.meta[TEST_SESSION_NAME], 'my-test-session')
})
testSuites.forEach(testSuite => {
assert.equal(testSuite.meta[TEST_SESSION_NAME], 'my-test-session')
})

done()
}).catch(done)
Expand Down
25 changes: 18 additions & 7 deletions integration-tests/mocha/mocha.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ const {
MOCHA_IS_PARALLEL,
TEST_SOURCE_START,
TEST_CODE_OWNERS,
TEST_SESSION_NAME
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES
} = require('../../packages/dd-trace/src/plugins/util/test')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')

Expand Down Expand Up @@ -133,6 +134,14 @@ describe('mocha CommonJS', function () {
receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
}
receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)

metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})

const events = payloads.flatMap(({ payload }) => payload.events)
const sessionEventContent = events.find(event => event.type === 'test_session_end').content
const moduleEventContent = events.find(event => event.type === 'test_module_end').content
Expand All @@ -149,15 +158,12 @@ describe('mocha CommonJS', function () {
)
assert.equal(suites.length, 2)
assert.exists(sessionEventContent)
assert.equal(sessionEventContent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(moduleEventContent)
assert.equal(moduleEventContent.meta[TEST_SESSION_NAME], 'my-test-session')

assert.include(testOutput, expectedStdout)
assert.include(testOutput, extraStdout)

tests.forEach(testEvent => {
assert.equal(testEvent.meta[TEST_SESSION_NAME], 'my-test-session')
assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true)
assert.exists(testEvent.metrics[TEST_SOURCE_START])
// Can read DD_TAGS
Expand All @@ -166,7 +172,6 @@ describe('mocha CommonJS', function () {
})

suites.forEach(testSuite => {
assert.equal(testSuite.meta[TEST_SESSION_NAME], 'my-test-session')
assert.isTrue(testSuite.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'))
assert.equal(testSuite.metrics[TEST_SOURCE_START], 1)
})
Expand Down Expand Up @@ -317,6 +322,14 @@ describe('mocha CommonJS', function () {
it('works with parallel mode', (done) => {
const eventsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)

metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})

const events = payloads.flatMap(({ payload }) => payload.events)
const sessionEventContent = events.find(event => event.type === 'test_session_end').content
const moduleEventContent = events.find(event => event.type === 'test_module_end').content
Expand All @@ -334,7 +347,6 @@ describe('mocha CommonJS', function () {
test_module_id: testModuleId,
test_session_id: testSessionId
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand All @@ -349,7 +361,6 @@ describe('mocha CommonJS', function () {
test_module_id: testModuleId,
test_session_id: testSessionId
}) => {
assert.equal(meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
Expand Down
15 changes: 10 additions & 5 deletions integration-tests/playwright/playwright.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ const {
TEST_EARLY_FLAKE_ENABLED,
TEST_SUITE,
TEST_CODE_OWNERS,
TEST_SESSION_NAME
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES
} = require('../../packages/dd-trace/src/plugins/util/test')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')

Expand Down Expand Up @@ -72,6 +73,14 @@ versions.forEach((version) => {
const reportUrl = reportMethod === 'agentless' ? '/api/v2/citestcycle' : '/evp_proxy/v2/api/v2/citestcycle'

receiver.gatherPayloadsMaxTimeout(({ url }) => url === reportUrl, payloads => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)

metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})

const events = payloads.flatMap(({ payload }) => payload.events)

const testSessionEvent = events.find(event => event.type === 'test_session_end')
Expand All @@ -81,10 +90,8 @@ versions.forEach((version) => {

const stepEvents = events.filter(event => event.type === 'span')

assert.equal(testSessionEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.include(testSessionEvent.content.resource, 'test_session.playwright test')
assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testModuleEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.include(testModuleEvent.content.resource, 'test_module.playwright test')
assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'browser')
Expand All @@ -106,7 +113,6 @@ versions.forEach((version) => {
])

testSuiteEvents.forEach(testSuiteEvent => {
assert.equal(testSuiteEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
if (testSuiteEvent.content.meta[TEST_STATUS] === 'fail') {
assert.exists(testSuiteEvent.content.meta[ERROR_MESSAGE])
}
Expand All @@ -130,7 +136,6 @@ versions.forEach((version) => {
])

testEvents.forEach(testEvent => {
assert.equal(testEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.exists(testEvent.content.metrics[TEST_SOURCE_START])
assert.equal(
testEvent.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/playwright-tests/'), true
Expand Down
13 changes: 9 additions & 4 deletions integration-tests/vitest/vitest.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const {
TEST_CODE_COVERAGE_LINES_PCT,
TEST_SESSION_NAME,
TEST_COMMAND,
TEST_LEVEL_EVENT_TYPES,
TEST_SOURCE_FILE,
TEST_SOURCE_START
} = require('../../packages/dd-trace/src/plugins/util/test')
Expand Down Expand Up @@ -54,17 +55,23 @@ versions.forEach((version) => {

it('can run and report tests', (done) => {
receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)

metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})

const events = payloads.flatMap(({ payload }) => payload.events)

const testSessionEvent = events.find(event => event.type === 'test_session_end')
const testModuleEvent = events.find(event => event.type === 'test_module_end')
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')

assert.equal(testSessionEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.include(testSessionEvent.content.resource, 'test_session.vitest run')
assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testModuleEvent.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.include(testModuleEvent.content.resource, 'test_module.vitest run')
assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test')
Expand Down Expand Up @@ -137,12 +144,10 @@ versions.forEach((version) => {
)

testEvents.forEach(test => {
assert.equal(test.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.equal(test.content.meta[TEST_COMMAND], 'vitest run')
})

testSuiteEvents.forEach(testSuite => {
assert.equal(testSuite.content.meta[TEST_SESSION_NAME], 'my-test-session')
assert.equal(testSuite.content.meta[TEST_COMMAND], 'vitest run')
assert.isTrue(
testSuite.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/vitest-tests/test-visibility')
Expand Down
9 changes: 2 additions & 7 deletions packages/datadog-plugin-cucumber/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ const {
TEST_MODULE,
TEST_MODULE_ID,
TEST_SUITE,
CUCUMBER_IS_PARALLEL,
TEST_SESSION_NAME
CUCUMBER_IS_PARALLEL
} = require('../../dd-trace/src/plugins/util/test')
const { RESOURCE_NAME } = require('../../../ext/tags')
const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants')
Expand All @@ -52,8 +51,7 @@ function getTestSuiteTags (testSuiteSpan) {
[TEST_SUITE_ID]: testSuiteSpan.context().toSpanId(),
[TEST_SESSION_ID]: testSuiteSpan.context().toTraceId(),
[TEST_COMMAND]: testSuiteSpan.context()._tags[TEST_COMMAND],
[TEST_MODULE]: 'cucumber',
[TEST_SESSION_NAME]: testSuiteSpan.context()._tags[TEST_SESSION_NAME]
[TEST_MODULE]: 'cucumber'
}
if (testSuiteSpan.context()._parentId) {
suiteTags[TEST_MODULE_ID] = testSuiteSpan.context()._parentId.toString(10)
Expand Down Expand Up @@ -144,9 +142,6 @@ class CucumberPlugin extends CiPlugin {
if (itrCorrelationId) {
testSuiteMetadata[ITR_CORRELATION_ID] = itrCorrelationId
}
if (this.testSessionName) {
testSuiteMetadata[TEST_SESSION_NAME] = this.testSessionName
}
if (testSourceFile) {
testSuiteMetadata[TEST_SOURCE_FILE] = testSourceFile
testSuiteMetadata[TEST_SOURCE_START] = 1
Expand Down
Loading

0 comments on commit f02900e

Please sign in to comment.