Skip to content

Commit

Permalink
fix parallel mode reporting
Browse files Browse the repository at this point in the history
  • Loading branch information
@juan-fernandez
juan-fernandez committed Apr 26, 2024
1 parent d325a31 commit f5021cc
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 182 deletions.
290 changes: 116 additions & 174 deletions integration-tests/cucumber/cucumber.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,81 +91,6 @@ versions.forEach(version => {
await receiver.stop()
})

it('works with parallel mode', (done) => {
const receiverPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
const events = payloads.flatMap(({ payload }) => payload.events)

const { content: testSessionEventContent } = events.find(event => event.type === 'test_session_end')
const { content: testModuleEventContent } = events.find(event => event.type === 'test_module_end')
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')

assert.equal(
testModuleEventContent.test_session_id.toString(10),
testSessionEventContent.test_session_id.toString(10)
)

assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true')

testSuiteEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
})

testEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
}
}) => {
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true')
})

// add a tag to tests to check that parallel mode is being used
assert.propertyVal(testSessionEventContent.meta, CUCUMBER_IS_PARALLEL, 'true')
}, 5000)

let testOutput
childProcess = exec(
parallelModeCommand,
{
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
DD_TRACE_DEBUG: 1,
DD_TRACE_LOG_LEVEL: 'warn'
},
stdio: 'inherit'
}
)
childProcess.stdout.on('data', (chunk) => {
testOutput += chunk.toString()
})
childProcess.stderr.on('data', (chunk) => {
testOutput += chunk.toString()
})
childProcess.on('exit', () => {
assert.notInclude(testOutput, 'TypeError')
receiverPromise.then(() => done()).catch(done)
})
})

const reportMethods = ['agentless', 'evp proxy']

reportMethods.forEach((reportMethod) => {
Expand All @@ -175,112 +100,129 @@ versions.forEach(version => {
isAgentless = reportMethod === 'agentless'
envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port)
})
it('can run and report tests', (done) => {
receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
const events = payloads.flatMap(({ payload }) => payload.events)

const testSessionEvent = events.find(event => event.type === 'test_session_end')
const testModuleEvent = events.find(event => event.type === 'test_module_end')
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')

const stepEvents = events.filter(event => event.type === 'span')

const { content: testSessionEventContent } = testSessionEvent
const { content: testModuleEventContent } = testModuleEvent

assert.exists(testSessionEventContent.test_session_id)
assert.exists(testSessionEventContent.meta[TEST_COMMAND])
assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN])
assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true)
assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail')

assert.exists(testModuleEventContent.test_session_id)
assert.exists(testModuleEventContent.test_module_id)
assert.exists(testModuleEventContent.meta[TEST_COMMAND])
assert.exists(testModuleEventContent.meta[TEST_MODULE])
assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true)
assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail')
assert.equal(
testModuleEventContent.test_session_id.toString(10),
testSessionEventContent.test_session_id.toString(10)
)
const runModes = ['serial', 'parallel']
runModes.forEach((runMode) => {
it(`(${runMode}) can run and report tests`, (done) => {
const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand

assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [
`test_suite.${featuresPath}farewell.feature`,
`test_suite.${featuresPath}greetings.feature`
])
assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [
'pass',
'fail'
])
const receiverPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
const events = payloads.flatMap(({ payload }) => payload.events)

testSuiteEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
})
const testSessionEvent = events.find(event => event.type === 'test_session_end')
const testModuleEvent = events.find(event => event.type === 'test_module_end')
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')

assert.includeMembers(testEvents.map(test => test.content.resource), [
`${featuresPath}farewell.feature.Say farewell`,
`${featuresPath}greetings.feature.Say greetings`,
`${featuresPath}greetings.feature.Say yeah`,
`${featuresPath}greetings.feature.Say yo`,
`${featuresPath}greetings.feature.Say skip`
])
assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [
'pass',
'pass',
'pass',
'fail',
'skip'
])
const stepEvents = events.filter(event => event.type === 'span')

const { content: testSessionEventContent } = testSessionEvent
const { content: testModuleEventContent } = testModuleEvent

if (runMode === 'parallel') {
assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true')
}

testEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
assert.exists(testSessionEventContent.test_session_id)
assert.exists(testSessionEventContent.meta[TEST_COMMAND])
assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN])
assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true)
assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail')

assert.exists(testModuleEventContent.test_session_id)
assert.exists(testModuleEventContent.test_module_id)
assert.exists(testModuleEventContent.meta[TEST_COMMAND])
assert.exists(testModuleEventContent.meta[TEST_MODULE])
assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true)
assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail')
assert.equal(
testModuleEventContent.test_session_id.toString(10),
testSessionEventContent.test_session_id.toString(10)
)

assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [
`test_suite.${featuresPath}farewell.feature`,
`test_suite.${featuresPath}greetings.feature`
])
assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [
'pass',
'fail'
])

testSuiteEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
})

assert.includeMembers(testEvents.map(test => test.content.resource), [
`${featuresPath}farewell.feature.Say farewell`,
`${featuresPath}greetings.feature.Say greetings`,
`${featuresPath}greetings.feature.Say yeah`,
`${featuresPath}greetings.feature.Say yo`,
`${featuresPath}greetings.feature.Say skip`
])
assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [
'pass',
'pass',
'pass',
'fail',
'skip'
])

testEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true)
// Can read DD_TAGS
assert.propertyVal(meta, 'test.customtag', 'customvalue')
assert.propertyVal(meta, 'test.customtag2', 'customvalue2')
if (runMode === 'parallel') {
assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true')
}
})

stepEvents.forEach(stepEvent => {
assert.equal(stepEvent.content.name, 'cucumber.step')
assert.property(stepEvent.content.meta, 'cucumber.step')
})
}, 5000)

childProcess = exec(
runCommand,
{
cwd,
env: {
...envVars,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
},
stdio: 'pipe'
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true)
// Can read DD_TAGS
assert.propertyVal(meta, 'test.customtag', 'customvalue')
assert.propertyVal(meta, 'test.customtag2', 'customvalue2')
})
)

stepEvents.forEach(stepEvent => {
assert.equal(stepEvent.content.name, 'cucumber.step')
assert.property(stepEvent.content.meta, 'cucumber.step')
childProcess.on('exit', () => {
receiverPromise.then(() => done()).catch(done)
})
}, 5000).then(() => done()).catch(done)

childProcess = exec(
runTestsCommand,
{
cwd,
env: {
...envVars,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
},
stdio: 'pipe'
}
)
})
})
context('intelligent test runner', () => {
it('can report git metadata', (done) => {
Expand Down
19 changes: 11 additions & 8 deletions packages/datadog-instrumentations/src/cucumber.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,26 +485,29 @@ function getWrappedParseWorkerMessage (parseWorkerMessageFunction) {
}
}

const parseWorkerResponse = parseWorkerMessageFunction.apply(this, arguments)

// after calling `parseWorkerMessageFunction`, the test status can already be read
if (parsed.testCaseFinished) {
// we can grab worstTestStepResult in addition to pickle to get the actual status
const { pickle } =
const { pickle, worstTestStepResult } =
this.eventDataCollector.getTestCaseAttempt(parsed.testCaseFinished.testCaseStartedId)

const testFileAbsolutePath = pickle.uri
const { status } = getStatusFromResultLatest(worstTestStepResult)

// TODO: GET ACTUAL STATUS from worstTestStepResult
pickleResultByFile[testFileAbsolutePath].finished.push('pass')
const testFileAbsolutePath = pickle.uri
const { finished } = pickleResultByFile[testFileAbsolutePath]
finished.push(status)

if (pickleResultByFile[testFileAbsolutePath].finished.length === pickleByFile[testFileAbsolutePath].length) {
const testSuiteStatus = getSuiteStatusFromTestStatuses(pickleResultByFile[testFileAbsolutePath].finished)
if (finished.length === pickleByFile[testFileAbsolutePath].length) {
const testSuiteStatus = getSuiteStatusFromTestStatuses(finished)
testSuiteFinishCh.publish({
status: testSuiteStatus,
testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd())
})
}
}

return parseWorkerMessageFunction.apply(this, arguments)
return parseWorkerResponse
}
}

Expand Down

0 comments on commit f5021cc

Please sign in to comment.