diff --git a/README.md b/README.md index 77a6df231..c5dac1e0f 100644 --- a/README.md +++ b/README.md @@ -529,6 +529,36 @@ If you choose the CI-only reporting or even no reporting (CLI is always on) you $ backstop openReport ``` +### Reporting in custom format + +To format tests output in a custom way you may provide a script that will transform data. Add below configuration to your test config. + +```json +"customReports": { + "reports": [ + { + "script": "customReports/customReport.js", + "name": "resultName.json" + } + ], + "reportLocation": "backstop_data/custom_report" + } +``` + +Your script `customReport.js` should take `config, reporter, resultName` as parameters and return a `Promise` when result is processed. +To see example see `/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js`. It can also incorporate additional information passed within scenarios like below: +```json +"scenarios": [ + { + "label": "BackstopJstop_data/engine_scripts/cookies.json", + "url": "https://garris.github.io/BackstopJS/", + "metadata": ["TEST-1"] + } +``` + + + +Additionally #### Test report integration with a build system like Jenkins/Travis The following config would enable the CI - report (*default: junit format*) diff --git a/core/command/report.js b/core/command/report.js index f3b3b8587..8a4f57f3b 100644 --- a/core/command/report.js +++ b/core/command/report.js @@ -25,6 +25,26 @@ function replaceInFile (file, search, replace) { }); } +async function processCustomReports (config, reporter) { + const engineScriptsPath = config.engine_scripts; + const customReports = config.customReports.reports; + const results = []; + + if (customReports) { + for (let i = 0; i < customReports.length; i++) { + const customReportScript = path.resolve(engineScriptsPath, customReports[i].script); + + if (fs.existsSync(customReportScript)) { + const res = await require(customReportScript)(config, reporter, customReports[i].name); + results.push(res); + } else { + console.warn('WARNING: reporting script not found: ' + customReportScript); + } + } + } + return results; +} + function writeReport (config, reporter) { const promises = []; @@ -37,6 +57,7 @@ function writeReport (config, reporter) { } promises.push(writeBrowserReport(config, reporter)); + promises.push(processCustomReports(config, reporter)); return allSettled(promises); } diff --git a/core/util/engineTools.js b/core/util/engineTools.js index 8f8bda03b..4cce8d065 100644 --- a/core/util/engineTools.js +++ b/core/util/engineTools.js @@ -138,13 +138,11 @@ function generateTestPair (config, scenario, viewport, variantOrScenarioLabelSaf testLog: testLogFilePath, selector: selector, fileName: fileName, - label: scenario.label, requireSameDimensions: getRequireSameDimensions(scenario, config), misMatchThreshold: getMisMatchThreshHold(scenario, config), - url: scenario.url, - referenceUrl: scenario.referenceUrl, expect: getScenarioExpect(scenario), - viewportLabel: viewport.label + viewportLabel: viewport.label, + ...scenario }; } diff --git a/core/util/extendConfig.js b/core/util/extendConfig.js index 4c5ac9d33..825412eb4 100644 --- a/core/util/extendConfig.js +++ b/core/util/extendConfig.js @@ -24,6 +24,7 @@ function extendConfig (config, userConfig) { config.backstopVersion = version; config.dockerCommandTemplate = userConfig.dockerCommandTemplate; config.scenarioLogsInReports = userConfig.scenarioLogsInReports; + config.customReports = userConfig.customReports; return config; } diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d857d728..60909d80a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ ENV \ RUN apt-get update && \ apt-get install -y git sudo software-properties-common -RUN sudo npm install -g --unsafe-perm=true --allow-root backstopjs@${BACKSTOPJS_VERSION} +RUN sudo npm install -g --unsafe-perm=true --allow-root https://github.com/justyna-olszak-wttech/BackstopJS.git#v6.0.4 RUN wget https://dl-ssl.google.com/linux/linux_signing_key.pub && sudo apt-key add linux_signing_key.pub RUN sudo add-apt-repository "deb http://dl.google.com/linux/chrome/deb/ stable main" diff --git a/test/configs/backstop.json b/test/configs/backstop.json index ec9569df4..9ada14357 100644 --- a/test/configs/backstop.json +++ b/test/configs/backstop.json @@ -30,7 +30,7 @@ "postInteractionWait": 0, "selectors": [], "selectorExpansion": true, - "misMatchThreshold" : 0.1, + "misMatchThreshold": 0.1, "requireSameDimensions": true } ], diff --git a/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js b/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js new file mode 100644 index 000000000..580c20f07 --- /dev/null +++ b/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js @@ -0,0 +1,75 @@ +const { writeFile } = require('fs'); +const { ensureDir } = require('fs-extra'); +const path = require('path'); +const _ = require('lodash'); +const cloneDeep = require('lodash/cloneDeep'); +const util = require('util'); + +module.exports = function (config, reporter, resultName) { + + function toAbsolute (p) { + return path.isAbsolute(p) ? p : path.join(config.projectPath, p); + } + + function transformTestCases (testCases) { + const transformedTestCases = []; + + for (const testName in testCases) { + let testStatus = 'PASSED'; + const testCase = testCases[testName]; + const xrayTestResult = { + 'iterations': [], + 'testInfo': {} + }; + const metadata = testCase[0].pair.metadata; + const projectKey = metadata ? metadata[0].split('-')[0] : ''; + + testCase.forEach((testedViewport) => { + let { pair: { viewportLabel: name }, status } = testedViewport; + + status = `${status}ed`.toUpperCase(); + if (status === 'FAILED') { + testStatus = status; + } + xrayTestResult.iterations.push({ name, status }); + }); + xrayTestResult.status = testStatus; + xrayTestResult.testInfo.requirementKeys = metadata; + xrayTestResult.testInfo.projectKey = projectKey; + xrayTestResult.testInfo.summary = testCase[0].pair.label; + xrayTestResult.testInfo.type = "Generic"; + transformedTestCases.push( + xrayTestResult + ); + } + + return transformedTestCases; + } + + function transformToXrayJson (json) { + const results = {} + const namedTestCases = _.groupBy(json, 'pair.label'); + + results.tests = transformTestCases(namedTestCases); + return results; + } + + const jsonReporter = cloneDeep(reporter); + const ensureDirPromise = util.promisify(ensureDir); + const writeFilePromise = util.promisify(writeFile); + + return ensureDirPromise(toAbsolute(config.customReports.reportLocation)).then(function () { + const res = transformToXrayJson(jsonReporter.tests); + const reportPath = toAbsolute(path.join(config.customReports.reportLocation, resultName)); + + return writeFilePromise(reportPath, JSON.stringify(res, null, 2)).then( + function () { + console.log('Wrote Xray report to: ' + reportPath); + }, + function (err) { + console.error('Failed writing Xray report to: ' + reportPath); + throw err; + } + ); + }); +}; \ No newline at end of file