From cec1f277b3ea4d09e9ff90addb0a386c033034d5 Mon Sep 17 00:00:00 2001 From: Justyna Olszak Date: Mon, 25 Oct 2021 09:27:42 +0200 Subject: [PATCH 1/7] Metadata for scenario added for json report --- core/util/engineTools.js | 3 ++- test/configs/backstop.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/util/engineTools.js b/core/util/engineTools.js index f0982ab7e..4aaf5d8ac 100644 --- a/core/util/engineTools.js +++ b/core/util/engineTools.js @@ -119,7 +119,8 @@ function generateTestPair (config, scenario, viewport, variantOrScenarioLabelSaf url: scenario.url, referenceUrl: scenario.referenceUrl, expect: getScenarioExpect(scenario), - viewportLabel: viewport.label + viewportLabel: viewport.label, + metadata: scenario.metadata }; } diff --git a/test/configs/backstop.json b/test/configs/backstop.json index 76710312e..420fe8261 100644 --- a/test/configs/backstop.json +++ b/test/configs/backstop.json @@ -31,7 +31,8 @@ "selectors": [], "selectorExpansion": true, "misMatchThreshold" : 0.1, - "requireSameDimensions": true + "requireSameDimensions": true, + "metadata": ["TEST-1"] } ], "paths": { From e914db83e65e47c2ac069da2f5c6dd4faf113fe5 Mon Sep 17 00:00:00 2001 From: Justyna Olszak Date: Fri, 26 Nov 2021 14:26:15 +0100 Subject: [PATCH 2/7] Force Docker to use current setup --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d857d728..9c2423509 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ ENV \ RUN apt-get update && \ apt-get install -y git sudo software-properties-common -RUN sudo npm install -g --unsafe-perm=true --allow-root backstopjs@${BACKSTOPJS_VERSION} +RUN sudo npm install -g --unsafe-perm=true --allow-root https://github.com/justyna-olszak-wttech/BackstopJS.git RUN wget https://dl-ssl.google.com/linux/linux_signing_key.pub && sudo apt-key add linux_signing_key.pub RUN sudo add-apt-repository "deb http://dl.google.com/linux/chrome/deb/ stable main" From 82f75673b2ab1656a1229a18dfca2db9fbfdd9c4 Mon Sep 17 00:00:00 2001 From: Justyna Olszak Date: Mon, 3 Jan 2022 14:56:08 +0100 Subject: [PATCH 3/7] Details for Xray format --- core/command/report.js | 15 ++++---- core/util/writeXrayReport.js | 69 ++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 core/util/writeXrayReport.js diff --git a/core/command/report.js b/core/command/report.js index 0bbd25093..086274736 100644 --- a/core/command/report.js +++ b/core/command/report.js @@ -7,6 +7,7 @@ const allSettled = require('../util/allSettled'); const fs = require('../util/fs'); const logger = require('../util/logger')('report'); const compare = require('../util/compare/'); +const writeXrayReport = require('../util/writeXrayReport'); function replaceInFile (file, search, replace) { return new Promise((resolve, reject) => { @@ -30,14 +31,16 @@ function writeReport (config, reporter) { if (config.report && config.report.indexOf('CI') > -1 && config.ciReport.format === 'junit') { promises.push(writeJunitReport(config, reporter)); + } else if (config.report && config.report.indexOf('Xray') > -1) { + logger.log('Initialize Xray report'); + promises.push(writeXrayReport(config, reporter)) + } else { + if (config.report && config.report.indexOf('json') > -1) { + promises.push(writeJsonReport(config, reporter)); + } + promises.push(writeBrowserReport(config, reporter)); } - if (config.report && config.report.indexOf('json') > -1) { - promises.push(writeJsonReport(config, reporter)); - } - - promises.push(writeBrowserReport(config, reporter)); - return allSettled(promises); } diff --git a/core/util/writeXrayReport.js b/core/util/writeXrayReport.js new file mode 100644 index 000000000..3224f4a57 --- /dev/null +++ b/core/util/writeXrayReport.js @@ -0,0 +1,69 @@ +const fs = require('../util/fs'); +const path = require('path'); +const logger = require('../util/logger')('Xray report'); +const _ = require('lodash'); +const cloneDeep = require('lodash/cloneDeep'); +const { stat } = require('fs'); + +module.exports = function (config, reporter) { + const jsonReporter = cloneDeep(reporter); + + function toAbsolute (p) { + return path.isAbsolute(p) ? p : path.join(config.projectPath, p); + } + + function transformTestCases (testCases) { + const transformedTestCases = []; + let testStatus = 'PASSED'; + + for (const testName in testCases) { + const testCase = testCases[testName]; + const xrayTestResult = { + 'iterations': [], + 'testInfo': {} + }; + + testCase.forEach((testedViewport) => { + let { pair: { viewportLabel: name }, status } = testedViewport; + + status = `${status}ed`.toUpperCase(); + if (status === 'FAILED') { + testStatus = status; + } + xrayTestResult.iterations.push({ name, status }); + }); + + xrayTestResult.status = testStatus; + xrayTestResult.testInfo.requirementKeys = testCase[0].pair.metadata; + xrayTestResult.testInfo.summary = testCase[0].pair.label; + xrayTestResult.testInfo.type = "Generic"; + transformedTestCases.push( + xrayTestResult + ); + } + + return transformedTestCases; + } + + function transformToXrayJson (json) { + const results = {} + const namedTestCases = _.groupBy(json, 'pair.label'); + return results.tests = transformTestCases(namedTestCases); + } + + logger.log('Writing Xray json report'); + + return fs.ensureDir(toAbsolute(config.json_report)).then(function () { + const res = transformToXrayJson(jsonReporter.tests); + + return fs.writeFile(toAbsolute(config.compareJsonFileName), JSON.stringify(res, null, 2)).then( + function () { + logger.log('Wrote Xray Json report to: ' + toAbsolute(config.compareJsonFileName)); + }, + function (err) { + logger.error('Failed writing Xray Json report to: ' + toAbsolute(config.compareJsonFileName)); + throw err; + } + ); + }); +}; From dbf9a7a3ac2312366cd8dd18d32138b6202e9c77 Mon Sep 17 00:00:00 2001 From: Justyna Olszak Date: Tue, 11 Jan 2022 10:08:28 +0100 Subject: [PATCH 4/7] Adjust json structure --- core/util/writeXrayReport.js | 11 ++++++++--- docker/Dockerfile | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/core/util/writeXrayReport.js b/core/util/writeXrayReport.js index 31ce6effd..5c103fda9 100644 --- a/core/util/writeXrayReport.js +++ b/core/util/writeXrayReport.js @@ -22,6 +22,8 @@ module.exports = function (config, reporter) { 'iterations': [], 'testInfo': {} }; + const metadata = testCase[0].pair.metadata; + const projectKey = metadata ? metadata[0].split('-')[0] : ''; testCase.forEach((testedViewport) => { let { pair: { viewportLabel: name }, status } = testedViewport; @@ -32,9 +34,9 @@ module.exports = function (config, reporter) { } xrayTestResult.iterations.push({ name, status }); }); - xrayTestResult.status = testStatus; - xrayTestResult.testInfo.requirementKeys = testCase[0].pair.metadata; + xrayTestResult.testInfo.requirementKeys = metadata; + xrayTestResult.testInfo.projectKey = projectKey; xrayTestResult.testInfo.summary = testCase[0].pair.label; xrayTestResult.testInfo.type = "Generic"; transformedTestCases.push( @@ -42,13 +44,16 @@ module.exports = function (config, reporter) { ); } + debugger; return transformedTestCases; } function transformToXrayJson (json) { const results = {} const namedTestCases = _.groupBy(json, 'pair.label'); - return results.tests = transformTestCases(namedTestCases); + + results.tests = transformTestCases(namedTestCases); + return results; } logger.log('Writing Xray json report'); diff --git a/docker/Dockerfile b/docker/Dockerfile index a612274fb..1d0e09b8a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ ENV \ RUN apt-get update && \ apt-get install -y git sudo software-properties-common -RUN sudo npm install -g --unsafe-perm=true --allow-root https://github.com/justyna-olszak-wttech/BackstopJS.git#5.4.5 +RUN sudo npm install -g --unsafe-perm=true --allow-root https://github.com/justyna-olszak-wttech/BackstopJS.git#v5.4.5 RUN wget https://dl-ssl.google.com/linux/linux_signing_key.pub && sudo apt-key add linux_signing_key.pub RUN sudo add-apt-repository "deb http://dl.google.com/linux/chrome/deb/ stable main" From 4788b30e5505868f2ed176432c697fd3892889e4 Mon Sep 17 00:00:00 2001 From: Justyna Olszak Date: Fri, 14 Jan 2022 15:15:43 +0100 Subject: [PATCH 5/7] Extensible approach to custom reports --- core/command/report.js | 28 +++++++++++++------ core/util/engineTools.js | 7 ++--- core/util/extendConfig.js | 1 + test/configs/backstop.json | 7 ++++- .../customReports/xrayReport.js | 23 +++++++-------- 5 files changed, 41 insertions(+), 25 deletions(-) rename core/util/writeXrayReport.js => test/configs/backstop_data/engine_scripts/customReports/xrayReport.js (73%) diff --git a/core/command/report.js b/core/command/report.js index 0b3b21d48..c999efe31 100644 --- a/core/command/report.js +++ b/core/command/report.js @@ -7,7 +7,6 @@ const allSettled = require('../util/allSettled'); const fs = require('../util/fs'); const logger = require('../util/logger')('report'); const compare = require('../util/compare/'); -const writeXrayReport = require('../util/writeXrayReport'); function replaceInFile (file, search, replace) { return new Promise((resolve, reject) => { @@ -26,20 +25,33 @@ function replaceInFile (file, search, replace) { }); } +async function processCustomReport (config, reporter) { + const engineScriptsPath = config.engine_scripts; + const customReport = config.customReport; + if (customReport) { + const customReportScript = path.resolve(engineScriptsPath, customReport.script); + if (fs.existsSync(customReportScript)) { + return await require(customReportScript)(config, reporter); + } else { + console.warn('WARNING: reporting script not found: ' + customReportScript); + } + } +} + function writeReport (config, reporter) { const promises = []; if (config.report && config.report.indexOf('CI') > -1 && config.ciReport.format === 'junit') { promises.push(writeJunitReport(config, reporter)); - } else if (config.report && config.report.indexOf('Xray') > -1) { - promises.push(writeXrayReport(config, reporter)) - } else { - if (config.report && config.report.indexOf('json') > -1) { - promises.push(writeJsonReport(config, reporter)); - } - promises.push(writeBrowserReport(config, reporter)); } + if (config.report && config.report.indexOf('json') > -1) { + promises.push(writeJsonReport(config, reporter)); + } + + promises.push(writeBrowserReport(config, reporter)); + promises.push(processCustomReport(config, reporter)); + return allSettled(promises); } diff --git a/core/util/engineTools.js b/core/util/engineTools.js index 4aaf5d8ac..1cb642ced 100644 --- a/core/util/engineTools.js +++ b/core/util/engineTools.js @@ -113,14 +113,11 @@ function generateTestPair (config, scenario, viewport, variantOrScenarioLabelSaf test: testFilePath, selector: selector, fileName: fileName, - label: scenario.label, requireSameDimensions: getRequireSameDimensions(scenario, config), misMatchThreshold: getMisMatchThreshHold(scenario, config), - url: scenario.url, - referenceUrl: scenario.referenceUrl, expect: getScenarioExpect(scenario), - viewportLabel: viewport.label, - metadata: scenario.metadata + veportLabel: viewport.label, + ...scenario }; } diff --git a/core/util/extendConfig.js b/core/util/extendConfig.js index 92790bf36..fd91ecd38 100644 --- a/core/util/extendConfig.js +++ b/core/util/extendConfig.js @@ -23,6 +23,7 @@ function extendConfig (config, userConfig) { config.asyncCompareLimit = userConfig.asyncCompareLimit; config.backstopVersion = version; config.dockerCommandTemplate = userConfig.dockerCommandTemplate; + config.customReport = userConfig.customReport; return config; } diff --git a/test/configs/backstop.json b/test/configs/backstop.json index 420fe8261..ef2a839b8 100644 --- a/test/configs/backstop.json +++ b/test/configs/backstop.json @@ -30,7 +30,7 @@ "postInteractionWait": 0, "selectors": [], "selectorExpansion": true, - "misMatchThreshold" : 0.1, + "misMatchThreshold": 0.1, "requireSameDimensions": true, "metadata": ["TEST-1"] } @@ -47,6 +47,11 @@ "engineOptions": { "args": ["--no-sandbox"] }, + "customReport": { + "script": "customReports/xrayReport.js", + "reportLocation": "backstop_data/xray_report", + "reportName": "xrayReport.json" + }, "asyncCaptureLimit": 5, "asyncCompareLimit": 50, "debug": false, diff --git a/core/util/writeXrayReport.js b/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js similarity index 73% rename from core/util/writeXrayReport.js rename to test/configs/backstop_data/engine_scripts/customReports/xrayReport.js index 5c103fda9..07fd82aa2 100644 --- a/core/util/writeXrayReport.js +++ b/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js @@ -1,12 +1,11 @@ -const fs = require('../util/fs'); +const { writeFile } = require('fs'); +const { ensureDir } = require('fs-extra'); const path = require('path'); -const logger = require('../util/logger')('Xray report'); const _ = require('lodash'); const cloneDeep = require('lodash/cloneDeep'); -const { stat } = require('fs'); +const util = require('util'); module.exports = function (config, reporter) { - const jsonReporter = cloneDeep(reporter); function toAbsolute (p) { return path.isAbsolute(p) ? p : path.join(config.projectPath, p); @@ -44,7 +43,6 @@ module.exports = function (config, reporter) { ); } - debugger; return transformedTestCases; } @@ -56,19 +54,22 @@ module.exports = function (config, reporter) { return results; } - logger.log('Writing Xray json report'); + const jsonReporter = cloneDeep(reporter); + const ensureDirPromise = util.promisify(ensureDir); + const writeFilePromise = util.promisify(writeFile); - return fs.ensureDir(toAbsolute(config.json_report)).then(function () { + return ensureDirPromise(toAbsolute(config.customReport.reportLocation)).then(function () { const res = transformToXrayJson(jsonReporter.tests); + const reportPath = toAbsolute(path.join(config.customReport.reportLocation, config.customReport.reportName)); - return fs.writeFile(toAbsolute(config.compareJsonFileName), JSON.stringify(res, null, 2)).then( + return writeFilePromise(reportPath, JSON.stringify(res, null, 2)).then( function () { - logger.log('Wrote Xray Json report to: ' + toAbsolute(config.compareJsonFileName)); + console.log('Wrote Xray report to: ' + reportPath); }, function (err) { - logger.error('Failed writing Xray Json report to: ' + toAbsolute(config.compareJsonFileName)); + console.error('Failed writing Xray report to: ' + reportPath); throw err; } ); }); -}; +}; \ No newline at end of file From 0649e321ed50827f230c1b5be593108f46dd35dc Mon Sep 17 00:00:00 2001 From: Justyna Olszak Date: Mon, 17 Jan 2022 11:47:21 +0100 Subject: [PATCH 6/7] Multiple custom reports --- core/command/report.js | 25 ++++++++++++------- core/util/extendConfig.js | 2 +- test/configs/backstop.json | 6 ----- .../customReports/xrayReport.js | 6 ++--- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/core/command/report.js b/core/command/report.js index c999efe31..b13a16880 100644 --- a/core/command/report.js +++ b/core/command/report.js @@ -25,17 +25,24 @@ function replaceInFile (file, search, replace) { }); } -async function processCustomReport (config, reporter) { +async function processCustomReports (config, reporter) { const engineScriptsPath = config.engine_scripts; - const customReport = config.customReport; - if (customReport) { - const customReportScript = path.resolve(engineScriptsPath, customReport.script); - if (fs.existsSync(customReportScript)) { - return await require(customReportScript)(config, reporter); - } else { - console.warn('WARNING: reporting script not found: ' + customReportScript); + const customReports = config.customReports.reports; + const results = []; + + if (customReports) { + for (let i = 0; i < customReports.length; i++) { + const customReportScript = path.resolve(engineScriptsPath, customReports[i].script); + + if (fs.existsSync(customReportScript)) { + const res = await require(customReportScript)(config, reporter, customReports[i].name); + results.push(res); + } else { + console.warn('WARNING: reporting script not found: ' + customReportScript); + } } } + return results; } function writeReport (config, reporter) { @@ -50,7 +57,7 @@ function writeReport (config, reporter) { } promises.push(writeBrowserReport(config, reporter)); - promises.push(processCustomReport(config, reporter)); + promises.push(processCustomReports(config, reporter)); return allSettled(promises); } diff --git a/core/util/extendConfig.js b/core/util/extendConfig.js index fd91ecd38..18ec429e9 100644 --- a/core/util/extendConfig.js +++ b/core/util/extendConfig.js @@ -23,7 +23,7 @@ function extendConfig (config, userConfig) { config.asyncCompareLimit = userConfig.asyncCompareLimit; config.backstopVersion = version; config.dockerCommandTemplate = userConfig.dockerCommandTemplate; - config.customReport = userConfig.customReport; + config.customReports = userConfig.customReports; return config; } diff --git a/test/configs/backstop.json b/test/configs/backstop.json index ef2a839b8..0b1b8bfd8 100644 --- a/test/configs/backstop.json +++ b/test/configs/backstop.json @@ -32,7 +32,6 @@ "selectorExpansion": true, "misMatchThreshold": 0.1, "requireSameDimensions": true, - "metadata": ["TEST-1"] } ], "paths": { @@ -47,11 +46,6 @@ "engineOptions": { "args": ["--no-sandbox"] }, - "customReport": { - "script": "customReports/xrayReport.js", - "reportLocation": "backstop_data/xray_report", - "reportName": "xrayReport.json" - }, "asyncCaptureLimit": 5, "asyncCompareLimit": 50, "debug": false, diff --git a/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js b/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js index 07fd82aa2..580c20f07 100644 --- a/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js +++ b/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js @@ -5,7 +5,7 @@ const _ = require('lodash'); const cloneDeep = require('lodash/cloneDeep'); const util = require('util'); -module.exports = function (config, reporter) { +module.exports = function (config, reporter, resultName) { function toAbsolute (p) { return path.isAbsolute(p) ? p : path.join(config.projectPath, p); @@ -58,9 +58,9 @@ module.exports = function (config, reporter) { const ensureDirPromise = util.promisify(ensureDir); const writeFilePromise = util.promisify(writeFile); - return ensureDirPromise(toAbsolute(config.customReport.reportLocation)).then(function () { + return ensureDirPromise(toAbsolute(config.customReports.reportLocation)).then(function () { const res = transformToXrayJson(jsonReporter.tests); - const reportPath = toAbsolute(path.join(config.customReport.reportLocation, config.customReport.reportName)); + const reportPath = toAbsolute(path.join(config.customReports.reportLocation, resultName)); return writeFilePromise(reportPath, JSON.stringify(res, null, 2)).then( function () { From 14aaf2380bec400356342fd636adf7bd3c5805e4 Mon Sep 17 00:00:00 2001 From: Justyna Olszak Date: Mon, 17 Jan 2022 13:10:34 +0100 Subject: [PATCH 7/7] Custom reports Readme --- README.md | 30 ++++++++++++++++++++++++++++++ core/util/engineTools.js | 2 +- docker/Dockerfile | 2 +- test/configs/backstop.json | 2 +- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 77a6df231..c5dac1e0f 100644 --- a/README.md +++ b/README.md @@ -529,6 +529,36 @@ If you choose the CI-only reporting or even no reporting (CLI is always on) you $ backstop openReport ``` +### Reporting in custom format + +To format tests output in a custom way you may provide a script that will transform data. Add below configuration to your test config. + +```json +"customReports": { + "reports": [ + { + "script": "customReports/customReport.js", + "name": "resultName.json" + } + ], + "reportLocation": "backstop_data/custom_report" + } +``` + +Your script `customReport.js` should take `config, reporter, resultName` as parameters and return a `Promise` when result is processed. +To see example see `/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js`. It can also incorporate additional information passed within scenarios like below: +```json +"scenarios": [ + { + "label": "BackstopJstop_data/engine_scripts/cookies.json", + "url": "https://garris.github.io/BackstopJS/", + "metadata": ["TEST-1"] + } +``` + + + +Additionally #### Test report integration with a build system like Jenkins/Travis The following config would enable the CI - report (*default: junit format*) diff --git a/core/util/engineTools.js b/core/util/engineTools.js index 3a1681802..4cce8d065 100644 --- a/core/util/engineTools.js +++ b/core/util/engineTools.js @@ -141,7 +141,7 @@ function generateTestPair (config, scenario, viewport, variantOrScenarioLabelSaf requireSameDimensions: getRequireSameDimensions(scenario, config), misMatchThreshold: getMisMatchThreshHold(scenario, config), expect: getScenarioExpect(scenario), - veportLabel: viewport.label, + viewportLabel: viewport.label, ...scenario }; } diff --git a/docker/Dockerfile b/docker/Dockerfile index 1d0e09b8a..60909d80a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ ENV \ RUN apt-get update && \ apt-get install -y git sudo software-properties-common -RUN sudo npm install -g --unsafe-perm=true --allow-root https://github.com/justyna-olszak-wttech/BackstopJS.git#v5.4.5 +RUN sudo npm install -g --unsafe-perm=true --allow-root https://github.com/justyna-olszak-wttech/BackstopJS.git#v6.0.4 RUN wget https://dl-ssl.google.com/linux/linux_signing_key.pub && sudo apt-key add linux_signing_key.pub RUN sudo add-apt-repository "deb http://dl.google.com/linux/chrome/deb/ stable main" diff --git a/test/configs/backstop.json b/test/configs/backstop.json index 47fd770a1..9ada14357 100644 --- a/test/configs/backstop.json +++ b/test/configs/backstop.json @@ -31,7 +31,7 @@ "selectors": [], "selectorExpansion": true, "misMatchThreshold": 0.1, - "requireSameDimensions": true, + "requireSameDimensions": true } ], "paths": {