Metadata for scenario added for json report

README.md

@@ -529,6 +529,36 @@ If you choose the CI-only reporting or even no reporting (CLI is always on) you
 $ backstop openReport
 ```
 
+### Reporting in custom format
+
+To format tests output in a custom way you may provide a script that will transform data. Add below configuration to your test config.
+
+```json
+"customReports": {
+    "reports": [
+      {
+        "script": "customReports/customReport.js",
+        "name": "resultName.json"
+      }
+    ],
+    "reportLocation": "backstop_data/custom_report"
+  }
+```
+
+Your script `customReport.js` should take `config, reporter, resultName` as parameters and return a `Promise` when result is processed. 
+To see example see `/test/configs/backstop_data/engine_scripts/customReports/xrayReport.js`. It can also incorporate additional information passed within scenarios like below: 
+```json
+"scenarios": [
+    {
+      "label": "BackstopJstop_data/engine_scripts/cookies.json",
+      "url": "https://garris.github.io/BackstopJS/",
+      "metadata": ["TEST-1"]
+    }
+```
+
+
+
+Additionally 
 #### Test report integration with a build system like Jenkins/Travis
 
 The following config would enable the CI - report (*default: junit format*)

core/command/report.js

@@ -25,6 +25,26 @@ function replaceInFile (file, search, replace) {
   });
 }
 
+async function processCustomReports (config, reporter) {
+  const engineScriptsPath = config.engine_scripts;
+  const customReports = config.customReports.reports;
+  const results = [];
+
+  if (customReports) {
+    for (let i = 0; i < customReports.length; i++) {
+      const customReportScript = path.resolve(engineScriptsPath, customReports[i].script);
+
+      if (fs.existsSync(customReportScript)) {
+        const res = await require(customReportScript)(config, reporter, customReports[i].name);
+        results.push(res);
+      } else {
+        console.warn('WARNING: reporting script not found: ' + customReportScript);
+      }
+    }
+  }
+  return results;
+}
+
 function writeReport (config, reporter) {
   const promises = [];
 
@@ -37,6 +57,7 @@ function writeReport (config, reporter) {
   }
 
   promises.push(writeBrowserReport(config, reporter));
+  promises.push(processCustomReports(config, reporter));
 
   return allSettled(promises);
 }

core/util/engineTools.js

@@ -138,13 +138,11 @@ function generateTestPair (config, scenario, viewport, variantOrScenarioLabelSaf
     testLog: testLogFilePath,
     selector: selector,
     fileName: fileName,
-    label: scenario.label,
     requireSameDimensions: getRequireSameDimensions(scenario, config),
     misMatchThreshold: getMisMatchThreshHold(scenario, config),
-    url: scenario.url,
-    referenceUrl: scenario.referenceUrl,
     expect: getScenarioExpect(scenario),
-    viewportLabel: viewport.label
+    viewportLabel: viewport.label,
+    ...scenario
   };
 }
 

core/util/extendConfig.js

@@ -24,6 +24,7 @@ function extendConfig (config, userConfig) {
   config.backstopVersion = version;
   config.dockerCommandTemplate = userConfig.dockerCommandTemplate;
   config.scenarioLogsInReports = userConfig.scenarioLogsInReports;
+  config.customReports = userConfig.customReports;
   return config;
 }
 

docker/Dockerfile

@@ -9,7 +9,7 @@ ENV \
 RUN apt-get update && \
 	apt-get install -y git sudo software-properties-common
 
-RUN sudo npm install -g --unsafe-perm=true --allow-root backstopjs@${BACKSTOPJS_VERSION}
+RUN sudo npm install -g --unsafe-perm=true --allow-root https://github.com/justyna-olszak-wttech/BackstopJS.git#v6.0.4
 
 RUN wget https://dl-ssl.google.com/linux/linux_signing_key.pub && sudo apt-key add linux_signing_key.pub
 RUN sudo add-apt-repository "deb http://dl.google.com/linux/chrome/deb/ stable main"

test/configs/backstop.json

@@ -30,7 +30,7 @@
       "postInteractionWait": 0,
       "selectors": [],
       "selectorExpansion": true,
-      "misMatchThreshold" : 0.1,
+      "misMatchThreshold": 0.1,
       "requireSameDimensions": true
     }
   ],

test/configs/backstop_data/engine_scripts/customReports/xrayReport.js

@@ -0,0 +1,75 @@
+const { writeFile } = require('fs');
+const { ensureDir } = require('fs-extra');
+const path = require('path');
+const _ = require('lodash');
+const cloneDeep = require('lodash/cloneDeep');
+const util = require('util');
+
+module.exports = function (config, reporter, resultName) {
+
+  function toAbsolute (p) {
+    return path.isAbsolute(p) ? p : path.join(config.projectPath, p);
+  }
+
+  function transformTestCases (testCases) {
+    const transformedTestCases = [];
+
+    for (const testName in testCases) {
+      let testStatus = 'PASSED';
+      const testCase = testCases[testName];
+      const xrayTestResult = {
+        'iterations': [],
+        'testInfo': {}
+      };
+      const metadata = testCase[0].pair.metadata;
+      const projectKey = metadata ? metadata[0].split('-')[0] : '';
+
+      testCase.forEach((testedViewport) => {
+        let { pair: { viewportLabel: name }, status } = testedViewport;
+
+        status = `${status}ed`.toUpperCase();
+        if (status === 'FAILED') {
+          testStatus = status;
+        }
+        xrayTestResult.iterations.push({ name, status });
+      });
+      xrayTestResult.status = testStatus;
+      xrayTestResult.testInfo.requirementKeys = metadata;
+      xrayTestResult.testInfo.projectKey = projectKey;
+      xrayTestResult.testInfo.summary = testCase[0].pair.label;
+      xrayTestResult.testInfo.type = "Generic";
+      transformedTestCases.push(
+        xrayTestResult
+      );
+    }
+
+    return transformedTestCases;
+  }
+
+  function transformToXrayJson (json) {
+    const results = {}
+    const namedTestCases = _.groupBy(json, 'pair.label');
+
+    results.tests = transformTestCases(namedTestCases);
+    return results;
+  }
+
+  const jsonReporter = cloneDeep(reporter);
+  const ensureDirPromise = util.promisify(ensureDir);
+  const writeFilePromise = util.promisify(writeFile);
+
+  return ensureDirPromise(toAbsolute(config.customReports.reportLocation)).then(function () {
+    const res = transformToXrayJson(jsonReporter.tests);
+    const reportPath = toAbsolute(path.join(config.customReports.reportLocation, resultName));
+
+    return writeFilePromise(reportPath, JSON.stringify(res, null, 2)).then(
+      function () {
+        console.log('Wrote Xray report to: ' + reportPath);
+      },
+      function (err) {
+        console.error('Failed writing Xray report to: ' + reportPath);
+        throw err;
+      }
+    );
+  });
+};