fix: exit code is 1 when expectations are not fulfilled

bin/cli.js

@@ -6,6 +6,7 @@
 /* eslint-disable no-logger */
 const sysPath = require('path');
 const fs = require('fs');
+const os = require('os');
 const yargs = require('yargs');
 
 const PWMetrics = require('../lib/index');
@@ -64,6 +65,11 @@ const cliFlags = yargs
     'type': 'boolean',
     'default': false
   })
+  .option('fail-on-error', {
+    'describe': 'Exit PWMetrics with an error status code after the first unfilled expectation',
+    'type': 'boolean',
+    'default': false
+  })
   .check((argv) => {
     // Make sure pwmetrics has been passed a url, either from cli or config fileg()
 
@@ -101,20 +107,20 @@ const writeToDisk = function(fileName, data) {
 };
 
 const pwMetrics = new PWMetrics(options.url, options);
-pwMetrics.start()
-  .then(data => {
-    if (options.flags.json) {
-      // serialize accordingly
-      data = JSON.stringify(data, null, 2) + '\n';
-      // output to file.
-      if (options.flags.outputPath != 'stdout') {
-        return writeToDisk(options.flags.outputPath, data);
-        // output to stdout
-      } else if (data) {
-        process.stdout.write(data);
-      }
+pwMetrics.start(data => {
+  if (options.flags.json) {
+    // serialize accordingly
+    const formattedData = JSON.stringify(data, null, 2) + os.EOL;
+    // output to file.
+    if (options.flags.outputPath !== 'stdout') {
+      writeToDisk(options.flags.outputPath, formattedData);
+    // output to stdout
+    } else if (formattedData) {
+      process.stdout.write(formattedData);
     }
-  }).then(() => {
+  }
+})
+  .then(() => {
     process.exit(0);
   }).catch(err => {
     logger.error(err);

lib/index.ts

@@ -1,6 +1,12 @@
 // Copyright 2016 Google Inc. All Rights Reserved.
 // Licensed under the Apache License, Version 2.0. See LICENSE
 
+declare var process: {
+  stdout: {
+    columns: string;
+  }
+};
+
 const opn = require('opn');
 const path = require('path');
 
@@ -37,7 +43,8 @@ class PWMetrics {
     expectations: false,
     json: false,
     chromeFlags: '',
-    showOutput: true
+    showOutput: true,
+    failOnError: false,
   };
   runs: number;
   sheets: SheetsConfig;
@@ -67,7 +74,7 @@ class PWMetrics {
     this.logger = Logger.getInstance({showOutput: this.flags.showOutput});
   }
 
-  async start() {
+  async start(outputDataCallback) {
     const runs = Array.apply(null, {length: +this.runs}).map(Number.call, Number);
     let metricsResults: MetricsResults[] = [];
 
@@ -93,28 +100,38 @@ class PWMetrics {
       await sheets.appendResults(results.runs);
     }
 
+    if (outputDataCallback) {
+      outputDataCallback(results);
+    }
+
     if (this.flags.expectations) {
       const resultsToCompare = this.runs > 1 ? results.median.timings : results[0].timings;
-      if (this.resultHasExpectationErrors(resultsToCompare)) {
+      const hasExpectationsWarnings = this.resultHasExpectationIssues(resultsToCompare, 'warn');
+      const hasExpectationsErrors = this.resultHasExpectationIssues(resultsToCompare, 'error');
+
+      if (hasExpectationsWarnings || hasExpectationsErrors) {
         checkExpectations(resultsToCompare, this.normalizedExpectations);
-        this.logger.error(getMessage('HAS_EXPECTATION_ERRORS'));
+
+        if (hasExpectationsErrors && this.flags.failOnError) {
+          throw new Error(getMessage('HAS_EXPECTATION_ERRORS'));
+        }
+        else {
+          this.logger.warn(getMessage('HAS_EXPECTATION_ERRORS'));
+        }
       }
     }
 
     return results;
   }
 
-  resultHasExpectationErrors(timings: Timing[]): boolean {
+  resultHasExpectationIssues(timings: Timing[], issueType: 'warn' | 'error'): boolean {
     return timings.some((timing: Timing) => {
       const expectation = this.normalizedExpectations[timing.id];
       if (!expectation) {
         return false;
       }
-      const expectedErrorLimit = expectation.error;
-      const expectedWarningLimit = expectation.warn;
-      const hasErrors = expectedErrorLimit !== undefined && timing.timing >= expectedErrorLimit;
-      const hasWarnings = expectedWarningLimit !== undefined && timing.timing >= expectedWarningLimit;
-      return hasErrors || hasWarnings;
+      const expectedLimit = expectation[issueType];
+      return expectedLimit !== undefined && timing.timing >= expectedLimit;
     });
   }
 
@@ -160,7 +177,7 @@ class PWMetrics {
 
     const fullWidthInMs = Math.max(...timings.map(result => result.timing));
     const maxLabelWidth = Math.max(...timings.map(result => result.title.length));
-    const terminalWidth = process.stdout.columns || 90;
+    const terminalWidth = +process.stdout.columns || 90;
 
     drawChart(timings, {
       // 90% of terminal width to give some right margin

lib/utils/logger.ts

@@ -20,6 +20,12 @@ export class Logger {
     }
   }
 
+  warn(msg: any, ...args: any[]) {
+    if(Logger.options.showOutput){
+      console.warn(msg, ...args);
+    }
+  }
+
   error(msg: any, ...args: any[]) {
     if(Logger.options.showOutput){
       console.error(msg, ...args);

readme.md

@@ -68,6 +68,9 @@ pwmetrics --view
 # --expectations  Assert metrics results against provides values. See _Defining expectations_ below.
 pwmetrics --expectations
 
+# --fail-on-error  Exit PWMetrics with an error status code after the first unfilled expectation.
+pwmetrics --fail-on-error
+
 
 ```
 
@@ -123,6 +126,7 @@ module.exports = {
     chromeFlags: '', // custom flags to pass to Chrome. For a full list of flags, see http://peter.sh/experiments/chromium-command-line-switches/.
     // Note: pwmetrics supports all flags from Lighthouse
     showOutput: true // not required, set to false for pwmetrics not output any console.log messages
+    failOnError: false // not required, set to true if you want to fail the process on expectations errors
   },
   expectations: {
     // these expectations values are examples, for your cases set your own

types/types.ts

@@ -28,6 +28,7 @@ export interface FeatureFlags {
   chromePath?: string;
   port?: number;
   showOutput: Boolean;
+  failOnError: Boolean;
 }
 
 export interface MetricsResults {