WIP: make smoke tests more reusable - turn into class

 🐿 v2.6.0

lib/smoke/checks.js

@@ -0,0 +1,132 @@
+const verifyCacheHeaders = require('./verify-cache-headers');
+
+module.exports = {
+	status: (testPage) => {
+		if (testPage.check.status === 204) {
+			console.info('204 status checks are not supported yet!');
+			return { expected: 204, actual: '¯\_(ツ)_/¯ ', result: true };
+			return true;
+		} else if (typeof testPage.check.status === 'string') {
+			return {
+				expected: `redirect to ${testPage.check.status}`,
+				actual: `redirect to ${testPage.redirect.to}`,
+				result: testPage.check.status === testPage.redirect.to
+			};
+		} else {
+			return { expected: testPage.check.status, actual: testPage.status, result: testPage.status === testPage.check.status };
+		}
+	},
+
+	pageErrors: (testPage) => {
+		console.debug('Errors on page: ' + testPage.pageErrors);
+		return {
+			expected: `no more than ${testPage.check.pageErrors} console errors`,
+			actual: `${testPage.pageErrors.length} errors`,
+			result: testPage.pageErrors.length <= testPage.check.pageErrors
+		};
+	},
+
+	content: async (testPage) => {
+		const headers = testPage.response.headers();
+		const isHTML = headers['content-Type'] && headers['content-Type'].includes('text/html');
+
+		const content = isHTML ? await testPage.page.content() : await testPage.response.text();
+
+		let validation;
+		if(typeof testPage.check.content === 'function') {
+			validation = testPage.check.content(content);
+		} else {
+			validation = (content).includes(testPage.check.content);
+		}
+
+		return {
+			expected: 'Response content should validate against provided function',
+			actual: validation,
+			result: !!validation
+		};
+	},
+
+		elements: async (testPage) => {
+			const results = [];
+			for(const selector in testPage.check.elements) {
+					if(testPage.check.elements.hasOwnProperty(selector)) {
+					const assertion = testPage.check.elements[selector];
+					if(typeof assertion === 'number') {
+
+						const count = await testPage.page.$$eval(selector, els => els.filter(el => {
+							//Filter out elements that are not visible
+							const rect = el.getBoundingClientRect();
+							return rect.height > 0 && rect.width > 0;
+						}).length);
+						results.push({
+							expected: `should have ${assertion} visible elements matching selector ${selector}`,
+							actual: count,
+							result: count === assertion
+						});
+					} else if(typeof assertion === 'string') {
+							await testPage.page.waitForSelector(selector);
+							const elText = await testPage.page.$eval(selector, el => el.innerText);
+							results.push({
+								expected: `element with selector selector ${selector} should contain text ${assertion}`,
+								actual: elText,
+								result: elText.includes(assertion)
+							});
+					}
+				};
+
+			}
+			return results;
+		},
+
+	cacheHeaders: (testPage) => {
+		let okay = true;
+		let problems;
+		try {
+			verifyCacheHeaders(testPage.headers, testPage.url);
+		} catch(errors) {
+			okay = false;
+			problems = errors;
+		}
+		return {
+			expected: 'Cache-Control headers should be sensible',
+			actual: okay || problems,
+			result: okay
+		};
+	},
+
+	cssCoverage: (testPage) => {
+		const results = [];
+		for(const url in testPage.check.cssCoverage) {
+			if(testPage.check.cssCoverage.hasOwnProperty(url)) {
+				const threshold = testPage.check.cssCoverage[url];
+				const percentage = testPage.coverageFor(url);
+
+				results.push({
+					expected: `should be using at least ${threshold}% of the CSS in ${url}`,
+					actual: percentage ? `${percentage}%` : 'No coverage report found',
+					result: percentage >= threshold
+				});
+			}
+		}
+
+		return results;
+	},
+
+	performance: async (testPage) => {
+		const threshold = Number.isInteger(testPage.check.performance) ? parseInt(testPage.check.performance) : 2000;
+		const paints = await testPage.page.evaluate(() => {
+			const result = {};
+			performance.getEntriesByType('paint').map(entry => {
+				result[entry.name] = entry.startTime;
+			});
+			return result;
+		});
+
+		return [
+			{ expected: `First Paint to be less than ${threshold}ms`, actual: `${paints['first-paint']}ms`, result: paints['first-paint'] < threshold },
+			{ expected: `First Contentful Paint to be less than ${threshold}ms`, actual: `${paints['first-contentful-paint']}ms`, result: paints['first-contentful-paint'] < threshold }
+		];
+	}
+
+
+};

lib/smoke/index.js

@@ -1,23 +1,89 @@
 const path = require('path');
 const fs = require('fs');
+const puppeteer = require('puppeteer');
+const directly = require('directly');
+
 
 const test = require('./test');
+const checks = require('./checks');
 const filterConfigs = require('./filter-configs');
+const verifyUrl = require('./verify-url');
+const TestPage = require('./test-page');
+
+
+class SmokeTest {
+
+	constructor (opts) {
+		this.globalHeaders = opts.headers || {};
+		this.checks = Object.assign(opts.checks || {}, checks, {});
+		this.configFile = path.join(process.cwd(), opts.config || 'test/smoke.js');
+		this.isInteractive = opts.interactive;
+		this.host = opts.host || 'http://localhost:3002';
+
+		if (!/https?\:\/\//.test(this.host)) {
+			this.host = 'http://' + this.host;
+		}
+
+		if (!fs.existsSync(this.configFile)) {
+			throw new Error(`Config file for smoke test does not exist at ${this.configFile}. Either create a config file at ./test/smoke.js, or pass in a path using --config.`);
+		}
+	}
+
+	addCheck (name, fn) {
+		this.checks[name] = fn;
+	};
+
+	async runSuite (suiteOpts) {
+		const urlTests = [];
 
-module.exports.run = async (opts, sets) => {
-	const configFile = path.join(process.cwd(), opts.config || 'test/smoke.js');
-	if (!fs.existsSync(configFile)) {
-		throw new Error(`Config file for smoke test does not exist at ${configFile}. Either create a config file at ./test/smoke.js, or pass in a path using --config.`);
-	} else {
+		Object.entries(suiteOpts.urls).forEach(async ([url, urlOpts]) => {
 
-		let host = opts.host || 'http://localhost:3002';
+			if (typeof urlOpts !== 'object') {
+				urlOpts = { status: urlOpts };
+			}
 
-		if (!/https?\:\/\//.test(host)) {
-			host = 'http://' + host;
+			urlOpts.headers = Object.assign(this.globalHeaders, urlOpts.headers || {}, suiteOpts.headers || {}, {});
+			urlOpts.method = urlOpts.method || suiteOpts.method;
+			urlOpts.body = urlOpts.body || suiteOpts.body;
+			urlOpts.https = urlOpts.https || suiteOpts.https;
+
+			const fullUrl = `${this.host}${url}`;
+			const testPage = new TestPage(fullUrl, urlOpts);
+
+			urlTests.push(
+				verifyUrl(testPage, this.browser, this.checks)
+			);
+		});
+
+		return directly(5, urlTests);
+	}
+
+
+	async run (sets) {
+		const configsToRun = await filterConfigs(this.configFile, sets, this.isInteractive);
+		this.browser = await puppeteer.launch();
+
+		let results = [];
+
+		for(let suiteOpts of configsToRun) {
+			const suiteResults = await this.runSuite(suiteOpts);
+			results = results.concat(suiteResults);
 		}
 
-		const configsToRun = await filterConfigs(configFile, sets, opts.interactive);
+		this.browser.close();
 
-		return test(configsToRun, host, opts.auth);
+		const totalResults = {
+			urlsTested: results.length,
+			passed: results.filter(url => url.failed === 0),
+			failed: results.filter(url => url.failed > 0)
+		};
+		if(totalResults.failed.length) {
+			return Promise.reject(totalResults);
+		} else {
+			return Promise.resolve(totalResults);
+		}
 	}
+
 };
+
+module.exports = SmokeTest;

lib/smoke/open.js

@@ -4,8 +4,6 @@ const puppeteer = require('puppeteer');
 const TestPage = require('./test-page');
 const filterConfigs = require('./filter-configs');
 
-
-
 module.exports = async (opts, sets) => {
 	const configFile = path.join(process.cwd(), opts.config || 'test/smoke.js');
 	if (!fs.existsSync(configFile)) {

lib/smoke/verify-cache-headers.js

@@ -1,6 +1,5 @@
 module.exports = (headers, path) => {
 
-
 	const cacheErrors = [];
 
 	if (!headers['surrogate-control'] && !headers['cache-control']) {