diff --git a/lib/smoke/checks.js b/lib/smoke/checks.js new file mode 100644 index 0000000..3117bd5 --- /dev/null +++ b/lib/smoke/checks.js @@ -0,0 +1,132 @@ +const verifyCacheHeaders = require('./verify-cache-headers'); + +module.exports = { + status: (testPage) => { + if (testPage.check.status === 204) { + console.info('204 status checks are not supported yet!'); + return { expected: 204, actual: '¯\_(ツ)_/¯ ', result: true }; + return true; + } else if (typeof testPage.check.status === 'string') { + return { + expected: `redirect to ${testPage.check.status}`, + actual: `redirect to ${testPage.redirect.to}`, + result: testPage.check.status === testPage.redirect.to + }; + } else { + return { expected: testPage.check.status, actual: testPage.status, result: testPage.status === testPage.check.status }; + } + }, + + pageErrors: (testPage) => { + console.debug('Errors on page: ' + testPage.pageErrors); + return { + expected: `no more than ${testPage.check.pageErrors} console errors`, + actual: `${testPage.pageErrors.length} errors`, + result: testPage.pageErrors.length <= testPage.check.pageErrors + }; + }, + + content: async (testPage) => { + const headers = testPage.response.headers(); + const isHTML = headers['content-Type'] && headers['content-Type'].includes('text/html'); + + const content = isHTML ? await testPage.page.content() : await testPage.response.text(); + + let validation; + if(typeof testPage.check.content === 'function') { + validation = testPage.check.content(content); + } else { + validation = (content).includes(testPage.check.content); + } + + return { + expected: 'Response content should validate against provided function', + actual: validation, + result: !!validation + }; + }, + + elements: async (testPage) => { + const results = []; + for(const selector in testPage.check.elements) { + if(testPage.check.elements.hasOwnProperty(selector)) { + const assertion = testPage.check.elements[selector]; + if(typeof assertion === 'number') { + + const count = await testPage.page.$$eval(selector, els => els.filter(el => { + //Filter out elements that are not visible + const rect = el.getBoundingClientRect(); + return rect.height > 0 && rect.width > 0; + }).length); + results.push({ + expected: `should have ${assertion} visible elements matching selector ${selector}`, + actual: count, + result: count === assertion + }); + } else if(typeof assertion === 'string') { + await testPage.page.waitForSelector(selector); + const elText = await testPage.page.$eval(selector, el => el.innerText); + results.push({ + expected: `element with selector selector ${selector} should contain text ${assertion}`, + actual: elText, + result: elText.includes(assertion) + }); + } + }; + + } + return results; + }, + + cacheHeaders: (testPage) => { + let okay = true; + let problems; + try { + verifyCacheHeaders(testPage.headers, testPage.url); + } catch(errors) { + okay = false; + problems = errors; + } + return { + expected: 'Cache-Control headers should be sensible', + actual: okay || problems, + result: okay + }; + }, + + cssCoverage: (testPage) => { + const results = []; + for(const url in testPage.check.cssCoverage) { + if(testPage.check.cssCoverage.hasOwnProperty(url)) { + const threshold = testPage.check.cssCoverage[url]; + const percentage = testPage.coverageFor(url); + + results.push({ + expected: `should be using at least ${threshold}% of the CSS in ${url}`, + actual: percentage ? `${percentage}%` : 'No coverage report found', + result: percentage >= threshold + }); + } + } + + return results; + }, + + performance: async (testPage) => { + const threshold = Number.isInteger(testPage.check.performance) ? parseInt(testPage.check.performance) : 2000; + const paints = await testPage.page.evaluate(() => { + const result = {}; + performance.getEntriesByType('paint').map(entry => { + result[entry.name] = entry.startTime; + }); + return result; + }); + + return [ + { expected: `First Paint to be less than ${threshold}ms`, actual: `${paints['first-paint']}ms`, result: paints['first-paint'] < threshold }, + { expected: `First Contentful Paint to be less than ${threshold}ms`, actual: `${paints['first-contentful-paint']}ms`, result: paints['first-contentful-paint'] < threshold } + ]; + } + + +}; diff --git a/lib/smoke/index.js b/lib/smoke/index.js index e35de7f..ab161c9 100644 --- a/lib/smoke/index.js +++ b/lib/smoke/index.js @@ -1,23 +1,89 @@ const path = require('path'); const fs = require('fs'); +const puppeteer = require('puppeteer'); +const directly = require('directly'); + const test = require('./test'); +const checks = require('./checks'); const filterConfigs = require('./filter-configs'); +const verifyUrl = require('./verify-url'); +const TestPage = require('./test-page'); + + +class SmokeTest { + + constructor (opts) { + this.globalHeaders = opts.headers || {}; + this.checks = Object.assign(opts.checks || {}, checks, {}); + this.configFile = path.join(process.cwd(), opts.config || 'test/smoke.js'); + this.isInteractive = opts.interactive; + this.host = opts.host || 'http://localhost:3002'; + + if (!/https?\:\/\//.test(this.host)) { + this.host = 'http://' + this.host; + } + + if (!fs.existsSync(this.configFile)) { + throw new Error(`Config file for smoke test does not exist at ${this.configFile}. Either create a config file at ./test/smoke.js, or pass in a path using --config.`); + } + } + + addCheck (name, fn) { + this.checks[name] = fn; + }; + + async runSuite (suiteOpts) { + const urlTests = []; -module.exports.run = async (opts, sets) => { - const configFile = path.join(process.cwd(), opts.config || 'test/smoke.js'); - if (!fs.existsSync(configFile)) { - throw new Error(`Config file for smoke test does not exist at ${configFile}. Either create a config file at ./test/smoke.js, or pass in a path using --config.`); - } else { + Object.entries(suiteOpts.urls).forEach(async ([url, urlOpts]) => { - let host = opts.host || 'http://localhost:3002'; + if (typeof urlOpts !== 'object') { + urlOpts = { status: urlOpts }; + } - if (!/https?\:\/\//.test(host)) { - host = 'http://' + host; + urlOpts.headers = Object.assign(this.globalHeaders, urlOpts.headers || {}, suiteOpts.headers || {}, {}); + urlOpts.method = urlOpts.method || suiteOpts.method; + urlOpts.body = urlOpts.body || suiteOpts.body; + urlOpts.https = urlOpts.https || suiteOpts.https; + + const fullUrl = `${this.host}${url}`; + const testPage = new TestPage(fullUrl, urlOpts); + + urlTests.push( + verifyUrl(testPage, this.browser, this.checks) + ); + }); + + return directly(5, urlTests); + } + + + async run (sets) { + const configsToRun = await filterConfigs(this.configFile, sets, this.isInteractive); + this.browser = await puppeteer.launch(); + + let results = []; + + for(let suiteOpts of configsToRun) { + const suiteResults = await this.runSuite(suiteOpts); + results = results.concat(suiteResults); } - const configsToRun = await filterConfigs(configFile, sets, opts.interactive); + this.browser.close(); - return test(configsToRun, host, opts.auth); + const totalResults = { + urlsTested: results.length, + passed: results.filter(url => url.failed === 0), + failed: results.filter(url => url.failed > 0) + }; + if(totalResults.failed.length) { + return Promise.reject(totalResults); + } else { + return Promise.resolve(totalResults); + } } + }; + +module.exports = SmokeTest; diff --git a/lib/smoke/open.js b/lib/smoke/open.js index a96b0e5..78b304b 100644 --- a/lib/smoke/open.js +++ b/lib/smoke/open.js @@ -4,8 +4,6 @@ const puppeteer = require('puppeteer'); const TestPage = require('./test-page'); const filterConfigs = require('./filter-configs'); - - module.exports = async (opts, sets) => { const configFile = path.join(process.cwd(), opts.config || 'test/smoke.js'); if (!fs.existsSync(configFile)) { diff --git a/lib/smoke/test.js b/lib/smoke/test.js deleted file mode 100644 index 6619e6d..0000000 --- a/lib/smoke/test.js +++ /dev/null @@ -1,62 +0,0 @@ -const puppeteer = require('puppeteer'); -const TestPage = require('./test-page'); -const verifyUrl = require('./verify-url'); - -const directly = require('directly'); - -const runSuite = async (suiteOpts, host, authenticate, browser) => { - const urlTests = []; - - Object.entries(suiteOpts.urls).forEach(async ([url, urlOpts]) => { - - if (typeof urlOpts !== 'object') { - urlOpts = { status: urlOpts }; - } - - urlOpts.headers = Object.assign(urlOpts.headers || {}, suiteOpts.headers || {}, {}); - - if(authenticate && process.env.FT_NEXT_BACKEND_KEY) { - urlOpts.headers['FT-Next-Backend-Key'] = process.env.FT_NEXT_BACKEND_KEY; - } - - urlOpts.method = urlOpts.method || suiteOpts.method; - urlOpts.body = urlOpts.body || suiteOpts.body; - urlOpts.https = urlOpts.https || suiteOpts.https; - - const fullUrl = `${host}${url}`; - const testPage = new TestPage(fullUrl, urlOpts); - - urlTests.push( - verifyUrl(testPage, browser) - ); - }); - - return directly(5, urlTests); -}; - -const run = async (config, host, authenticate) => { - const browser = await puppeteer.launch(); - - let results = []; - - for(let suiteOpts of config) { - const suiteResults = await runSuite(suiteOpts, host, authenticate, browser); - results = results.concat(suiteResults); - } - - browser.close(); - - const totalResults = { - urlsTested: results.length, - passed: results.filter(url => url.failed === 0), - failed: results.filter(url => url.failed > 0) - }; - if(totalResults.failed.length) { - return Promise.reject(totalResults); - } else { - return Promise.resolve(totalResults); - } - -}; - -module.exports = run; diff --git a/lib/smoke/verify-cache-headers.js b/lib/smoke/verify-cache-headers.js index 01e292f..447cb5c 100644 --- a/lib/smoke/verify-cache-headers.js +++ b/lib/smoke/verify-cache-headers.js @@ -1,6 +1,5 @@ module.exports = (headers, path) => { - const cacheErrors = []; if (!headers['surrogate-control'] && !headers['cache-control']) { diff --git a/lib/smoke/verify-url.js b/lib/smoke/verify-url.js index 422db07..e3f2b84 100644 --- a/lib/smoke/verify-url.js +++ b/lib/smoke/verify-url.js @@ -1,139 +1,5 @@ /* eslint-disable no-console */ - -const verifyCacheHeaders = require('./verify-cache-headers'); - -const tests = { - - status: (testPage) => { - if (testPage.check.status === 204) { - console.info('204 status checks are not supported yet!'); - return { expected: 204, actual: '¯\_(ツ)_/¯ ', result: true }; - return true; - } else if (typeof testPage.check.status === 'string') { - return { - expected: `redirect to ${testPage.check.status}`, - actual: `redirect to ${testPage.redirect.to}`, - result: testPage.check.status === testPage.redirect.to - }; - } else { - return { expected: testPage.check.status, actual: testPage.status, result: testPage.status === testPage.check.status }; - } - }, - - pageErrors: (testPage) => { - console.debug('Errors on page: ' + testPage.pageErrors); - return { - expected: `no more than ${testPage.check.pageErrors} console errors`, - actual: `${testPage.pageErrors.length} errors`, - result: testPage.pageErrors.length <= testPage.check.pageErrors - }; - }, - - content: async (testPage) => { - const headers = testPage.response.headers(); - const isHTML = headers['content-type'] && headers['content-type'].includes('text/html'); - - const content = isHTML ? await testPage.page.content() : await testPage.response.text(); - - let validation; - if(typeof testPage.check.content === 'function') { - validation = testPage.check.content(content); - } else { - validation = (content).includes(testPage.check.content); - } - - return { - expected: 'Response content should validate against provided function', - actual: validation, - result: !!validation - }; - }, - - elements: async (testPage) => { - const results = []; - for(const selector in testPage.check.elements) { - if(testPage.check.elements.hasOwnProperty(selector)) { - const assertion = testPage.check.elements[selector]; - if(typeof assertion === 'number') { - - const count = await testPage.page.$$eval(selector, els => els.filter(el => { - //Filter out elements that are not visible - const rect = el.getBoundingClientRect(); - return rect.height > 0 && rect.width > 0; - }).length); - results.push({ - expected: `should have ${assertion} visible elements matching selector ${selector}`, - actual: count, - result: count === assertion - }); - } else if(typeof assertion === 'string') { - await testPage.page.waitForSelector(selector); - const elText = await testPage.page.$eval(selector, el => el.innerText); - results.push({ - expected: `element with selector selector ${selector} should contain text ${assertion}`, - actual: elText, - result: elText.includes(assertion) - }); - } - }; - - } - return results; - }, - - cacheHeaders: (testPage) => { - let okay = true; - let problems; - try { - verifyCacheHeaders(testPage.headers, testPage.url); - } catch(errors) { - okay = false; - problems = errors; - } - return { - expected: 'Cache-Control headers should be sensible', - actual: okay || problems, - result: okay - }; - }, - - cssCoverage: (testPage) => { - const results = []; - for(const url in testPage.check.cssCoverage) { - if(testPage.check.cssCoverage.hasOwnProperty(url)) { - const threshold = testPage.check.cssCoverage[url]; - const percentage = testPage.coverageFor(url); - - results.push({ - expected: `should be using at least ${threshold}% of the CSS in ${url}`, - actual: percentage ? `${percentage}%` : 'No coverage report found', - result: percentage >= threshold - }); - } - } - - return results; - }, - - performance: async (testPage) => { - const threshold = Number.isInteger(testPage.check.performance) ? parseInt(testPage.check.performance) : 2000; - const paints = await testPage.page.evaluate(() => { - const result = {}; - performance.getEntriesByType('paint').map(entry => { - result[entry.name] = entry.startTime; - }); - return result; - }); - - return [ - { expected: `First Paint to be less than ${threshold}ms`, actual: `${paints['first-paint']}ms`, result: paints['first-paint'] < threshold }, - { expected: `First Contentful Paint to be less than ${threshold}ms`, actual: `${paints['first-contentful-paint']}ms`, result: paints['first-contentful-paint'] < threshold } - ]; - } - -}; - -const verifyUrl = (testPage, browser) => async () => { +const verifyUrl = (testPage, browser, tests) => async () => { //open the page await testPage.init(browser); diff --git a/tasks/smoke.js b/tasks/smoke.js index cb7e7a8..178ec5a 100644 --- a/tasks/smoke.js +++ b/tasks/smoke.js @@ -1,15 +1,21 @@ -const smokeTests = require('../lib/smoke'); +const SmokeTests = require('../lib/smoke'); module.exports = (program) => { + + + program .command('smoke [sets...]') .option('-a, --auth', 'Authenticate with FT_NEXT_BACKEND_KEY') .option('-H, --host [value]', 'Set the hostname to use for all tests') - .option('-c, --config [value]', 'Path to config file used to test. Defaults to ./test/smoke.json') + .option('-c, --config [value]', 'Set the path to the global config file. Defaults to .n-test.js') + .option('-t, --testFile [value]', 'Path to config file used to test. Defaults to ./test/smoke.json') .option('-i, --interactive [value]', 'Interactively choose which tests to run. Defaults to false') .description('Tests that a given set of urls for an app respond as expected. Expects the config file ./test/smoke.json to exist') .action((sets, opts) => { - smokeTests.run(opts, sets).catch(err => { + + const smokeTests = new SmokeTests(opts); + smokeTests.run(sets).catch(err => { if(err.failed) { // eslint-disable-next-line no-console console.error(`${err.failed.length} URLs failed their check.`); diff --git a/test/tasks/smoke.js b/test/tasks/smoke.js index 49ab39c..25385e3 100644 --- a/test/tasks/smoke.js +++ b/test/tasks/smoke.js @@ -1,7 +1,7 @@ /*globals beforeAll, expect */ const server = require('../server/app'); -const smoke = require('../../lib/smoke'); +const SmokeTests = require('../../lib/smoke'); describe('Smoke Tests of the Smoke', () => { @@ -12,11 +12,11 @@ describe('Smoke Tests of the Smoke', () => { describe('status checks', () => { test('tests should pass if all the urls return the correct status', (done) => { - - return smoke.run({ + const smoke = new SmokeTests({ host: 'http://localhost:3004', config: 'test/fixtures/smoke-status-pass.js' - }) + }); + return smoke.run() .then((results) => { expect(results.passed.length).toEqual(7); expect(results.failed.length).toEqual(0); @@ -26,10 +26,11 @@ describe('Smoke Tests of the Smoke', () => { test('tests should fail if some urls return the incorrect status code', (done) => { - return smoke.run({ + const smoke = new SmokeTests({ host: 'http://localhost:3004', - config: 'test/fixtures/smoke-status-fail.js', - }) + config: 'test/fixtures/smoke-status-fail.js' + }); + return smoke.run() .catch((results) => { expect(results.passed.length).toEqual(1); expect(results.failed.length).toEqual(1); @@ -41,10 +42,11 @@ describe('Smoke Tests of the Smoke', () => { describe('CSS coverage', () => { test('tests should pass if CSS is well covered',(done) => { - return smoke.run({ + const smoke = new SmokeTests({ host: 'http://localhost:3004', config: 'test/fixtures/smoke-coverage-pass.js' - }) + }); + return smoke.run() .then((results) => { expect(results.passed.length).toEqual(2); expect(results.failed.length).toEqual(0); @@ -55,10 +57,11 @@ describe('Smoke Tests of the Smoke', () => { test('tests should fail if CSS coverage is below threshold', (done) => { - smoke.run({ + const smoke = new SmokeTests({ host: 'http://localhost:3004', - config: 'test/fixtures/smoke-coverage-fail.js', - }) + config: 'test/fixtures/smoke-coverage-fail.js' + }); + return smoke.run() .catch((results) => { expect(results.passed.length).toEqual(0); expect(results.failed.length).toEqual(2);