Merge pull request #11199 from wellcomecollection/pa11y-e2e

Adding pa11y to e2e step in PRs
wellcomecollection · Sep 26, 2024 · 6d6a181 · 6d6a181
2 parents 324bc61 + 39a938b
commit 6d6a181
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 7 deletions.
diff --git a/.buildkite/e2e_on_pull_requests/pipeline.e2e-pull-requests.yml b/.buildkite/e2e_on_pull_requests/pipeline.e2e-pull-requests.yml
@@ -36,6 +36,13 @@
   agents:
     queue: nano
 
+- label: "Run pa11y tests"
+  depends_on: deploy-to-e2e-environment
+  plugins:
+    - docker-compose#v3.5.0:
+        run: pa11y
+        command: [ "yarn", "write-report", "--isPullRequestRun" ]
+
 - wait
 
 - label: "Scale down e2e environment"

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -126,7 +126,6 @@
         command: ['yarn', 'test']
 
 # This step kicks of the "end-to-end on pull requests" flow.
-#
 # It doesn't run on main, because we run e2es on main by deploying
 # into our stage/prod environments instead.
 

diff --git a/pa11y/webapp/package.json b/pa11y/webapp/package.json
@@ -7,7 +7,9 @@
   },
   "dependencies": {
     "aws-sdk": "^2.1659.0",
+    "chalk": "^4.1.2",
     "pa11y": "8.0.0",
-    "puppeteer": "22.13.1"
+    "puppeteer": "22.13.1",
+    "yargs": "^17.7.2"
   }
 }
diff --git a/pa11y/webapp/write-report.js b/pa11y/webapp/write-report.js
@@ -1,15 +1,26 @@
+const chalk = require('chalk');
 const events = require('events');
 const fs = require('fs');
 const pa11y = require('pa11y');
 const { promisify } = require('util');
+const yargs = require('yargs');
 
 const writeFile = promisify(fs.writeFile);
 
 events.EventEmitter.defaultMaxListeners = 25;
 
+const { isPullRequestRun } = yargs(process.argv.slice(2))
+  .usage('Usage: $0 --isPullRequestRun [boolean]')
+  .options({
+    isPullRequestRun: { type: 'boolean' },
+  })
+  .parseSync();
+
 console.info('Pa11y: Starting report');
 
-const baseUrl = 'https://wellcomecollection.org';
+const baseUrl = isPullRequestRun
+  ? 'https://www-e2e.wellcomecollection.org'
+  : 'https://wellcomecollection.org';
 
 // Note: if you add a URL to this list, make sure to also add it to the list
 // of URLs checked by the URL checker.
@@ -63,8 +74,39 @@ const promises = urls.map(url =>
 
 Promise.all(promises)
   .then(async results => {
-    await fs.promises.mkdir('./.dist', { recursive: true });
-    await writeFile('./.dist/report.json', JSON.stringify({ results }));
-    console.info('Reporting done!');
+    if (isPullRequestRun) {
+      const resultsLog = results
+        .map(result => {
+          return result.issues.length > 0
+            ? {
+                title: result.documentTitle,
+                url: result.pageUrl,
+                errors: result.issues.map(issue => ({
+                  type: issue.type,
+                  message: issue.message,
+                })),
+              }
+            : undefined;
+        })
+        .filter(f => f)
+        .flat();
+
+      if (resultsLog.length > 0) {
+        console.error(`!!! ${chalk.redBright('Fix these before merging')}`);
+        console.log(...resultsLog);
+
+        // TODO do we want it to stop people from merging also when it's of type "warning" or "notice"?
+        const hasErrors = results.find(result =>
+          result.issues.find(issue => issue.type === 'error')
+        );
+        if (hasErrors) process.exit(1);
+      } else {
+        console.log(chalk.greenBright('Report done, no errors found'));
+      }
+    } else {
+      await fs.promises.mkdir('./.dist', { recursive: true });
+      await writeFile('./.dist/report.json', JSON.stringify({ results }));
+      console.info(chalk.greenBright('Reporting done!'));
+    }
   })
   .catch(e => console.info(e));
diff --git a/pa11y/webapp/yarn.lock b/pa11y/webapp/yarn.lock
@@ -77,7 +77,7 @@ ansi-styles@^3.2.1:
   dependencies:
     color-convert "^1.9.0"
 
-ansi-styles@^4.0.0:
+ansi-styles@^4.0.0, ansi-styles@^4.1.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937"
   integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==
@@ -235,6 +235,14 @@ chalk@^2.4.2:
     escape-string-regexp "^1.0.5"
     supports-color "^5.3.0"
 
+chalk@^4.1.2:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01"
+  integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==
+  dependencies:
+    ansi-styles "^4.1.0"
+    supports-color "^7.1.0"
+
 check-types@^11.2.3:
   version "11.2.3"
   resolved "https://registry.yarnpkg.com/check-types/-/check-types-11.2.3.tgz#1ffdf68faae4e941fce252840b1787b8edc93b71"
@@ -540,6 +548,11 @@ has-flag@^3.0.0:
   resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd"
   integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==
 
+has-flag@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b"
+  integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==
+
 has-property-descriptors@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854"
@@ -1043,6 +1056,13 @@ supports-color@^5.3.0:
   dependencies:
     has-flag "^3.0.0"
 
+supports-color@^7.1.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da"
+  integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==
+  dependencies:
+    has-flag "^4.0.0"
+
 tar-fs@^3.0.6:
   version "3.0.6"
   resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.6.tgz#eaccd3a67d5672f09ca8e8f9c3d2b89fa173f217"