Visual tests (#27)

* Added visual testing mechanism

---------

Co-authored-by: Satya Deep Maheshwari <[email protected]>

.github/visual-tests/collect-urls.mjs

@@ -0,0 +1,27 @@
+import { writeFileSync } from 'fs';
+
+// This script is used to collect all the URLs that we want to test.
+// It will fetch the URLs from the index and write them to a file.
+
+// check if the environment variables are set
+if (!process.env.DOMAIN_MAIN || !process.env.TEST_PATHS || !process.env.TEST_PATHS_INDEXES) {
+  console.error('Please set the environment variables DOMAIN_MAIN, TEST_PATHS and TEST_PATHS_INDEXES');
+  process.exit(1);
+}
+
+const paths = process.env.TEST_PATHS.split(' ').map((path) => path.trim());
+
+console.log(process.env.TEST_PATHS_INDEXES);
+for (const index of process.env.TEST_PATHS_INDEXES.split(' ')) {
+  if (!index.trim().length) continue;
+
+  const indexUrl = `https://${process.env.DOMAIN_MAIN}${index}`;
+  console.log(`fetching from ${indexUrl}`);
+  const response = await fetch(indexUrl)
+  const json = await response.json();
+  paths.push(...json.blocks.data.map((item) => {
+    const url = new URL(item.path);
+    return url.pathname
+  }));
+}
+writeFileSync('./generated-test-paths.txt', paths.join("\n"));

.github/visual-tests/compare.spec.ts

@@ -0,0 +1,102 @@
+import { Page, test } from '@playwright/test';
+import { getComparator } from 'playwright-core/lib/utils';
+import { unlink, writeFile } from 'fs/promises';
+
+
+function getScreenshotPath(testPath: string, suffix) {
+  const title = testPath.replace(/[/]/g, '-');
+  return `./screenshots/${(title.toLowerCase())}-${suffix}.png`;
+}
+
+/**
+ * Wait for all images on document to be loaded.
+ *
+ * @param page {page}
+ * @param timeout {number}: how many milliseconds to wait until reject and cancel the execution.
+ * @param tickrate {number}: how many milliseconds to wait until recheck all images again.
+ * @returns {Promise}
+ *   A promise which resolve when all img on document gets fetched.
+ *   The promise get rejected if it reach the @timeout time to execute.
+ *
+ *  Based on https://stackoverflow.com/a/51652947/79461
+ */
+async function allImagesLoaded(page, timeout = 15 * 1000, tickrate = 250) {
+  const images = await page.locator('img').all();
+  const startTime = new Date().getTime();
+
+  return new Promise((resolve, reject) => {
+
+    function checkImages() {
+      const currentTime = new Date().getTime();
+
+      if (currentTime - startTime > timeout) {
+        reject({
+          message: `CheckImgReadyTimeoutException: images taking to loong to load.`
+        });
+      }
+
+      if (images.every(img => img.evaluate(el => el.complete))) {
+        resolve(images);
+      } else {
+        setTimeout(checkImages, tickrate);
+      }
+    }
+
+    checkImages();
+  });
+}
+
+async function loadAndScreenshot(page: Page, url: string, testPath: string, suffix: string) {
+  // load page and wait for network to be idle
+  await page.goto(url, { waitUntil: 'networkidle' });
+  // just to be sure, wait until footer is loaded
+  if(!url.includes('/sidekick/blocks')) {
+    await page.locator('footer div.footer.block[data-block-status="loaded"]').waitFor();
+  }
+
+  // to be extra sure, also wait until all images are loaded
+  await allImagesLoaded(page);
+
+  return await page.screenshot({
+    path: getScreenshotPath(testPath, suffix),
+    fullPage: true
+  });
+}
+
+
+for (let testPath of process.env.TEST_PATHS.split(/\s+/g)) {
+  testPath = testPath.trim();
+  if (!testPath) continue;
+
+  test(`${testPath}`, async ({ page }, testInfo) => {
+    const urlMain = `https://${process.env.DOMAIN_MAIN}${testPath}`;
+    const urlBranch = `https://${process.env.DOMAIN_BRANCH}${testPath}`;
+
+    const beforeImage = await loadAndScreenshot(page, urlMain, testPath, "main");
+    const afterImage = await loadAndScreenshot(page, urlBranch, testPath, "branch");
+
+    const comparator = getComparator('image/png');
+    const result = comparator(beforeImage, afterImage, {
+      maxDiffPixelRatio: 0.01,
+    });
+
+    if (result && result.errorMessage) {
+      await writeFile(getScreenshotPath(testPath, 'diff'), result.diff);
+
+      // print markdown summary to console
+      const markdownSummary = ` - **${testPath}** ([main](${urlMain}) vs [branch](${urlBranch}))<br>${result.errorMessage}`;
+      console.log(markdownSummary);
+      testInfo.attachments.push({
+        name: getScreenshotPath(testPath, 'diff'),
+        contentType: `image/png`,
+        path: getScreenshotPath(testPath, 'diff')
+      });
+      throw new Error(markdownSummary);
+    } else {
+      // if there is no difference, delete the images to save space in the artifact
+      await unlink(getScreenshotPath(testPath, 'main'));
+      await unlink(getScreenshotPath(testPath, 'branch'));
+    }
+  })
+
+}

.github/visual-tests/optimize-screenshots.mjs

@@ -0,0 +1,21 @@
+import sharp from "sharp";
+import * as path from "path";
+import { readdirSync, unlinkSync } from "fs";
+
+// list files in the directory 'snapshots'
+const snapshots = readdirSync("screenshots");
+
+snapshots
+	.filter(file => file.endsWith(".png"))
+	.forEach(async (file) => {
+		const filepath = path.join("screenshots", file);
+		const newFilepath = filepath.replace(".png", ".webp");
+
+		await sharp(filepath, {})
+			.resize({ width: 400 })
+			.webp({ quality: 80 })
+			.toFile(newFilepath);
+
+		// remove the original file
+		unlinkSync(filepath);
+	});