👷 visual regression tests (#1553)

.github/actions/netlify-wait-for-build

@@ -9,8 +9,8 @@ function get_status() {
   curl --header "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/actualbudget/actual/commits/$COMMIT_SHA/statuses" > /tmp/status.json
   cat /tmp/status.json
   echo "::endgroup::"
-  netlify=$(jq '[.[] | select(.context == "netlify/actualbudget/deploy-preview")][0]' /tmp/status.json)
-  state=$(jq -r '.state' <<< "$netlify")
+  netlify=$(yarn jq '[.[] | select(.context == "netlify/actualbudget/deploy-preview")][0]' /tmp/status.json)
+  state=$(yarn jq -r '.state' <<< "$netlify")
   echo "::group::Netlify Status"
   echo "$netlify"
   echo "::endgroup::"
@@ -32,7 +32,7 @@ done
 
 if [ "$state" == "success" ]; then
   echo -e "\033[0;32mNetlify build succeeded!\033[0m"
-  jq -r '"url=" + .target_url' <<< "$netlify" > $GITHUB_OUTPUT
+  yarn jq -r '"url=" + .target_url' <<< "$netlify" > $GITHUB_OUTPUT
   exit 0
 else
   echo -e "\033[0;31mNetlify build failed. Cancelling end-to-end tests.\033[0m"

.github/workflows/e2e-test.yml

@@ -10,15 +10,15 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: Run end-to-end tests on Netlify PR preview
+  functional:
+    name: Functional
     runs-on: ubuntu-latest
+    container:
+      image: mcr.microsoft.com/playwright:v1.37.0-jammy
     steps:
       - uses: actions/checkout@v3
       - name: Set up environment
         uses: ./.github/actions/setup
-      - name: Setup Playwright
-        run: npx playwright install chromium --with-deps
       - name: Wait for Netlify build to finish
         id: netlify
         env:
@@ -35,3 +35,28 @@ jobs:
           name: desktop-client-test-results
           path: packages/desktop-client/test-results/
           retention-days: 30
+  vrt:
+    name: Visual regression
+    runs-on: ubuntu-latest
+    container:
+      image: mcr.microsoft.com/playwright:v1.37.0-jammy
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Wait for Netlify build to finish
+        id: netlify
+        env:
+          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./.github/actions/netlify-wait-for-build
+      - name: Run VRT Tests on Netlify URL
+        run: yarn vrt
+        env:
+          E2E_START_URL: ${{ steps.netlify.outputs.url }}
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: desktop-client-test-results
+          path: packages/desktop-client/test-results/
+          retention-days: 30

package.json

@@ -33,11 +33,13 @@
     "test": "yarn workspaces foreach --parallel --verbose run test",
     "test:debug": "yarn workspaces foreach --verbose run test",
     "e2e": "yarn workspaces foreach --parallel --verbose run e2e",
+    "vrt": "yarn workspaces foreach --parallel --verbose run vrt",
     "rebuild-electron": "./node_modules/.bin/electron-rebuild -f -m ./packages/loot-core",
     "rebuild-node": "yarn workspace loot-core rebuild",
     "lint": "eslint . --max-warnings 0",
     "lint:verbose": "DEBUG=eslint:cli-engine eslint . --max-warnings 0",
     "typecheck": "yarn tsc",
+    "jq": "./node_modules/node-jq/bin/jq",
     "postinstall": "patch-package"
   },
   "devDependencies": {
@@ -49,6 +51,7 @@
     "eslint-plugin-prettier": "4.2.1",
     "eslint-plugin-react": "7.32.2",
     "eslint-plugin-rulesdir": "^0.2.2",
+    "node-jq": "^4.0.1",
     "npm-run-all": "^4.1.3",
     "patch-package": "^6.1.2",
     "prettier": "2.8.2",

packages/desktop-client/README.md

@@ -4,11 +4,13 @@ Actual on the web
 
 E2E (end-to-end) tests use [Playwright](https://playwright.dev/). Running them requires an Actual server to be running either locally or on a remote server.
 
+### Functional
+
 Running against the local server:
 
 ```sh
 # Start the development server
-yarn start:browser
+yarn start
 
 # Run against the local server (localhost:3001)
 yarn e2e
@@ -19,3 +21,37 @@ Running against a remote server:
 ```sh
 E2E_START_URL=http://my-remote-server.com yarn e2e
 ```
+
+### Visual regression
+
+Visual regression tests (also known as screenshot tests) check that the visual appearance of the product has not regressed. Each environment has slightly different colors, fonts etc. Mac differs from Windows which differs from Linux. In order to have a stable test environment for visual comparisons - you must use a standartised docker container. This ensures that the tests are always ran in a consistent environment.
+
+Prerequisites:
+
+- Docker installed
+
+#### Running against the local server
+
+First start the dev server:
+
+```sh
+HTTPS=true yarn start
+```
+
+Next, run the standartised docker container and launch the visual regression tests from within it.
+
+```sh
+# Run docker container
+docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v1.37.0-jammy /bin/bash
+
+# Run the VRT tests: important - they MUST be ran against a HTTPS server
+E2E_START_URL=https://192.168.0.178:3001 yarn vrt
+```
+
+#### Running against a remote server
+
+You can also run the tests against a remote server by passing the URL:
+
+```sh
+E2E_START_URL=https://my-remote-server.com yarn vrt
+```

packages/desktop-client/e2e/accounts.test.js

@@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test';
 
 import { ConfigurationPage } from './page-models/configuration-page';
 import { Navigation } from './page-models/navigation';
+import screenshotConfig from './screenshot.config';
 
 test.describe('Accounts', () => {
   let page;
@@ -28,13 +29,13 @@ test.describe('Accounts', () => {
       balance: 100,
     });
 
-    expect(await accountPage.getNthTransaction(0)).toMatchObject({
-      payee: 'Starting Balance',
-      notes: '',
-      category: 'Starting Balances',
-      debit: '',
-      credit: '100.00',
-    });
+    const transaction = accountPage.getNthTransaction(0);
+    await expect(transaction.payee).toHaveText('Starting Balance');
+    await expect(transaction.notes).toHaveText('');
+    await expect(transaction.category).toHaveText('Starting Balances');
+    await expect(transaction.debit).toHaveText('');
+    await expect(transaction.credit).toHaveText('100.00');
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 
   test('closes an account', async () => {
@@ -44,8 +45,10 @@ test.describe('Accounts', () => {
 
     const modal = await accountPage.clickCloseAccount();
     await modal.selectTransferAccount('Vanguard 401k');
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
     await modal.closeAccount();
 
     await expect(accountPage.accountName).toHaveText('Closed: Roth IRA');
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 });

packages/desktop-client/e2e/budget.test.js

@@ -1,6 +1,7 @@
 import { test, expect } from '@playwright/test';
 
 import { ConfigurationPage } from './page-models/configuration-page';
+import screenshotConfig from './screenshot.config';
 
 test.describe('Budget', () => {
   let page;
@@ -13,6 +14,11 @@ test.describe('Budget', () => {
 
     await page.goto('/');
     budgetPage = await configurationPage.createTestFile();
+
+    // Move mouse to corner of the screen;
+    // sometimes the mouse hovers on a budget element thus rendering an input box
+    // and this breaks screenshot tests
+    await page.mouse.move(0, 0);
   });
 
   test.afterAll(async () => {
@@ -28,6 +34,7 @@ test.describe('Budget', () => {
     await expect(summary.getByText(/^Overspent in /)).toBeVisible();
     await expect(summary.getByText('Budgeted')).toBeVisible();
     await expect(summary.getByText('For Next Month')).toBeVisible();
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 
   test('transfer funds to another category', async () => {
@@ -40,6 +47,7 @@ test.describe('Budget', () => {
     expect(await budgetPage.getBalanceForRow(2)).toEqual(
       currentFundsA + currentFundsB,
     );
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 
   test('budget table is rendered', async () => {

packages/desktop-client/e2e/mobile.test.js

@@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test';
 
 import { ConfigurationPage } from './page-models/configuration-page';
 import { MobileNavigation } from './page-models/mobile-navigation';
+import screenshotConfig from './screenshot.config';
 
 test.describe('Mobile', () => {
   let page;
@@ -43,15 +44,17 @@ test.describe('Mobile', () => {
       'Water',
       'Power',
     ]);
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 
   test('opens the accounts page and asserts on balances', async () => {
     const accountsPage = await navigation.goToAccountsPage();
 
     const account = await accountsPage.getNthAccount(0);
 
-    expect(account.name).toEqual('Ally Savings');
-    expect(account.balance).toBeGreaterThan(0);
+    await expect(account.name).toHaveText('Ally Savings');
+    await expect(account.balance).toHaveText('7,653.00');
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 
   test('opens individual account page and checks that filtering is working', async () => {
@@ -62,17 +65,21 @@ test.describe('Mobile', () => {
     expect(await accountPage.getBalance()).toBeGreaterThan(0);
 
     await expect(accountPage.noTransactionsFoundError).not.toBeVisible();
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
 
     await accountPage.searchByText('nothing should be found');
     await expect(accountPage.noTransactionsFoundError).toBeVisible();
     await expect(accountPage.transactions).toHaveCount(0);
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
 
     await accountPage.searchByText('Kroger');
     await expect(accountPage.transactions).not.toHaveCount(0);
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 
   test('creates a transaction via footer button', async () => {
     const transactionEntryPage = await navigation.goToTransactionEntryPage();
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
 
     await expect(transactionEntryPage.header).toHaveText('New Transaction');
 
@@ -89,12 +96,14 @@ test.describe('Mobile', () => {
       page.getByTestId('account-field'),
       'Ally Savings',
     );
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
 
     const accountPage = await transactionEntryPage.createTransaction();
 
     await expect(accountPage.transactions.nth(0)).toHaveText(
       'KrogerClothing-12.34',
     );
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 
   test('creates a transaction from `/accounts/:id` page', async () => {
@@ -103,6 +112,7 @@ test.describe('Mobile', () => {
     const transactionEntryPage = await accountPage.clickCreateTransaction();
 
     await expect(transactionEntryPage.header).toHaveText('New Transaction');
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
 
     await transactionEntryPage.amountField.fill('12.34');
     await transactionEntryPage.fillField(
@@ -123,6 +133,7 @@ test.describe('Mobile', () => {
 
   test('checks that settings page can be opened', async () => {
     const settingsPage = await navigation.goToSettingsPage();
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
 
     const downloadPromise = page.waitForEvent('download');
 
@@ -133,5 +144,6 @@ test.describe('Mobile', () => {
     expect(await download.suggestedFilename()).toMatch(
       /^\d{4}-\d{2}-\d{2}-.*.zip$/,
     );
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 });

packages/desktop-client/e2e/onboarding.test.js

@@ -5,6 +5,7 @@ import { test, expect } from '@playwright/test';
 import { AccountPage } from './page-models/account-page';
 import { ConfigurationPage } from './page-models/configuration-page';
 import { Navigation } from './page-models/navigation';
+import screenshotConfig from './screenshot.config';
 
 test.describe('Onboarding', () => {
   let page;
@@ -23,6 +24,14 @@ test.describe('Onboarding', () => {
     await page.close();
   });
 
+  test('checks the page visuals', async () => {
+    await expect(configurationPage.heading).toHaveText('Where’s the server?');
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
+
+    await configurationPage.clickOnNoServer();
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
+  });
+
   test('creates a new budget file by importing YNAB4 budget', async () => {
     await configurationPage.clickOnNoServer();
     const budgetPage = await configurationPage.importBudget(
@@ -64,7 +73,7 @@ test.describe('Onboarding', () => {
       path.resolve(__dirname, 'data/actual-demo-budget.zip'),
     );
 
-    await expect(budgetPage.budgetTable).toBeVisible();
+    await expect(budgetPage.budgetTable).toBeVisible({ timeout: 20_000 });
 
     const accountPage = await navigation.goToAccountPage('Ally Savings');
     await expect(accountPage.accountBalance).toHaveText('1,772.80');

packages/desktop-client/e2e/page-models/account-page.js

@@ -70,15 +70,15 @@ export class AccountPage {
    * Retrieve the data for the nth-transaction.
    * 0-based index
    */
-  async getNthTransaction(index) {
+  getNthTransaction(index) {
     const row = this.transactionTableRow.nth(index);
 
     return {
-      payee: await row.getByTestId('payee').textContent(),
-      notes: await row.getByTestId('notes').textContent(),
-      category: await row.getByTestId('category').textContent(),
-      debit: await row.getByTestId('debit').textContent(),
-      credit: await row.getByTestId('credit').textContent(),
+      payee: row.getByTestId('payee'),
+      notes: row.getByTestId('notes'),
+      category: row.getByTestId('category'),
+      debit: row.getByTestId('debit'),
+      credit: row.getByTestId('credit'),
     };
   }
 

packages/desktop-client/e2e/page-models/mobile-accounts-page.js

@@ -14,11 +14,8 @@ export class MobileAccountsPage {
     const accountRow = this.accounts.nth(idx);
 
     return {
-      name: await accountRow.getByTestId('account-name').textContent(),
-      balance: parseInt(
-        await accountRow.getByTestId('account-balance').textContent(),
-        10,
-      ),
+      name: accountRow.getByTestId('account-name'),
+      balance: accountRow.getByTestId('account-balance'),
     };
   }
 

packages/desktop-client/e2e/reports.test.js

@@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test';
 
 import { ConfigurationPage } from './page-models/configuration-page';
 import { Navigation } from './page-models/navigation';
+import screenshotConfig from './screenshot.config';
 
 test.describe('Reports', () => {
   let page;
@@ -31,5 +32,6 @@ test.describe('Reports', () => {
     const reports = await reportsPage.getAvailableReportList();
 
     expect(reports).toEqual(['Net Worth', 'Cash Flow']);
+    await expect(page).toHaveScreenshot(screenshotConfig(page));
   });
 });