Merge branch 'main' into feature/8070-AA-brick-errors

.env.example

@@ -10,6 +10,9 @@ CHROME_EXTENSION_ID=mpjjildhmpddojocokjkgmlkkkfjnepo
 # Shadow DOM mode for all components. Default is SHADOW_DOM=closed in regular webpack builds, open elsewhere
 # SHADOW_DOM=open
 
+# Enable telemetry in development. Default is false
+# DEV_EVENT_TELEMETRY=true
+
 # This makes all optional permissions required in the manifest.json to avoid permission popups. Only required for Playwright tests.
 # REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST=1
 

.eslintrc.js

@@ -143,10 +143,39 @@ module.exports = {
     {
       // TODO: consider packaging e2e tests in a mono-repo structure for specific linting rules
       files: ["end-to-end-tests/**"], // Or *.test.js
+      extends: "plugin:playwright/recommended",
       rules: {
         "no-restricted-imports": "off",
         "unicorn/prefer-dom-node-dataset": "off",
         "unicorn/prefer-module": "off", // `import.meta.dirname` throws "cannot use 'import meta' outside a module"
+        "playwright/no-skipped-test": [
+          "error",
+          {
+            allowConditional: true,
+          },
+        ],
+        "playwright/no-wait-for-timeout": "error",
+        "playwright/no-useless-not": "error",
+        "playwright/expect-expect": [
+          "error",
+          { assertFunctionNames: ["checkUnavailibilityForNavigationMethod"] },
+        ],
+        "playwright/no-conditional-in-test": "error",
+        "playwright/no-conditional-expect": "error",
+        "playwright/no-commented-out-tests": "error",
+        "playwright/no-hooks": "error", // Use fixtures instead to share common setup / teardown code
+        "playwright/no-get-by-title": "error",
+        // Disabled because raw locators are sometimes necessary for specific test scenarios, and we don't want noisy warnings in the IDE
+        // However, our README encourages writing tests using the recommended built-in locators where possible
+        // "playwright/no-raw-locators": "warn",
+        "playwright/prefer-comparison-matcher": "error",
+        "playwright/prefer-equality-matcher": "error",
+        "playwright/prefer-strict-equal": "error",
+        "playwright/prefer-to-be": "error",
+        "playwright/prefer-to-contain": "error",
+        "playwright/prefer-to-have-count": "error",
+        "playwright/prefer-to-have-length": "error",
+        "playwright/require-to-throw-message": "error",
         "no-restricted-syntax": [
           "error",
           {

.github/workflows/beta-release.yaml

@@ -5,12 +5,21 @@ on:
     inputs:
       # This is the branch that will trigger the workflow
       branch:
-        description: "Branch to build. Should be a release/* branch"
+        description: "Branch to publish. Should be a release/* branch"
         required: true
+      skip_tests:
+        description: "Skip the end-to-end tests"
+        required: true
+        default: false
 
 env:
   # Creates and uploads sourcemaps to Application error telemetry, and save the built extension as an artifact
   PUBLIC_RELEASE: true
+  CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs9/BXaFQsYPDxqbVvi11xhWdxfygfrF5YCLiboJooQyTkqIGpGxtI3JF/hkAXDcoqI+N5ATsGPYW34CdOc7uBCU91Ig+gHFiicnkzJaoOBjIwqx452l2/mp7cqNdavtCq40YENkF13ouj5loPwMMYY0L/sSvab+6eO20i1+Ulbsn9onS/fDd16clOaIbUVJ1PhyYvrU0HGVUqW5wUIDLyRezr3aTQLtDIQp/7DTBQ60S2G5KPpAW1UEphnXRLwl6cR5MiYw20OStfTZaA2qpWQvLAQtBoPNjP0Ld6rzI/e3uaC5qUMMCusitKeCA5HOFQDz2IJ0kS8Cn5fxzhXFi6QIDAQAB
+  MV: 3
+  PUBLIC_NAME: -beta
+  DATADOG_APPLICATION_ID: ${{ secrets.DATADOG_APPLICATION_ID }}
+  DATADOG_CLIENT_TOKEN: ${{ secrets.DATADOG_CLIENT_TOKEN }}
 
 jobs:
   build:
@@ -35,11 +44,6 @@ jobs:
           cache: npm
       - run: npm ci
       - run: npm run build
-        env:
-          MV: 3
-          PUBLIC_NAME: -beta
-          DATADOG_APPLICATION_ID: ${{ secrets.DATADOG_APPLICATION_ID }}
-          DATADOG_CLIENT_TOKEN: ${{ secrets.DATADOG_CLIENT_TOKEN }}
       - run: bash scripts/upload-sourcemaps.sh
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.SOURCEMAP_USER_ID }}
@@ -56,7 +60,7 @@ jobs:
           retention-days: 30
           if-no-files-found: error
       - name: Create production version
-        run: npx [email protected] dist/manifest.json key 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs9/BXaFQsYPDxqbVvi11xhWdxfygfrF5YCLiboJooQyTkqIGpGxtI3JF/hkAXDcoqI+N5ATsGPYW34CdOc7uBCU91Ig+gHFiicnkzJaoOBjIwqx452l2/mp7cqNdavtCq40YENkF13ouj5loPwMMYY0L/sSvab+6eO20i1+Ulbsn9onS/fDd16clOaIbUVJ1PhyYvrU0HGVUqW5wUIDLyRezr3aTQLtDIQp/7DTBQ60S2G5KPpAW1UEphnXRLwl6cR5MiYw20OStfTZaA2qpWQvLAQtBoPNjP0Ld6rzI/e3uaC5qUMMCusitKeCA5HOFQDz2IJ0kS8Cn5fxzhXFi6QIDAQAB'
+        run: npx [email protected] dist/manifest.json key '${{ env.CHROME_MANIFEST_KEY }}'
       - name: Save production extension
         uses: actions/upload-artifact@v4
         with:
@@ -65,9 +69,61 @@ jobs:
           retention-days: 30
           if-no-files-found: error
 
+  end-to-end-tests:
+    name: end-to-end-tests
+    if: ${{ github.event.inputs.skip_tests != 'true' }}
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    env:
+      SHADOW_DOM: open
+      SERVICE_URL: https://app.pixiebrix.com
+      REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST: 1
+      E2E_TEST_USER_EMAIL_UNAFFILIATED: ${{ secrets.E2E_TEST_USER_EMAIL_UNAFFILIATED }}
+      E2E_TEST_USER_PASSWORD_UNAFFILIATED: ${{ secrets.E2E_TEST_USER_PASSWORD_UNAFFILIATED }}
+      DEV_EVENT_TELEMETRY: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: package.json
+          cache: npm
+      - name: Install dependencies
+        run: npm ci
+      - name: Install playwright browsers
+        uses: Wandalen/[email protected]
+        with:
+          command: npx playwright install chrome msedge
+          with: |
+            fail_ci_if_error: true
+            verbose: true
+            token: ${{ secrets.CODECOV_TOKEN }}
+          # Installing msedge is flaky due to failed checksums from the Microsoft CDN. Unclear why this happens.
+          # Retry every 15 seconds, for up to 10 minutes
+          attempt_delay: 15000
+          attempt_limit: 40
+      - name: Build the extension
+        run: npm run build:webpack
+      - name: Run end to end tests
+        # Xvfb is required to run the tests in headed mode. Headed mode is required to run tests for browser extensions
+        # in Playwright, see https://playwright.dev/docs/ci#running-headed
+        run: xvfb-run npm run test:e2e
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: end-to-end-tests-report
+          path: end-to-end-tests/.report
+          retention-days: 5
+
   publish:
     # https://github.com/fregante/chrome-webstore-upload-keys
-    needs: build
+    needs:
+      - build
+      - end-to-end-tests
+    # https://stackoverflow.com/a/69354134
+    if: |
+      always() &&
+      needs.build.result == 'success' &&
+      (needs.end-to-end-tests.result == 'success' || needs.end-to-end-tests.result == 'skipped')
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v4

.github/workflows/ci.yml

@@ -1,14 +1,12 @@
 name: CI
 
-on: push
+on: [push, pull_request]
 
 env:
   # Creates and uploads sourcemaps to Application error telemetry, and save the built extension as an artifact
   PUBLIC_RELEASE: ${{ github.ref == 'refs/heads/main' }}
   # Staging URL, also directly used by webpack
   SERVICE_URL: https://app-stg.pixiebrix.com/
-  E2E_TEST_USER_EMAIL_UNAFFILIATED: ${{ secrets.E2E_TEST_USER_EMAIL_UNAFFILIATED }}
-  E2E_TEST_USER_PASSWORD_UNAFFILIATED: ${{ secrets.E2E_TEST_USER_PASSWORD_UNAFFILIATED }}
 
 jobs:
   test:
@@ -191,20 +189,21 @@ jobs:
         include:
           - MV: 2
             PUBLIC_NAME: ""
-            CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
           - MV: 3
             PUBLIC_NAME: "-mv3"
-            CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs9/BXaFQsYPDxqbVvi11xhWdxfygfrF5YCLiboJooQyTkqIGpGxtI3JF/hkAXDcoqI+N5ATsGPYW34CdOc7uBCU91Ig+gHFiicnkzJaoOBjIwqx452l2/mp7cqNdavtCq40YENkF13ouj5loPwMMYY0L/sSvab+6eO20i1+Ulbsn9onS/fDd16clOaIbUVJ1PhyYvrU0HGVUqW5wUIDLyRezr3aTQLtDIQp/7DTBQ60S2G5KPpAW1UEphnXRLwl6cR5MiYw20OStfTZaA2qpWQvLAQtBoPNjP0Ld6rzI/e3uaC5qUMMCusitKeCA5HOFQDz2IJ0kS8Cn5fxzhXFi6QIDAQAB
     timeout-minutes: 60
     runs-on: ubuntu-latest
     env:
       SHADOW_DOM: open
       SERVICE_URL: https://app.pixiebrix.com
       MV: ${{ matrix.MV }}
-      CHROME_MANIFEST_KEY: ${{ matrix.CHROME_MANIFEST_KEY }}
+      CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
       DATADOG_APPLICATION_ID: ${{ secrets.DATADOG_APPLICATION_ID }}
       DATADOG_CLIENT_TOKEN: ${{ secrets.DATADOG_CLIENT_TOKEN }}
       REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST: 1
+      E2E_TEST_USER_EMAIL_UNAFFILIATED: ${{ secrets.E2E_TEST_USER_EMAIL_UNAFFILIATED }}
+      E2E_TEST_USER_PASSWORD_UNAFFILIATED: ${{ secrets.E2E_TEST_USER_PASSWORD_UNAFFILIATED }}
+      DEV_EVENT_TELEMETRY: true
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -237,13 +236,9 @@ jobs:
           name: end-to-end-tests-report${{ matrix.PUBLIC_NAME }}
           path: end-to-end-tests/.report
           retention-days: 5
-  # Analyzer for checking for inclusive terminology in code. For more information, see
-  # https://github.com/microsoft/InclusivenessAnalyzer
-  Inclusiveness-Analyser-scan:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Inclusiveness Analyzer
-        uses: microsoft/[email protected]
+      - uses: daun/playwright-report-summary@v3
+        with:
+          comment-title: "Playwright test results - MV${{matrix.MV}}"
+          report-file: end-to-end-tests/.report/report.json
+          report-tag: manifest-version-${{ matrix.MV }}
+        if: (success() || failure()) && github.event_name == 'pull_request'

.github/workflows/inclusiveness-analyzer.yaml

@@ -0,0 +1,13 @@
+name: Inclusiveness Analyzer
+on: push
+jobs:
+  # Analyzer for checking for inclusive terminology in code. For more information, see
+  # https://github.com/microsoft/InclusivenessAnalyzer
+  Inclusiveness-Analyser-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Inclusiveness Analyzer
+        uses: microsoft/[email protected]

.github/workflows/release.yaml

@@ -9,6 +9,7 @@ on:
 env:
   # Creates and uploads sourcemaps to Application error telemetry, and save the built extension as an artifact
   PUBLIC_RELEASE: true
+  CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
 
 jobs:
   build:
@@ -25,10 +26,8 @@ jobs:
         include:
           - MV: 2
             PUBLIC_NAME: "-mv2"
-            CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
           - MV: 3
             PUBLIC_NAME: "-mv3"
-            CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
 
     name: build${{ matrix.PUBLIC_NAME }}
     steps:
@@ -60,7 +59,7 @@ jobs:
           retention-days: 30
           if-no-files-found: error
       - name: Create production version
-        run: npx [email protected] dist/manifest.json key '${{ matrix.CHROME_MANIFEST_KEY }}'
+        run: npx [email protected] dist/manifest.json key '${{ env.CHROME_MANIFEST_KEY }}'
       - name: Save production extension
         uses: actions/upload-artifact@v4
         with:

end-to-end-tests/README.md

@@ -15,11 +15,13 @@ One-time setup:
     - The test user password `E2E_TEST_USER_PASSWORD_UNAFFILIATED`
     - Uncomment `REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST=1`
     - Uncomment `SHADOW_DOM=open`
+    - Uncomment `DATADOG_CLIENT_TOKEN` (used for telemetry tests, can be set to a fake token, e.g. `secret123`)
+    - Uncomment `DEV_EVENT_TELEMETRY` (used for telemetry tests, actual telemetry requests should be mocked during testing)
   - `MV` will determine the manifest version for the both the extension and the tests (defaulted to 3 if not defined.)
 - Install browsers: Execute `npx playwright install chromium chrome msedge`.
 
 1. Install dependencies: Run `npm install`
-2. Build the extension: Run: `npm run build:webpack` (or `npm run watch`)
+2. Build the extension: Run: `npm run watch`
 3. Run the tests: Use the command `npm run test:e2e`.
 
 - To run tests in interactive UI mode, use `npm run test:e2e -- --ui`. This view shows you the entire test suite and
@@ -42,9 +44,8 @@ Adhere to these principles, based on the [Playwright Best Practices](https://pla
 
 - Utilize `test` from `extensionBase.ts` for test environment setup and extension interaction.
 - Employ page objects from `./end-to-end-tests/pageObjects` for web page interactions.
-- Ensure tests are self-contained, handling their own setup and cleanup.
-  Leverage [Playwright fixtures](https://playwright.dev/docs/test-fixtures) for shared code.
-- Rely on Playwright's auto-waiting feature for actions like clicking or typing.
+- Ensure tests are self-contained, leveraging [Playwright fixtures](https://playwright.dev/docs/test-fixtures) for shared setup/teardown.
+- Use the [recommended built-in locator methods](https://playwright.dev/docs/locators#quick-guide) that have auto-waiting and retry features.
 
 When testing mod functionality, use our testing playground website, https://pbx.vercel.app, for a consistent
 environment. It is configured

end-to-end-tests/auth.setup.ts

@@ -16,7 +16,7 @@
  */
 
 import { expect, type Page } from "@playwright/test";
-import { test } from "./fixtures/authSetupFixture";
+import { test } from "./fixtures/authSetup";
 import {
   E2E_TEST_USER_EMAIL_UNAFFILIATED,
   E2E_TEST_USER_PASSWORD_UNAFFILIATED,
@@ -34,7 +34,7 @@ test("authenticate", async ({ contextAndPage: { context, page } }) => {
     page.getByText(
       "Successfully linked the Browser Extension to your PixieBrix account",
     ),
-    { timeout: 10_000 },
+    { timeout: 12_000 },
   );
   await expect(page.getByText(E2E_TEST_USER_EMAIL_UNAFFILIATED)).toBeVisible();
   await expect(page.getByText("Admin Console")).toBeVisible();

end-to-end-tests/env.ts

@@ -24,6 +24,7 @@ const requiredEnvVariables = [
   "SERVICE_URL",
   "E2E_TEST_USER_EMAIL_UNAFFILIATED",
   "E2E_TEST_USER_PASSWORD_UNAFFILIATED",
+  "SHADOW_DOM",
 ] as const;
 
 // It's not strictly required for the test run itself, but the extension manifest.json must have been built with
@@ -45,20 +46,24 @@ type OptionalEnvVariables = Record<
   string | undefined
 >;
 
-for (const key of requiredEnvVariables) {
-  // eslint-disable-next-line security/detect-object-injection -- key is a constant
-  if (process.env[key] === undefined) {
-    throw new Error(`Required environment variable is not configured: ${key}`);
-  }
+export const assertRequiredEnvVariables = () => {
+  for (const key of requiredEnvVariables) {
+    // eslint-disable-next-line security/detect-object-injection -- key is a constant
+    if (process.env[key] === undefined) {
+      throw new Error(
+        `Required environment variable is not configured: ${key}`,
+      );
+    }
 
-  // eslint-disable-next-line security/detect-object-injection -- key is a constant
-  if (typeof process.env[key] !== "string") {
-    // For the time being we expect all of our requiredEnvVariables to be strings
-    throw new TypeError(
-      `Required environment variable is not configured: ${key}`,
-    );
+    // eslint-disable-next-line security/detect-object-injection -- key is a constant
+    if (typeof process.env[key] !== "string") {
+      // For the time being we expect all of our requiredEnvVariables to be strings
+      throw new TypeError(
+        `Required environment variable is not configured: ${key}`,
+      );
+    }
   }
-}
+};
 
 export const {
   SERVICE_URL,
@@ -71,5 +76,4 @@ export const {
   MV = "3",
   SLOWMO,
   PWDEBUG,
-  REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST,
 } = process.env as OptionalEnvVariables;