Test: fix flaky firewall test INTER-320

.github/workflows/ci.yml

@@ -106,12 +106,7 @@ jobs:
         run: yarn build
 
       - name: Run Playwright tests
-        run: yarn playwright test --grep-invert CHROME_ONLY --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
-
-      # Some tests are only run on Chrome, marked with CHROME_ONLY in their name
-      - name: Run Chrome-only Playwright tests
-        run: yarn playwright test --grep CHROME_ONLY --project='chromium'
-        if: matrix.shardIndex == 1
+        run: yarn playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
       - name: Upload Playwright report
         uses: actions/upload-artifact@v4

e2e/bot-firewall.spec.ts

@@ -1,4 +1,4 @@
-import { expect, test } from '@playwright/test';
+import { Locator, Page, expect, test } from '@playwright/test';
 import { resetScenarios } from './resetHelper';
 import { TEST_IDS } from '../src/client/testIDs';
 import { BOT_FIREWALL_COPY } from '../src/client/bot-firewall/botFirewallCopy';
@@ -9,17 +9,22 @@
   : 'https://staging.fingerprinthub.com/web-scraping';
 
 /**
- * CHROME_ONLY flag tells the GitHub action to run this test only using Chrome.
- * This test relies on a single common Cloudflare ruleset, we we cannot run multiple instances of it at the same time.
+ * Only run this test in Chrome
+ * This test relies on a single common Cloudflare ruleset, we cannot run multiple instances of it at the same time.
  */
+test.skip(({ browserName }) => browserName !== 'chromium', 'Chrome-only');
+/**
+ * Increase timeout to give Cloudflare time to update the ruleset
+ */
+test.setTimeout(60000);
+
 test.describe('Bot Firewall Demo CHROME_ONLY', () => {
   test.beforeEach(async ({ page }) => {
     await page.goto('/coupon-fraud');
     await resetScenarios(page);
   });
 
   test('Should display bot visit and allow blocking/unblocking its IP address', async ({ page, context }) => {
-    test.setTimeout(60000);
     // Record bot visit in web-scraping page
     await page.goto('/web-scraping');
     await expect(page.getByTestId(TEST_IDS.common.alert)).toContainText('Malicious bot detected');
@@ -28,28 +33,56 @@
     await page.goto('/bot-firewall');
     await page.getByRole('button', { name: BOT_FIREWALL_COPY.blockIp }).first().click();
     await page.getByText('was blocked in the application firewall').waitFor();
-    await page.waitForTimeout(3000);
 
     /**
-     * Try to visit web-scraping page, should be blocked by Cloudflare
-     * Checking the response code here as parsing the actual page if flaky for some reason.
+     * Try to visit web-scraping page, should be blocked by Cloudflare.
      * Using a separate tab also seems to help with flakiness.
      */
     const secondTab = await context.newPage();
     await secondTab.goto(WEB_SCRAPING_URL);
-    await secondTab.reload();
-    await secondTab.getByRole('heading', { name: 'Sorry, you have been blocked' }).waitFor();
+
+    await assertElementWhileRepeatedlyReloadingPage(
+      secondTab,
+      secondTab.getByRole('heading', { name: 'Sorry, you have been blocked' }),
+    );
 
     // Unblock IP
     await page.goto('/bot-firewall');
     await page.getByRole('button', { name: BOT_FIREWALL_COPY.unblockIp }).first().click();
     await page.getByText('was unblocked in the application firewall').waitFor();
-    // Give Cloudflare some time to change the firewall rule
-    await page.waitForTimeout(20000);
 
     // Try to visit web-scraping page, should be allowed again
     await secondTab.goto(WEB_SCRAPING_URL);
-    await secondTab.reload();
-    await expect(secondTab.getByRole('heading', { name: 'Web Scraping Prevention' })).toBeVisible();
+    await assertElementWhileRepeatedlyReloadingPage(
+      secondTab,
+      secondTab.getByRole('heading', { name: 'Web Scraping Prevention' }),
+    );
   });
 });
+
+/**
+ * Asserts the visibility of a given element by repeatedly reloading the page and waiting for the element to become visible.
+ * This is useful for testing non-SPA pages where updates require a page reload.
+ *
+ * @param {Page} page - The page object to interact with.
+ * @param {Locator} locator - The locator for the element to be checked for visibility.
+ * @param {number} waitBetweenAttempts - The time to wait between each visibility check attempt, in milliseconds. Defaults to 5000.
+ * @param {number} tries - The number of attempts to check the visibility of the element. Defaults to 5.
+ * @return {Promise<void>} - A promise that resolves when the element becomes visible, or rejects with an error if the element is not visible after the specified number of attempts.
+ */
+const assertElementWhileRepeatedlyReloadingPage = async (
+  page: Page,
+  locator: Locator,
+  waitBetweenAttempts = 5000,
+  tries = 5,
+) => {
+  for (let i = 0; i < tries; i++) {
+    const isVisible = await locator.isVisible();
+    if (isVisible) {
+      break;
+    }
+    await page.waitForTimeout(waitBetweenAttempts);
+    await page.reload();
+  }
+  await expect(locator).toBeVisible();
+};