feat: add playwright renederer and test project (scullyio#1469)

* feat: add playwright renederer and test project

* refactor(playwright): change pw to plugin system

* fix: run pw deps install before crunning pw

* fix: add better messaging

* tests: fix the failing test with dummy

* refactor: small changes based on review

.github/workflows/workflow.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        node: [ 16.x]
+        node: [16.x]
 
     steps:
       - name: Checkout branch
@@ -117,6 +117,38 @@ jobs:
           name: static-sites-docs
           path: dist/static/doc-sites
 
+  buildPWSample:
+    name: Build playwright Sample-blog
+    needs: buildlibs
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node: [16.x]
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v2
+      - name: Setup node
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node }}
+      - name: Cache Node Modules
+        id: cache-node-modules
+        uses: actions/cache@v2
+        with:
+          path: node_modules
+          key: node-modules-${{ matrix.node }}-${{secrets.CACHEKEY}}-${{ hashFiles('package-lock.json') }}
+      - name: Install Dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: npm ci
+      - name: Build lib using NX cache
+        run: |
+          npm run symlinks
+          npm run nx -- run-many --target=build --all --prod
+      - name: build pw-sample-blog
+        run: |
+          node ./dist/libs/scully/src/scully --project pw-sample-blog --tds --RSD --scan --404=index --ks
+
   jest-test:
     name: Run Jest tests
     needs: [buildDocs, buildSample]
@@ -213,7 +245,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [ 16.x ]
+        node: [16.x]
     steps:
       - name: Checking out
         uses: actions/checkout@master
@@ -247,7 +279,7 @@ jobs:
 
           # Optional options appended to `git-commit`
           # See https://git-scm.com/docs/git-commit for a list of available options
-          commit_options: '--no-verify --signoff'
+          commit_options: "--no-verify --signoff"
           file_pattern: releaseChecksums.json
 
           # Optional commit user and author settings
@@ -261,7 +293,7 @@ jobs:
 
           # Optional options appended to `git-push`
           # See git-push documentation for details: https://git-scm.com/docs/git-push#_options
-          push_options: '--force'
+          push_options: "--force"
 
           # Optional: Disable dirty check and always try to create a commit and push
           skip_dirty_check: false
@@ -275,7 +307,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        prop: [ { node: 14, angular: 12 }]
+        prop: [{ node: 14, angular: 12 }]
     steps:
       - name: Setup angular project and run all tests
         uses: actions/setup-node@v2

jest.config.js

@@ -1,3 +1,3 @@
 module.exports = {
-  projects: ['<rootDir>/tests/jest/src', '<rootDir>/libs/platform-server', '<rootDir>/libs/plugins/scully-plugin-puppeteer'],
+  projects: ['<rootDir>/tests/jest/src', '<rootDir>/libs/platform-server', '<rootDir>/libs/plugins/scully-plugin-puppeteer', '<rootDir>/libs/plugins/scully-plugin-playwright'],
 };

libs/plugins/scully-plugin-playwright/.babelrc

@@ -0,0 +1,3 @@
+{
+  "presets": [["@nrwl/web/babel", { "useBuiltIns": "usage" }]]
+}

libs/plugins/scully-plugin-playwright/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

libs/plugins/scully-plugin-playwright/README.md

@@ -0,0 +1,11 @@
+# scully-plugin-playwright
+
+This is the playwright render plugin for Scully.
+
+The interface for a renderPlugin is:
+
+```ts
+  (route:HandledRoute) => Promise<string>
+```
+
+This plugin will be called for every route that is in the `handledRoute[]` When it throws its retried for 3 times. If it fails after that, the route is skipped.

libs/plugins/scully-plugin-playwright/jest.config.js

@@ -0,0 +1,15 @@
+module.exports = {
+  displayName: 'plugins-scully-plugin-playwright',
+  preset: '../../../jest.preset.js',
+  globals: {
+    'ts-jest': {
+      tsconfig: '<rootDir>/tsconfig.spec.json',
+    },
+  },
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]sx?$': 'ts-jest',
+  },
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+  coverageDirectory: '../../../coverage/libs/plugins/scully-plugin-playwright',
+};

libs/plugins/scully-plugin-playwright/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@scullyio/scully-plugin-playwright",
+  "version": "0.0.1",
+  "repository": {
+    "type": "GIT",
+    "url": "https://github.com/scullyio/scully/tree/main/libs/plugins/scully-plugin-playwright"
+  },
+  "keywords": [
+    "angular",
+    "scully",
+    "seo",
+    "scully-plugin",
+    "plugin",
+    "playwright"
+  ],
+  "dependencies": {
+    "tslib": "^1.13.0"
+  },
+  "peerDependencies": {
+    "@scullyio/scully": "*",
+    "playwright": "^1.16.3"
+  }
+}

libs/plugins/scully-plugin-playwright/src/index.ts

@@ -0,0 +1,39 @@
+import { green, log, logError, logOk, registerPlugin, routeRenderer } from '@scullyio/scully';
+import { exec } from 'child_process';
+import { LaunchOptions } from 'playwright';
+import {  playwrightRenderer } from './lib/plugins-scully-plugin-playwright';
+import { launchedBrowser, launchedBrowser$ } from './lib/plugins-scully-plugin-playwright-utils';
+
+async function runScript(cmd: string) {
+  return new Promise((resolve, reject) => {
+    exec(cmd, (err, stdout, stderr) => {
+      if (err) {
+        log(stderr);
+        reject(err);
+      } else {
+        resolve(stdout);
+      }
+    });
+  });
+}
+const plugin = async () => {
+  await runScript(`npx playwright install`).catch(() => {
+    logError(`Playwright install failed. Please fix the above errors in the app, and run Scully again.`);
+    process.exit(0);
+  });
+  log(`  ${green('✔')} Playwright installation successfully`);
+}
+export function enablePW() {
+  registerPlugin('beforeAll', 'installPWDeps', plugin);
+
+  registerPlugin('scullySystem', routeRenderer, playwrightRenderer, undefined, { replaceExistingPlugin: true });
+
+  registerPlugin('enterprise', 'getPWLaunchedBrowser', async () => launchedBrowser$)
+  registerPlugin('beforeAll', 'startLaunching the browser', async () => {
+    logOk('Playwright is being launched')
+    launchedBrowser();
+  })
+}
+
+export { playwrightRender } from './lib/plugins-scully-plugin-playwright';
+export type BrowserLaunchOptions = LaunchOptions & { browser: string };

libs/plugins/scully-plugin-playwright/src/lib/plugins-scully-plugin-playwright-utils.ts

@@ -0,0 +1,156 @@
+import { loadConfig, logError, logWarn, white, yellow } from '@scullyio/scully';
+import { showBrowser } from '@scullyio/scully/src/lib/utils/cli-options';
+import * as playwright from "playwright";
+import { Browser, LaunchOptions } from "playwright";
+import { BehaviorSubject, catchError, delayWhen, filter, from, merge, Observable, of, shareReplay, switchMap, take, throttleTime, timer } from 'rxjs';
+
+const defaultConfig: LaunchOptions = {
+  headless: true,
+  channel: 'chrome',
+  browser: 'chromium',
+} as any;
+const options = { ...defaultConfig };
+
+
+const launches = new BehaviorSubject<void>(undefined);
+
+export let browser: Browser;
+export function waitForIt(milliSeconds: number) {
+  return new Promise<void>((resolve) => setTimeout(() => resolve(), milliSeconds));
+}
+
+let usageCounter = 0;
+export const launchedBrowser: () => Promise<Browser> = async () => {
+  if (++usageCounter > 500) {
+    launches.next();
+    usageCounter = 0;
+  }
+  return launchedBrowser$.pipe(take(1)).toPromise();
+};
+
+export const reLaunch = (reason?: string): Promise<Browser> => {
+  if (reason) {
+    logWarn(
+      white(`
+  ========================================
+      Relaunch because of ${reason}
+  ========================================
+
+      `)
+    );
+  }
+  launches.next();
+  return launchedBrowser();
+};
+
+const launch = async (pluginConfig: any): Promise<Browser> => {
+  const browserType = pluginConfig.browser
+  const playrightBrowser = playwright[browserType];
+  const browser = await playrightBrowser.launch({ headless: pluginConfig.headless, channel: pluginConfig.channel });
+  return browser;
+}
+export const launchedBrowser$: Observable<Browser> = of('').pipe(
+  /** load config only after a subscription is made */
+  switchMap(() => loadConfig()),
+  /** give the system a bit of breathing room, and prevent race */
+  switchMap(() => from(waitForIt(50))),
+  switchMap(() => merge(obsBrowser(), launches)),
+  /** use shareReplay so the browser will stay in memory during the lifetime of the program */
+  shareReplay({ refCount: false, bufferSize: 1 }),
+  filter<Browser>((e) => e !== undefined)
+);
+
+function obsBrowser(): Observable<Browser> {
+  let isLaunching = false;
+  if (showBrowser) {
+    options.headless = false;
+  }
+  options.args = options.args || [];
+  return new Observable((obs) => {
+    const startPlaywright = () => {
+      if (!isLaunching) {
+        isLaunching = true;
+        launchPlayWrightWithRetry(options).then((b) => {
+          /** I will only come here when playwright is actually launched */
+          browser = b;
+          b.on('disconnected', () => reLaunch('disconnect'));
+          obs.next(b);
+          /** only allow a relaunch in a next cycle */
+          setTimeout(() => (isLaunching = false), 1000);
+        });
+      }
+    };
+
+    launches
+      .pipe(
+        /** ignore request while the browser is already starting, we can only launch 1 */
+        filter(() => !isLaunching),
+        /** the long throttleTime is to cater for the concurrently running browsers to crash and burn. */
+        throttleTime(15000),
+        // provide enough time for the current async operations to finish before killing the current browser instance
+        delayWhen(() =>
+          merge(
+            /** timout at 25 seconds */
+            timer(25000)
+          ).pipe(
+            /** use take 1 to make sure we complete when one of the above fires */
+            take(1),
+            /** if something _really_ unwieldy happens with the browser, ignore and go ahead */
+            catchError(() => of([]))
+          )
+        )
+      )
+      .subscribe({
+        next: () => {
+          try {
+            if (browser && browser.contexts() != null) {
+              browser.close();
+            }
+          } catch {
+            /** ignored */
+          }
+          startPlaywright();
+        },
+      });
+    return () => {
+      if (browser) {
+        browser.close();
+        browser = undefined;
+      }
+    };
+  });
+}
+function launchPlayWrightWithRetry(options, failedLaunches = 0): Promise<Browser> {
+  const timeout = (millisecs: number) => new Promise((_, reject) => setTimeout(() => reject('timeout'), millisecs));
+  return Promise.race([
+    /** use a 1 minute timeout to detect a stalled launch of playwright */
+    timeout(Math.max(/** serverTimeout,*/ 60 * 1000)),
+    launch(options).then((b) => {
+      return b as unknown as Browser;
+    }),
+  ])
+    .catch((e) => {
+      /** first stage catch check for retry */
+      if (e.message.includes('Could not find browser revision')) {
+        throw new Error('Failed launch');
+      }
+      if (++failedLaunches < 3) {
+        return launchPlayWrightWithRetry(options, failedLaunches);
+      }
+      throw new Error('failed 3 times to launch');
+    })
+    .catch((b) => {
+      /** second stage catch, houston, we have a problem, and will abort */
+      logError(`
+=================================================================================================
+Playwright cannot find or launch the browser. (by default chrome)
+This might happen because the default timeout (60 seconds) is to short on this system
+this can be fixed by adding the ${yellow('--serverTimeout=x')} cmd line option.
+   (where x = the new timeout in milliseconds)
+When this happens in CI/CD you can find some additional information here:
+https://playwright.dev/docs/troubleshooting
+=================================================================================================
+      `);
+      process.exit(15);
+    }) as unknown as Promise<Browser>;
+}

libs/plugins/scully-plugin-playwright/src/lib/plugins-scully-plugin-playwright.spec.ts

@@ -0,0 +1,7 @@
+import { playwrightRenderer } from './plugins-scully-plugin-playwright';
+
+describe('playwrightRenderer', () => {
+  it('should work', () => {
+    expect(playwrightRenderer).toEqual(playwrightRenderer);
+  });
+});