feat(scully-plugin-puppeteer): move Puppeteer to a stand-alone (optio…

…nal) plugin (scullyio#1470)

* feat(scully-plugin-puppeteer): move Puppeteer to a stand-alone (optional) plugin

ISSUES CLOSED: scullyio#1462

* fix(scully-plugin-puppeteer): fix circular dependency issue by ovoinf the name in TEXT messages.

ISSUES CLOSED: scullyio#1462

* refactor(scully-plugin-puppeteer): don't use a config setting anymore for the 'default' routeRenderer

.gitignore

@@ -57,3 +57,4 @@ testDocs
 scully/tsconfig.sps-sample.json
 adsample.js
 bs.sh
+consoleProxy.ts

.vscode/settings.json

@@ -20,9 +20,6 @@
     "statusBar.background": "#131f64",
     "statusBarItem.hoverBackground": "#1b2c8f",
     "statusBar.foreground": "#e7e7e7",
-    "panel.border": "#1b2c8f",
-    "sideBar.border": "#1b2c8f",
-    "editorGroup.border": "#1b2c8f",
     "sash.hoverBorder": "#1b2c8f",
     "statusBarItem.remoteBackground": "#131f64",
     "statusBarItem.remoteForeground": "#e7e7e7"

jest.config.js

@@ -1,3 +1,3 @@
 module.exports = {
-  projects: ['<rootDir>/tests/jest/src', '<rootDir>/libs/platform-server'],
+  projects: ['<rootDir>/tests/jest/src', '<rootDir>/libs/platform-server', '<rootDir>/libs/plugins/scully-plugin-puppeteer'],
 };

libs/plugins/extra/package.json

@@ -21,4 +21,4 @@
     "scully-plugin",
     "plugin"
   ]
-}
+}

libs/plugins/scully-plugin-local-cache/src/lib/installInterceptor.ts

@@ -1,10 +1,15 @@
-import { launchedBrowser$ } from '@scullyio/scully';
+import { findPlugin } from '@scullyio/scully';
 import { Browser, Page, Target } from 'puppeteer';
+import { Observable } from 'rxjs';
 import { config } from './config';
 import { handlePuppeteerRequest } from './handlePuppeteerRequest';
 import { handlePuppeteerResponse } from './handlePuppeteerResponse';
 
-export function installInterceptor() {
+export async function installInterceptor() {
+  const launchedBrowser$ = findPlugin('getPPTLaunchedBrowser')() as Observable<Browser>;
+  if (!launchedBrowser$) {
+    throw new Error('No launched browser found');
+  }
   launchedBrowser$.subscribe(async (browser: Browser) => {
     browser.on('targetcreated', async (ev: Target) => {
       const page: Page = await ev.page();

libs/plugins/scully-plugin-puppeteer/.babelrc

@@ -0,0 +1,3 @@
+{
+  "presets": [["@nrwl/web/babel", { "useBuiltIns": "usage" }]]
+}

libs/plugins/scully-plugin-puppeteer/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

libs/plugins/scully-plugin-puppeteer/README.md

@@ -0,0 +1,10 @@
+# scully-plugin-puppeteer
+
+This is the Puppeteer render plugin for Scully. 
+
+The interface for a renderPlugin is:
+```ts
+  (route:HandledRoute) => Promise<string>
+```
+
+This plugin will be called for every route that is in the `handledRoute[]` When it throws its retried for 3 times. If it fails after that, the route is skipped.

libs/plugins/scully-plugin-puppeteer/jest.config.js

@@ -0,0 +1,15 @@
+module.exports = {
+  displayName: 'plugins-scully-plugin-puppeteer',
+  preset: '../../../jest.preset.js',
+  globals: {
+    'ts-jest': {
+      tsconfig: '<rootDir>/tsconfig.spec.json',
+    },
+  },
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]sx?$': 'ts-jest',
+  },
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+  coverageDirectory: '../../../coverage/libs/plugins/scully-plugin-puppeteer',
+};

libs/plugins/scully-plugin-puppeteer/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@scullyio/scully-plugin-puppeteer",
+  "version": "2.0.0",
+  "description": "Scully Puppeteer render plugin",
+  "main": "./src/index.js",
+  "license": "MIT",
+  "dependencies": {
+    "tslib": "^1.13.0"
+  },
+  "peerDependencies": {
+    "@scullyio/scully": "*",
+    "puppeteer": "^10.2.0"
+  },
+  "repository": {
+    "type": "GIT",
+    "url": "https://github.com/scullyio/scully/tree/main/libs/plugins/scully-plugin-puppeteer"
+  },
+  "keywords": [
+    "angular",
+    "scully",
+    "seo",
+    "scully-plugin",
+    "puppeteer",
+    "plugin",
+    "render"
+  ]
+}

libs/plugins/scully-plugin-puppeteer/src/index.ts

@@ -0,0 +1,11 @@
+import { logOk, routeRenderer, registerPlugin } from '@scullyio/scully';
+import { launchedBrowser } from './lib/launchedBrowser';
+import { puppeteerRender } from './lib/plugins-scully-plugin-puppeteer';
+import { launchedBrowser$ } from './lib/launchedBrowser';
+
+registerPlugin('enterprise','getPPTLaunchedBrowser',async () => launchedBrowser$)
+registerPlugin('beforeAll', 'startLaunching the browser', async () => {
+  logOk('Puppeteer is being launched')
+  launchedBrowser();
+})
+registerPlugin('scullySystem', routeRenderer, puppeteerRender);

libs/scully/src/lib/renderPlugins/launchedBrowser.ts → libs/plugins/scully-plugin-puppeteer/src/lib/launchedBrowser.ts

@@ -1,11 +1,8 @@
+import { cliOptions, loadConfig, logError, logOk, scullyConfig, waitForIt, yellow } from '@scullyio/scully';
 import { Browser, launch } from 'puppeteer';
-import { BehaviorSubject, from, interval, merge, Observable, of, timer } from 'rxjs';
-import { catchError, delayWhen, filter, shareReplay, switchMap, take, throttleTime } from 'rxjs';
-import { showBrowser, serverTimeout } from '../utils/cli-options';
-import { loadConfig, scullyConfig } from '../utils/config';
-import { logOk, logError, yellow } from '../utils/log';
-import { waitForIt } from './puppeteerRenderPlugin';
+import { BehaviorSubject, catchError, delayWhen, filter, from, interval, merge, Observable, of, shareReplay, switchMap, take, throttleTime, timer } from 'rxjs';
 
+const { showBrowser, serverTimeout } = cliOptions;
 const launches = new BehaviorSubject<void>(undefined);
 /**
  * Returns an Observable with that will fire with the launched puppeteer in there.

libs/plugins/scully-plugin-puppeteer/src/lib/plugins-scully-plugin-puppeteer.spec.ts

@@ -0,0 +1,7 @@
+import { puppeteerRender } from './plugins-scully-plugin-puppeteer';
+
+describe('pluginsScullyPluginPuppeteer', () => {
+  it('should work', () => {
+    expect(puppeteerRender).toEqual(puppeteerRender)
+  });
+});

libs/scully/src/lib/renderPlugins/puppeteerRenderPlugin.ts → libs/plugins/scully-plugin-puppeteer/src/lib/plugins-scully-plugin-puppeteer.ts

@@ -1,19 +1,13 @@
 // tslint:disable: no-string-literal
-// const puppeteer = require('puppeteer');
+
+import { createFolderFor, HandledRoute, logError, logWarn, scullyConfig, title404, waitForIt, yellow } from '@scullyio/scully';
+import { captureException } from '@scullyio/scully/src/lib/utils/captureMessage';
+import { showBrowser, ssl } from '@scullyio/scully/src/lib/utils/cli-options';
 import { readFileSync } from 'fs-extra';
 import { jsonc } from 'jsonc';
 import { join } from 'path';
 import { Browser, Page, Serializable } from 'puppeteer';
-import { interval, Subject } from 'rxjs';
-import { filter, switchMap, take } from 'rxjs';
-import { registerPlugin, scullySystem } from '../pluginManagement';
-import { HandledRoute } from '../routerPlugins/handledRoute.interface';
-import { createFolderFor } from '../utils';
-import { showBrowser, ssl } from '../utils/cli-options';
-import { scullyConfig } from '../utils/config';
-import { logError, logWarn, yellow } from '../utils/log';
-import { captureException } from '../utils/captureMessage';
-import { title404 } from '../utils/serverstuff/title404';
+import { filter, interval, Subject, switchMap, take } from 'rxjs';
 import { launchedBrowser, reLaunch } from './launchedBrowser';
 
 const errorredPages = new Map<string, number>();
@@ -28,16 +22,16 @@ try {
   // version = jsonc.parse(readFileSync(join(__dirname, '../../../package.json')).toString()).version || '0.0.0';
 }
 
-export const puppeteerRender = 'puppeteerRender' as const;
 
-const plugin = async (route: HandledRoute): Promise<string> => {
+
+export const puppeteerRender = async (route: HandledRoute): Promise<string> => {
   const timeOutValueInSeconds = 25;
   const pageLoaded = new Subject<void>();
   const path = route.rawRoute
     ? route.rawRoute
     : scullyConfig.hostUrl
-    ? `${scullyConfig.hostUrl}${route.route}`
-    : `http${ssl ? 's' : ''}://${scullyConfig.hostName}:${scullyConfig.appPort}${route.route}`;
+      ? `${scullyConfig.hostUrl}${route.route}`
+      : `http${ssl ? 's' : ''}://${scullyConfig.hostName}:${scullyConfig.appPort}${route.route}`;
 
   let pageHtml: string;
   let browser: Browser;
@@ -221,17 +215,13 @@ const plugin = async (route: HandledRoute): Promise<string> => {
       /** give it a couple of secs */
       await waitForIt(3 * 1000);
       /** retry! */
-      return plugin(route);
+      return puppeteerRender(route);
     }
   }
 
   return pageHtml;
 };
 
-export function waitForIt(milliSeconds: number) {
-  return new Promise<void>((resolve) => setTimeout(() => resolve(), milliSeconds));
-}
-
 const windowSet = (page: Page, name: string, value: Serializable) =>
   page.evaluateOnNewDocument(`
     Object.defineProperty(window, '${name}', {
@@ -241,4 +231,3 @@ const windowSet = (page: Page, name: string, value: Serializable) =>
     })
   `);
 
-registerPlugin(scullySystem, puppeteerRender, plugin);

libs/plugins/scully-plugin-puppeteer/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

libs/plugins/scully-plugin-puppeteer/tsconfig.lib.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "../../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"]
+  },
+  "exclude": ["**/*.spec.ts"],
+  "include": ["**/*.ts"]
+}

libs/plugins/scully-plugin-puppeteer/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": ["**/*.spec.ts", "**/*.spec.tsx", "**/*.spec.js", "**/*.spec.jsx", "**/*.d.ts"]
+}

libs/scully/src/index.ts

@@ -4,18 +4,17 @@ import {
   getPluginConfig,
   setConfig,
   setPluginConfig,
-  setPluginPriority,
+  setPluginPriority
 } from './lib/pluginManagement/pluginConfig';
 import { configValidator, registerPlugin } from './lib/pluginManagement/pluginRepository';
 import './lib/pluginManagement/systemPlugins';
 import { ContentMetaData } from './lib/renderPlugins/content-render-utils/readFileAndCheckPrePublishSlug';
 import { renderRoute } from './lib/renderPlugins/executePlugins';
-import { launchedBrowser$ } from './lib/renderPlugins/launchedBrowser';
 import { ContentTextRoute, HandledRoute, RouteConfig } from './lib/routerPlugins/handledRoute.interface';
 import { WriteToStorage } from './lib/systemPlugins/writeToFs.plugin';
 import { createFolderFor } from './lib/utils';
 import { prod } from './lib/utils/cli-options';
-import { loadConfig, scullyConfig, updateScullyConfig } from './lib/utils/config';
+import { loadConfig, routeRenderer, scullyConfig, updateScullyConfig } from './lib/utils/config';
 import './lib/utils/exitHandler';
 import { handleTravesal } from './lib/utils/handlers/handleTravesal';
 import { routeDiscovery } from './lib/utils/handlers/routeDiscovery';
@@ -24,12 +23,18 @@ import { RouteTypes, ScullyConfig } from './lib/utils/interfacesandenums';
 import { replaceFirstRouteParamWithVal } from './lib/utils/replaceFirstRouteParamWithVal';
 import { routeSplit } from './lib/utils/routeSplit';
 import { staticServer } from './lib/utils/serverstuff/staticServer';
+import { title404 } from './lib/utils/serverstuff/title404';
 import { getHandledRoutes } from './lib/utils/services/routeStorage';
 import { startScully } from './lib/utils/startup';
+export * as cliOptions from './lib/utils/cli-options';
 export * from './lib/utils/log';
+export * from './lib/utils/platform-server';
 export * from './lib/utils/procesmanager';
-export * from './lib/utils/platform-server'
+export { waitForIt } from './lib/utils/waitForIt';
+export { SPSRouteRenderer } from './lib/utils/platform-server';
 export {
+  title404,
+  routeRenderer,
   configValidator,
   ContentMetaData,
   ContentTextRoute,
@@ -51,7 +56,7 @@ export {
   updateScullyConfig,
   /** WIP part, those might be remove again in near future. */
   handleTravesal,
-  launchedBrowser$,
+  // launchedBrowser$,
   loadConfig,
   renderRoute,
   routeDiscovery,
@@ -63,3 +68,6 @@ export {
   prod,
   setPluginPriority,
 };
+
+
+

libs/scully/src/lib/renderPlugins/executePlugins.ts

@@ -1,10 +1,9 @@
 import { findPlugin } from '../pluginManagement/pluginConfig';
 import { registerPlugin, scullySystem } from '../pluginManagement/pluginRepository';
 import { HandledRoute } from '../routerPlugins/handledRoute.interface';
-import { scullyConfig } from '../utils/config';
+import { scullyConfig,routeRenderer } from '../utils/config';
 import { logError, yellow, logWarn } from '../utils/log';
 import { captureException } from '../utils/captureMessage';
-import { puppeteerRender } from './puppeteerRenderPlugin';
 import { toJSDOM, fromJSDOM } from './jsdomPlugins';
 import { JSDOM } from 'jsdom';
 import { postProcessByDomPlugin, postProcessByHtmlPlugin } from '../pluginManagement';
@@ -30,7 +29,7 @@ const executePluginsForRoute = async (route: HandledRoute) => {
     }
   }
   // this support different renders: puppeteer / imgRender / sps / others...
-  const InitialHTML = (await (route.renderPlugin ? findPlugin(route.renderPlugin) : findPlugin(puppeteerRender))(route)) as string;
+  const InitialHTML = (await (route.renderPlugin ? findPlugin(route.renderPlugin) : findPlugin(routeRenderer))(route)) as string;
 
   // split out jsDom vs string renderers.
   const { jsDomRenders, renders: stringRenders } = handlers.reduce(

libs/scully/src/lib/renderPlugins/index.ts

@@ -2,6 +2,5 @@ export * from './contentRenderPlugin';
 export * from './contentTextRenderPlugin';
 export * from './executePlugins';
 export * from './jsdomPlugins';
-export * from './launchedBrowser';
-export * from './puppeteerRenderPlugin';
 export * from './seoHrefCompletionPlugin';
+

libs/scully/src/lib/utils/asyncPool.ts

@@ -1,6 +1,6 @@
-import { waitForIt } from '../renderPlugins/puppeteerRenderPlugin';
-import { log, logWarn, printProgress } from './log';
 import { performance } from 'perf_hooks';
+import { logWarn, printProgress } from './log';
+import { waitForIt } from './waitForIt';
 
 const progressTime = 100;
 /**

libs/scully/src/lib/utils/config.ts

@@ -9,6 +9,7 @@ import { readAngularJson } from './read-angular-json';
 import { validateConfig } from './validateConfig';
 export const angularRoot = findAngularJsonPath();
 export const scullyConfig: ScullyConfig = {} as ScullyConfig;
+export const routeRenderer = 'routeRenderer' as const;
 
 export const scullyDefaults: Partial<ScullyConfig> = {
   bareProject: false,

libs/scully/src/lib/utils/handlers/renderParallel.ts

@@ -1,12 +1,12 @@
 import { performance } from 'perf_hooks';
 import { findPlugin } from '../../pluginManagement/pluginConfig';
 import { renderRoute } from '../../renderPlugins/executePlugins';
-import { waitForIt } from '../../renderPlugins/puppeteerRenderPlugin';
 import { WriteToStorage } from '../../systemPlugins/writeToFs.plugin';
 import { asyncPool } from '../asyncPool';
 import { scullyConfig } from '../config';
 import { logWarn } from '../log';
 import { performanceIds } from '../performanceIds';
+import { waitForIt } from '../waitForIt';
 
 const writeToFs = findPlugin(WriteToStorage);
 

libs/scully/src/lib/utils/handlers/renderPlugin.ts

@@ -1,17 +1,11 @@
 import { registerPlugin, scullySystem } from '../../pluginManagement';
-import { launchedBrowser } from '../../renderPlugins/launchedBrowser';
 import { HandledRoute } from '../../routerPlugins/handledRoute.interface';
-import { printProgress } from '../log';
 import { renderParallel } from './renderParallel';
 
 
 export const renderPlugin = 'renderPlugin' as const;
 registerPlugin(scullySystem, renderPlugin, defaultRenderPlugin);
 async function defaultRenderPlugin(handledRoutes: HandledRoute[]) {
-  /** update progress to show what's going on  */
-  printProgress(false, 'Starting puppeteer');
-  /** launch the browser, its shared among renderers */
-  await launchedBrowser();
   /** start handling each route, works in chunked parallel mode  */
   await renderParallel(handledRoutes);
 }