Sander/rework wait fix (scullyio#808)

* fix(scully): Add wait time before spawning new browser instance

If there are more than 500 routes rendered, when this limit is reached
a new browser instance is spawned. The async jobs rendering the
routes at that moment in the initial browser instance are not waited
and thus they are failing with exception and cause the process to stop.
Update and add testserver data for 500+ routes.
Update test server to hold and serve 501 users.
Update snapshots for the newly added users
Add test to check extra routes.

* fix(scully): rework time calculation to not take minutes and poll for outstanding tasks on browser r

* test(scully): upds test for user1&2 and snapshots

Co-authored-by: Septimiu Criste <[email protected]>

libs/scully/src/lib/renderPlugins/launchedBrowser.ts

@@ -1,6 +1,6 @@
 import { Browser, launch, LaunchOptions } from 'puppeteer';
-import { BehaviorSubject, from, merge, Observable, of, timer } from 'rxjs';
-import { filter, shareReplay, switchMap, take, tap, delayWhen, throttleTime } from 'rxjs/operators';
+import { BehaviorSubject, from, merge, Observable, of, timer, interval } from 'rxjs';
+import { filter, shareReplay, switchMap, take, tap, delayWhen, throttleTime, catchError } from 'rxjs/operators';
 import { showBrowser } from '../utils/cli-options';
 import { loadConfig, scullyConfig } from '../utils/config';
 import { green, log, logError } from '../utils/log';
@@ -30,6 +30,7 @@ export const launchedBrowser: () => Promise<Browser> = async () => {
 let browser: Browser;
 
 /** ICE relaunch puppeteer. */
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 export const reLaunch = (reason?: string): Promise<Browser> => {
   //   if (reason) {
   //     logWarn(
@@ -94,17 +95,27 @@ function obsBrowser(options: LaunchOptions = scullyConfig.puppeteerLaunchOptions
       .pipe(
         /** ignore request while the browser is already starting, we can only launch 1 */
         filter(() => !isLaunching),
-        // tap(() => log(green('relaunch cmd received'))),
-        /** the long trottletime is to cater for the concurrently running browsers to crash and burn. */
+        /** the long throttleTime is to cater for the concurrently running browsers to crash and burn. */
         throttleTime(15000),
         // provide enough time for the current async operations to finish before killing the current browser instance
-        delayWhen(() => {
-          if (!browser) {
-            return of(0);
-          }
-
-          return timer(15000);
-        })
+        delayWhen(() =>
+          merge(
+            /** timout at 25 seconds */
+            timer(25000),
+            /** or then the number of pages hits <=1  */
+            interval(500).pipe(
+              /** if the browser is active get the pages promise */
+              switchMap(() => (browser ? browser.pages() : of([]))),
+              /** only go ahead when there is <=1 pages (the browser itself) */
+              filter((p) => browser === undefined || p.length <= 1)
+            )
+          ).pipe(
+            /** use take 1 to make sure we complete when one of the above fires */
+            take(1),
+            /** if something _really_ unwieldy happens with the browser, ignore and go ahead */
+            catchError(() => of([]))
+          )
+        )
       )
       .subscribe({
         next: () => {

libs/scully/src/lib/utils/startup.ts

@@ -3,11 +3,12 @@ import { performance, PerformanceObserver, PerformanceObserverCallback } from 'p
 import { watch, ssl } from './cli-options';
 import { scullyConfig } from './config';
 import { generateAll } from './handlers/defaultAction';
-import { log, yellow, green } from './log';
+import { log, yellow, green, printProgress } from './log';
 import { performanceIds } from './performanceIds';
 import { reloadAll } from '../watchMode';
 import { findPlugin } from '../pluginManagement';
 
+let measurePerfPerf: number;
 /**
  * Starts the entire process
  * @param config:ScullyConfig
@@ -22,10 +23,15 @@ export const startScully = (url?: string) => {
     obs.observe({ entryTypes: ['measure'], buffered: true });
     const numberOfRoutesProm = findPlugin(generateAll)(url)
       .then((routes) => {
+        log(`measuring performance`);
+        measurePerfPerf = Date.now();
         performance.mark('stopDuration');
         /** measure all performance checks */
         try {
-          [...performanceIds.values()].forEach((id) => performance.measure(id, `start${id}`, `stop${id}`));
+          let i = performanceIds.size;
+          for (const id of performanceIds) {
+            performance.measure(id, `start${id}`, `stop${id}`);
+          }
         } catch (e) {
           console.error(e);
         }
@@ -68,18 +74,13 @@ ${yellow('------------------------------------------------------------')}`
 
 function measurePerformance(resolve: (value?: unknown) => void): PerformanceObserverCallback {
   return (list, observer) => {
-    const durations = list.getEntries().reduce(
-      (acc, entry) => ({
-        ...acc,
-        [entry.name]: Math.floor(entry.duration * 100) / 100,
-      }),
-      {}
-    );
+    const durations = Object.fromEntries(list.getEntries().map((entry) => [entry.name, Math.floor(entry.duration * 100) / 100]));
     // console.log(durations);
     performance.clearMarks();
     observer.disconnect();
     performanceIds.clear();
     resolve(durations);
+    log(`measuring performance took ${Math.floor((Date.now() - measurePerfPerf) * 100) / 100}Ms`);
   };
 }
 

package.json

@@ -29,7 +29,7 @@
     "debug": "nodemon --legacy-watch --watch ./dist/libs/scully --inspect=0.0.0.0:5858 --nolazy \"./dist/libs/scully/scully --pr sample-blog\"",
     "nozspredeb": "npm run build:scully",
     "nopreduration": "npx tsc -m commonjs scully.sample-blog.config.ts",
-    "duration": "rm scully.log && npm run scully -- --cf=scully.heavy.config.ts --tds --RSD --nl",
+    "duration": "rm scully.log && npm run scully -- --cf=scully.heavy.config.ts --tds --RSD ",
     "deb": "node --inspect=0.0.0.0:5858 --nolazy ./dist/libs/scully/scully --pr sample-blog",
     "deploy:docs": "echo No deploy script yet",
     "affected:apps": "nx affected:apps",

scully.heavy.config.ts

@@ -2,6 +2,7 @@ import { config as original } from './scully.sample-blog.config';
 
 export const config = Object.assign({}, original);
 
-config.routes['/demo/:id'].numberOfPages = 2500;
+config.routes['/demo/:id'].numberOfPages = 50000;
+config.routes['/user/:userId'].userId.resultsHandler = undefined;
 /** crank it up a notch, more threads seems more likely to fail. */
-config.maxRenderThreads = 64;
+config.maxRenderThreads = 32;

scully.sample-blog.config.ts

@@ -55,6 +55,7 @@ export const config: ScullyConfig = {
        */
       userId: {
         url: 'http://localhost:8200/users',
+        resultsHandler: (raw) => raw.filter((row) => row.id < 3),
         property: 'id',
       },
     },