formalize the image test spec

.gitignore

@@ -2,5 +2,7 @@
 
 /playwright/test-results/
 /playwright/.temp-report
-/playwright/test-report.txt
+/playwright/test-report.json
+/playwright/image-test.har
+/playwright/har-files/
 

playwright/makefile

@@ -0,0 +1,7 @@
+deps:
+	npm clean-install
+	npx playwright install-deps
+
+test-image:
+	bash ./test-image-table.sh
+

playwright/src/scripts/image.spec.ts

@@ -4,76 +4,73 @@
  * the time it took from the load of the first image until the last image...
  *
  */
-import { test, expect } from '@playwright/test';
+import { test } from '@playwright/test';
 import { performance } from 'perf_hooks';
-import { interval } from '../utils/helpers';
-import TestReporter from './../utils/reporter';
+import { interval, waitForImages, waitForRecordsetMainData } from '../utils/helpers';
+import { ImageTestReport, ImageTestReportService } from '../utils/image-reporter';
 
-let num = parseInt(process.env.TEST_COUNT!)
-if (isNaN(num)) {
-  num = 2;
+let reloadCount = parseInt(process.env.TEST_COUNT!);
+if (isNaN(reloadCount) || reloadCount <= 1) {
+  reloadCount = 1;
 }
+
 let pageSize = parseInt(process.env.PAGE_SIZE!)
 if (isNaN(pageSize)) {
   pageSize = 100;
 }
 
-let absoulteStartTime, startTime, endTime;
-
-
+let startTime, endTime;
 
+const report = new ImageTestReport();
 
 test('load image table and note the image load time', async ({ page }) => {
-  absoulteStartTime = performance.now();
-  startTime = absoulteStartTime;
-
-  console.log(`page size=${pageSize}`)
-  const url = [
-    'https://staging.atlas-d2k.org/chaise/recordset/#2/Gene_Expression:Image/',
-    '*::facets::N4IghgdgJiBcDaoDOB7ArgJwMYFM4gBUALNAWwCMIwBLAGwH0AhATwBcckQAaEDSAcw5xEIUtQj4AjAAYATABZuo8fRwAPLLTRJqANzyxWGNDgC+AXQumgA@sort(RID)',
-    '?limit=' + pageSize
-  ].join('');
-
-  await page.goto(url);
-  await page.locator('.recordset-table').waitFor({ state: 'visible' });
-  await page.locator('.recordest-main-spinner').waitFor({ state: 'detached' });
-
-  endTime = performance.now();
-  console.log(`first page load: ${interval(startTime, endTime)}`);
-  startTime = endTime;
-
-  /**
-   * wait for thumbnails to load
-   * https://github.com/microsoft/playwright/issues/6046
-   */
-  await page.waitForFunction(() => {
-    const images = Array.from(document.querySelectorAll('img'));
-    return images.every(img => img.complete && img.naturalWidth !== 0);
-  });
+  // disable cache
+  page.route('**', route => route.continue());
 
-  endTime = performance.now();
-  console.log(`first page images: ${interval(startTime, endTime)}`);
+  await test.step('go to recordset page', async () => {
+    console.log(`page size=${pageSize}, reload count=${reloadCount}`);
 
-  await page.locator('.chaise-table-next-btn').click();
-  await page.locator('.recordest-main-spinner').waitFor({ state: 'visible' });
-  await page.locator('.recordest-main-spinner').waitFor({ state: 'detached' });
+    const url = [
+      'https://dev.atlas-d2k.org/chaise/recordset/#2/Gene_Expression:Image/',
+      '*::facets::N4IghgdgJiBcDaoDOB7ArgJwMYFM4gBUALNAWwCMIwBLAGwH0AhATwBcckQAaEDSAcw5xEIUtQj4AjAAYATABZuo8fRwAPLLTRJqANzyxWGNDgC+AXQumgA@sort(RID)',
+      '?limit=' + pageSize
+    ].join('');
 
-  endTime = performance.now();
-  console.log(`second page load: ${interval(startTime, endTime)}`);
-  startTime = endTime;
+    await page.goto(url);
+    await waitForRecordsetMainData(page);
+  });
 
+  for (let i = 0; i < reloadCount; i++) {
+    await test.step(`${i}: page load`, async () => {
 
-  await page.waitForFunction(() => {
-    const images = Array.from(document.querySelectorAll('img'));
-    return images.every(img => img.complete && img.naturalWidth !== 0);
-  });
+      startTime = performance.now();
+      await page.reload();
+      await waitForRecordsetMainData(page);
 
-  endTime = performance.now();
-  console.log(`second page images: ${interval(startTime, endTime)}`);
+      endTime = performance.now();
+      report.page_load.push(interval(startTime, endTime));
+      startTime = endTime;
+    });
 
-  console.log(`full runtime: ${interval(absoulteStartTime, endTime)}`);
+    await test.step(`${i}: all images load`, async () => {
+      await waitForImages(page);
 
-});
+      endTime = performance.now();
+      report.all_images_load.push(interval(startTime, endTime));
+      startTime = endTime;
 
+      if (i === reloadCount - 1) {
+        ImageTestReportService.saveCurrentReport(report);
+      }
+    });
+  }
+});
 
 
+/**
+ * do peak hour and then non-peak hour
+ *
+ * most probably the raw data should have the start time, ellapsed time, etc...
+ *
+ * we will run this in multiple time of the day and each will be marked whether it's peak or not.
+ */

playwright/src/utils/helpers.ts

@@ -1,3 +1,21 @@
 export function interval(start, end) {
   return ((end - start) / 1000).toFixed(3);
 }
+
+
+export const waitForImages = async (page) => {
+  /**
+   * wait for thumbnails to load
+   * https://github.com/microsoft/playwright/issues/6046
+   */
+  await page.waitForFunction(() => {
+    const images = Array.from(document.querySelectorAll('img'));
+    return images.every(img => img.complete && img.naturalWidth !== 0);
+  });
+};
+
+export const waitForRecordsetMainData = async (page) => {
+  await page.locator('.recordset-table').waitFor({ state: 'visible' });
+  await page.locator('.recordest-main-spinner').waitFor({ state: 'detached' });
+}
+

playwright/src/utils/image-reporter.ts

@@ -0,0 +1,99 @@
+import { min, max, mean, median, quantile } from 'simple-statistics'
+import fs from 'fs';
+
+export class ImageTestReport {
+  iso_time: string;
+  utc_hour: string;
+  public page_load: string[] = [];
+  all_images_load: string[] = [];
+
+  image_load_time: number[] = [];
+  image_size: number[] = [];
+
+  has_server_cache: boolean = false;
+  has_browesr_cache: boolean = false;
+
+  constructor() {
+    const date = new Date();
+    this.utc_hour = date.getUTCHours().toString();
+    this.iso_time = date.toISOString();
+  }
+
+  toJSON() {
+    return {
+      iso_time: this.iso_time,
+      utc_hour: this.utc_hour,
+      page_load: this.page_load,
+      all_images_load: this.all_images_load,
+      image_load_time: this.image_load_time,
+      image_size: this.image_size
+    }
+  }
+}
+
+export class ImageTestReportService {
+  private static CURR_REPORT_FILE_LOCATION = '.temp-report';
+  private static FULL_REPORT_FILE_LOCATION = 'test-report.json';
+
+  static saveCurrentReport(obj: ImageTestReport) {
+    fs.writeFileSync('.temp-report', JSON.stringify(obj.toJSON()));
+  }
+
+  static getCurrentReport() : ImageTestReport | undefined {
+    try {
+      const rep = fs.readFileSync(ImageTestReportService.CURR_REPORT_FILE_LOCATION, 'utf8');
+      return JSON.parse(rep);
+    } catch (exp) {
+      console.log('there is no current report');
+      return undefined;
+    }
+  }
+
+  static removeCurrentReport() {
+    try {
+      fs.unlinkSync(ImageTestReportService.CURR_REPORT_FILE_LOCATION);
+    } catch (exp) { }
+  }
+
+  static addToFullReport(obj: ImageTestReport) {
+    let fullRep : ImageTestReport[];
+    try {
+      const f = fs.readFileSync(ImageTestReportService.FULL_REPORT_FILE_LOCATION, 'utf8');
+      fullRep = JSON.parse(f);
+    } catch (exp) {
+      fullRep = [];
+    }
+
+    fullRep.push(obj);
+
+    const content = JSON.stringify(fullRep, undefined, 2);
+    fs.writeFileSync(ImageTestReportService.FULL_REPORT_FILE_LOCATION, content);
+  }
+
+  static createSummary() {
+    let fullRep : any[];
+    try {
+      const f = fs.readFileSync(ImageTestReportService.FULL_REPORT_FILE_LOCATION, 'utf8');
+      fullRep = JSON.parse(f);
+    } catch (exp) {
+      console.log('report file is either missing or invalid.');
+      return;
+    }
+
+    // fullRep.forEach(())
+  }
+
+  private static _createStepReport = (name, inp) => {
+    const arr = inp.map((val) => parseFloat(val));
+    return [
+      `${name}:`,
+      `count: ............................... ${arr.length}`,
+      `min: ................................. ${min(arr).toFixed(3)}`,
+      `max: ................................. ${max(arr).toFixed(3)}`,
+      `mean: ................................ ${mean(arr).toFixed(3)}`,
+      `median: .............................. ${median(arr).toFixed(3)}`,
+      `p95: ................................. ${quantile(arr, .95).toFixed(3)}`,
+      `p99: ................................. ${quantile(arr, .99).toFixed(3)}`,
+    ].join('\n')
+  }
+}

playwright/src/utils/image.teardown.ts

@@ -1,36 +1,57 @@
 import { type FullConfig } from '@playwright/test';
 import path from 'path';
 import { readFileSync, unlinkSync } from 'fs';
+import { ImageTestReportService } from './image-reporter';
 
 export default async function globalSetup(config: FullConfig) {
 
   try {
-    const filePath = path.join(__dirname, '..', '..', 'image-test.har');
-    const data = readFileSync(filePath);
+    const harFilePath = path.join(__dirname, '..', '..', 'image-test.har');
+    const data = readFileSync(harFilePath);
     const harContent: any = JSON.parse(data.toString());
 
-    const browserCached: any[] = [];
-    const serverCached: any[] = [];
+    let has_browesr_cache = false;
+    let has_server_cache = false;
+
+    let isReload = false;
+    const image_load: any[] = [];
+    const image_size: any[] = [];
     harContent.log.entries.forEach((e: any) => {
       if (e.response && e.response.status === 304) {
-        browserCached.push(e.startedDateTime);
+        has_browesr_cache = true;
       }
 
       if (e.response && e.response.bodySize === 0) {
-        serverCached.push(e.startedDateTime);
+        has_server_cache = true;
+      }
+
+      if (e.request && e.request.method === 'GET') {
+        if (isReload && e.request.url.startsWith('https://dev.atlas-d2k.org/hatrac/resources/gene_expression') && e.time !== -1) {
+          image_load.push(e.time);
+          image_size.push(e.response.bodySize);
+        }
+
+        if (!isReload && e.request.url.startsWith('https://dev.atlas-d2k.org/ermrest/catalog/2/attributegroup/M:=Gene_Expression:Image/Thumbnail_Bytes::gt::1024')) {
+          isReload = true;
+        }
       }
     })
 
-    console.log('browser cached:');
-    console.log(browserCached);
-    console.log('server cached:');
-    console.log(serverCached);
 
+    // save to full report
+    const rep = ImageTestReportService.getCurrentReport();
+    if (rep) {
+      rep.has_browesr_cache = has_browesr_cache;
+      rep.has_server_cache = has_server_cache;
 
-    // TODO find the time 
+      rep.image_load_time = image_load;
+      rep.image_size = image_size;
 
+      ImageTestReportService.addToFullReport(rep);
+      ImageTestReportService.removeCurrentReport();
+    }
 
-    unlinkSync(filePath);
+    // unlinkSync(harFilePath);
 
   } catch (exp) {
     console.log('something went wrong')

playwright/src/utils/reporter.ts

@@ -25,7 +25,7 @@ export default class TestReporter {
       return JSON.parse(rep);
     } catch (exp) {
       let res = {};
-      TestReporter.reportKeys.forEach((k) => res[k] = []);
+      TestReporter.reportKeys.forEach((k) => { res[k] = [] });
       return res;
     }
   }
@@ -34,6 +34,7 @@ export default class TestReporter {
     const arr = inp.map((val) => parseFloat(val));
     return [
       `${name}:`,
+      `count: ............................... ${arr.length}`,
       `min: ................................. ${min(arr).toFixed(3)}`,
       `max: ................................. ${max(arr).toFixed(3)}`,
       `mean: ................................ ${mean(arr).toFixed(3)}`,
@@ -55,10 +56,10 @@ export default class TestReporter {
   static createFullReportFile = () => {
     const val = TestReporter._getReportObject();
 
-    fs.unlinkSync(TestReporter.TEMP_FILE_LOCATION);
+    // fs.unlinkSync(TestReporter.TEMP_FILE_LOCATION);
 
     const content = TestReporter.reportKeys.map((k) => {
-      TestReporter._createStepReport(k, val[k])
+      return TestReporter._createStepReport(k, val[k])
     }).join('\n\n');
 
     console.log(content);

playwright/test-image-table.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/bash
+
+TEST_COUNT=2 PAGE_SIZE=100 npx playwright test image.spec.ts