From 1274ca29ec4651035ba133a0f8a606f6d2f17fee Mon Sep 17 00:00:00 2001 From: Aref Shafaei Date: Thu, 14 Mar 2024 10:59:45 -0700 Subject: [PATCH] formalize the image test spec --- .gitignore | 4 +- playwright/makefile | 7 ++ playwright/src/scripts/image.spec.ts | 99 +++++++++++++------------- playwright/src/utils/helpers.ts | 18 +++++ playwright/src/utils/image-reporter.ts | 99 ++++++++++++++++++++++++++ playwright/src/utils/image.teardown.ts | 45 ++++++++---- playwright/src/utils/reporter.ts | 7 +- playwright/test-image-table.sh | 3 + 8 files changed, 215 insertions(+), 67 deletions(-) create mode 100644 playwright/makefile create mode 100644 playwright/src/utils/image-reporter.ts create mode 100755 playwright/test-image-table.sh diff --git a/.gitignore b/.gitignore index 9d67180..d996890 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,7 @@ /playwright/test-results/ /playwright/.temp-report -/playwright/test-report.txt +/playwright/test-report.json +/playwright/image-test.har +/playwright/har-files/ diff --git a/playwright/makefile b/playwright/makefile new file mode 100644 index 0000000..40fc4db --- /dev/null +++ b/playwright/makefile @@ -0,0 +1,7 @@ +deps: + npm clean-install + npx playwright install-deps + +test-image: + bash ./test-image-table.sh + diff --git a/playwright/src/scripts/image.spec.ts b/playwright/src/scripts/image.spec.ts index 7f81fb6..38be08f 100644 --- a/playwright/src/scripts/image.spec.ts +++ b/playwright/src/scripts/image.spec.ts @@ -4,76 +4,73 @@ * the time it took from the load of the first image until the last image... * */ -import { test, expect } from '@playwright/test'; +import { test } from '@playwright/test'; import { performance } from 'perf_hooks'; -import { interval } from '../utils/helpers'; -import TestReporter from './../utils/reporter'; +import { interval, waitForImages, waitForRecordsetMainData } from '../utils/helpers'; +import { ImageTestReport, ImageTestReportService } from '../utils/image-reporter'; -let num = parseInt(process.env.TEST_COUNT!) -if (isNaN(num)) { - num = 2; +let reloadCount = parseInt(process.env.TEST_COUNT!); +if (isNaN(reloadCount) || reloadCount <= 1) { + reloadCount = 1; } + let pageSize = parseInt(process.env.PAGE_SIZE!) if (isNaN(pageSize)) { pageSize = 100; } -let absoulteStartTime, startTime, endTime; - - +let startTime, endTime; +const report = new ImageTestReport(); test('load image table and note the image load time', async ({ page }) => { - absoulteStartTime = performance.now(); - startTime = absoulteStartTime; - - console.log(`page size=${pageSize}`) - const url = [ - 'https://staging.atlas-d2k.org/chaise/recordset/#2/Gene_Expression:Image/', - '*::facets::N4IghgdgJiBcDaoDOB7ArgJwMYFM4gBUALNAWwCMIwBLAGwH0AhATwBcckQAaEDSAcw5xEIUtQj4AjAAYATABZuo8fRwAPLLTRJqANzyxWGNDgC+AXQumgA@sort(RID)', - '?limit=' + pageSize - ].join(''); - - await page.goto(url); - await page.locator('.recordset-table').waitFor({ state: 'visible' }); - await page.locator('.recordest-main-spinner').waitFor({ state: 'detached' }); - - endTime = performance.now(); - console.log(`first page load: ${interval(startTime, endTime)}`); - startTime = endTime; - - /** - * wait for thumbnails to load - * https://github.com/microsoft/playwright/issues/6046 - */ - await page.waitForFunction(() => { - const images = Array.from(document.querySelectorAll('img')); - return images.every(img => img.complete && img.naturalWidth !== 0); - }); + // disable cache + page.route('**', route => route.continue()); - endTime = performance.now(); - console.log(`first page images: ${interval(startTime, endTime)}`); + await test.step('go to recordset page', async () => { + console.log(`page size=${pageSize}, reload count=${reloadCount}`); - await page.locator('.chaise-table-next-btn').click(); - await page.locator('.recordest-main-spinner').waitFor({ state: 'visible' }); - await page.locator('.recordest-main-spinner').waitFor({ state: 'detached' }); + const url = [ + 'https://dev.atlas-d2k.org/chaise/recordset/#2/Gene_Expression:Image/', + '*::facets::N4IghgdgJiBcDaoDOB7ArgJwMYFM4gBUALNAWwCMIwBLAGwH0AhATwBcckQAaEDSAcw5xEIUtQj4AjAAYATABZuo8fRwAPLLTRJqANzyxWGNDgC+AXQumgA@sort(RID)', + '?limit=' + pageSize + ].join(''); - endTime = performance.now(); - console.log(`second page load: ${interval(startTime, endTime)}`); - startTime = endTime; + await page.goto(url); + await waitForRecordsetMainData(page); + }); + for (let i = 0; i < reloadCount; i++) { + await test.step(`${i}: page load`, async () => { - await page.waitForFunction(() => { - const images = Array.from(document.querySelectorAll('img')); - return images.every(img => img.complete && img.naturalWidth !== 0); - }); + startTime = performance.now(); + await page.reload(); + await waitForRecordsetMainData(page); - endTime = performance.now(); - console.log(`second page images: ${interval(startTime, endTime)}`); + endTime = performance.now(); + report.page_load.push(interval(startTime, endTime)); + startTime = endTime; + }); - console.log(`full runtime: ${interval(absoulteStartTime, endTime)}`); + await test.step(`${i}: all images load`, async () => { + await waitForImages(page); -}); + endTime = performance.now(); + report.all_images_load.push(interval(startTime, endTime)); + startTime = endTime; + if (i === reloadCount - 1) { + ImageTestReportService.saveCurrentReport(report); + } + }); + } +}); +/** + * do peak hour and then non-peak hour + * + * most probably the raw data should have the start time, ellapsed time, etc... + * + * we will run this in multiple time of the day and each will be marked whether it's peak or not. + */ diff --git a/playwright/src/utils/helpers.ts b/playwright/src/utils/helpers.ts index 72674c7..153ca4a 100644 --- a/playwright/src/utils/helpers.ts +++ b/playwright/src/utils/helpers.ts @@ -1,3 +1,21 @@ export function interval(start, end) { return ((end - start) / 1000).toFixed(3); } + + +export const waitForImages = async (page) => { + /** + * wait for thumbnails to load + * https://github.com/microsoft/playwright/issues/6046 + */ + await page.waitForFunction(() => { + const images = Array.from(document.querySelectorAll('img')); + return images.every(img => img.complete && img.naturalWidth !== 0); + }); +}; + +export const waitForRecordsetMainData = async (page) => { + await page.locator('.recordset-table').waitFor({ state: 'visible' }); + await page.locator('.recordest-main-spinner').waitFor({ state: 'detached' }); +} + diff --git a/playwright/src/utils/image-reporter.ts b/playwright/src/utils/image-reporter.ts new file mode 100644 index 0000000..9b6d332 --- /dev/null +++ b/playwright/src/utils/image-reporter.ts @@ -0,0 +1,99 @@ +import { min, max, mean, median, quantile } from 'simple-statistics' +import fs from 'fs'; + +export class ImageTestReport { + iso_time: string; + utc_hour: string; + public page_load: string[] = []; + all_images_load: string[] = []; + + image_load_time: number[] = []; + image_size: number[] = []; + + has_server_cache: boolean = false; + has_browesr_cache: boolean = false; + + constructor() { + const date = new Date(); + this.utc_hour = date.getUTCHours().toString(); + this.iso_time = date.toISOString(); + } + + toJSON() { + return { + iso_time: this.iso_time, + utc_hour: this.utc_hour, + page_load: this.page_load, + all_images_load: this.all_images_load, + image_load_time: this.image_load_time, + image_size: this.image_size + } + } +} + +export class ImageTestReportService { + private static CURR_REPORT_FILE_LOCATION = '.temp-report'; + private static FULL_REPORT_FILE_LOCATION = 'test-report.json'; + + static saveCurrentReport(obj: ImageTestReport) { + fs.writeFileSync('.temp-report', JSON.stringify(obj.toJSON())); + } + + static getCurrentReport() : ImageTestReport | undefined { + try { + const rep = fs.readFileSync(ImageTestReportService.CURR_REPORT_FILE_LOCATION, 'utf8'); + return JSON.parse(rep); + } catch (exp) { + console.log('there is no current report'); + return undefined; + } + } + + static removeCurrentReport() { + try { + fs.unlinkSync(ImageTestReportService.CURR_REPORT_FILE_LOCATION); + } catch (exp) { } + } + + static addToFullReport(obj: ImageTestReport) { + let fullRep : ImageTestReport[]; + try { + const f = fs.readFileSync(ImageTestReportService.FULL_REPORT_FILE_LOCATION, 'utf8'); + fullRep = JSON.parse(f); + } catch (exp) { + fullRep = []; + } + + fullRep.push(obj); + + const content = JSON.stringify(fullRep, undefined, 2); + fs.writeFileSync(ImageTestReportService.FULL_REPORT_FILE_LOCATION, content); + } + + static createSummary() { + let fullRep : any[]; + try { + const f = fs.readFileSync(ImageTestReportService.FULL_REPORT_FILE_LOCATION, 'utf8'); + fullRep = JSON.parse(f); + } catch (exp) { + console.log('report file is either missing or invalid.'); + return; + } + + // fullRep.forEach(()) + } + + private static _createStepReport = (name, inp) => { + const arr = inp.map((val) => parseFloat(val)); + return [ + `${name}:`, + `count: ............................... ${arr.length}`, + `min: ................................. ${min(arr).toFixed(3)}`, + `max: ................................. ${max(arr).toFixed(3)}`, + `mean: ................................ ${mean(arr).toFixed(3)}`, + `median: .............................. ${median(arr).toFixed(3)}`, + `p95: ................................. ${quantile(arr, .95).toFixed(3)}`, + `p99: ................................. ${quantile(arr, .99).toFixed(3)}`, + ].join('\n') + } +} diff --git a/playwright/src/utils/image.teardown.ts b/playwright/src/utils/image.teardown.ts index a989cbf..9f2a1ee 100644 --- a/playwright/src/utils/image.teardown.ts +++ b/playwright/src/utils/image.teardown.ts @@ -1,36 +1,57 @@ import { type FullConfig } from '@playwright/test'; import path from 'path'; import { readFileSync, unlinkSync } from 'fs'; +import { ImageTestReportService } from './image-reporter'; export default async function globalSetup(config: FullConfig) { try { - const filePath = path.join(__dirname, '..', '..', 'image-test.har'); - const data = readFileSync(filePath); + const harFilePath = path.join(__dirname, '..', '..', 'image-test.har'); + const data = readFileSync(harFilePath); const harContent: any = JSON.parse(data.toString()); - const browserCached: any[] = []; - const serverCached: any[] = []; + let has_browesr_cache = false; + let has_server_cache = false; + + let isReload = false; + const image_load: any[] = []; + const image_size: any[] = []; harContent.log.entries.forEach((e: any) => { if (e.response && e.response.status === 304) { - browserCached.push(e.startedDateTime); + has_browesr_cache = true; } if (e.response && e.response.bodySize === 0) { - serverCached.push(e.startedDateTime); + has_server_cache = true; + } + + if (e.request && e.request.method === 'GET') { + if (isReload && e.request.url.startsWith('https://dev.atlas-d2k.org/hatrac/resources/gene_expression') && e.time !== -1) { + image_load.push(e.time); + image_size.push(e.response.bodySize); + } + + if (!isReload && e.request.url.startsWith('https://dev.atlas-d2k.org/ermrest/catalog/2/attributegroup/M:=Gene_Expression:Image/Thumbnail_Bytes::gt::1024')) { + isReload = true; + } } }) - console.log('browser cached:'); - console.log(browserCached); - console.log('server cached:'); - console.log(serverCached); + // save to full report + const rep = ImageTestReportService.getCurrentReport(); + if (rep) { + rep.has_browesr_cache = has_browesr_cache; + rep.has_server_cache = has_server_cache; - // TODO find the time + rep.image_load_time = image_load; + rep.image_size = image_size; + ImageTestReportService.addToFullReport(rep); + ImageTestReportService.removeCurrentReport(); + } - unlinkSync(filePath); + // unlinkSync(harFilePath); } catch (exp) { console.log('something went wrong') diff --git a/playwright/src/utils/reporter.ts b/playwright/src/utils/reporter.ts index 49f6163..0f99cb5 100644 --- a/playwright/src/utils/reporter.ts +++ b/playwright/src/utils/reporter.ts @@ -25,7 +25,7 @@ export default class TestReporter { return JSON.parse(rep); } catch (exp) { let res = {}; - TestReporter.reportKeys.forEach((k) => res[k] = []); + TestReporter.reportKeys.forEach((k) => { res[k] = [] }); return res; } } @@ -34,6 +34,7 @@ export default class TestReporter { const arr = inp.map((val) => parseFloat(val)); return [ `${name}:`, + `count: ............................... ${arr.length}`, `min: ................................. ${min(arr).toFixed(3)}`, `max: ................................. ${max(arr).toFixed(3)}`, `mean: ................................ ${mean(arr).toFixed(3)}`, @@ -55,10 +56,10 @@ export default class TestReporter { static createFullReportFile = () => { const val = TestReporter._getReportObject(); - fs.unlinkSync(TestReporter.TEMP_FILE_LOCATION); + // fs.unlinkSync(TestReporter.TEMP_FILE_LOCATION); const content = TestReporter.reportKeys.map((k) => { - TestReporter._createStepReport(k, val[k]) + return TestReporter._createStepReport(k, val[k]) }).join('\n\n'); console.log(content); diff --git a/playwright/test-image-table.sh b/playwright/test-image-table.sh new file mode 100755 index 0000000..73cc764 --- /dev/null +++ b/playwright/test-image-table.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash + +TEST_COUNT=2 PAGE_SIZE=100 npx playwright test image.spec.ts