From 5dd49bc5e6151bf098bf451d2d2e6488efa92f40 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Wed, 19 Jun 2024 11:16:17 -0300 Subject: [PATCH] WIP Generate reports on private API usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (and upload them) TODO if this is going to go into main, then it needs to handle hooks and test definition still TODO remove the exceptions and then put them on the other branch Resolves ECO-4834. update static context to handle all stuff on `main` TODO why are the only hook things now coming from push test? ah i think it’s because of exclusions further considering hooks for private API usage hook check further --- .github/workflows/test-browser.yml | 6 +- .github/workflows/test-node.yml | 6 +- .gitignore | 1 + package-lock.json | 76 +++++++++++ package.json | 6 +- scripts/processPrivateApiData/csv.ts | 121 ++++++++++++++++++ scripts/processPrivateApiData/dto.ts | 53 ++++++++ scripts/processPrivateApiData/exclusions.ts | 40 ++++++ scripts/processPrivateApiData/grouping.ts | 49 +++++++ scripts/processPrivateApiData/load.ts | 16 +++ scripts/processPrivateApiData/run.ts | 49 +++++++ .../processPrivateApiData/runtimeContext.ts | 30 +++++ .../processPrivateApiData/staticContext.ts | 115 +++++++++++++++++ scripts/processPrivateApiData/utils.ts | 45 +++++++ .../withoutPrivateAPIUsage.ts | 71 ++++++++++ 15 files changed, 680 insertions(+), 4 deletions(-) create mode 100644 scripts/processPrivateApiData/csv.ts create mode 100644 scripts/processPrivateApiData/dto.ts create mode 100644 scripts/processPrivateApiData/exclusions.ts create mode 100644 scripts/processPrivateApiData/grouping.ts create mode 100644 scripts/processPrivateApiData/load.ts create mode 100644 scripts/processPrivateApiData/run.ts create mode 100644 scripts/processPrivateApiData/runtimeContext.ts create mode 100644 scripts/processPrivateApiData/staticContext.ts create mode 100644 scripts/processPrivateApiData/utils.ts create mode 100644 scripts/processPrivateApiData/withoutPrivateAPIUsage.ts diff --git a/.github/workflows/test-browser.yml b/.github/workflows/test-browser.yml index 5cb471fd1..9bc9360df 100644 --- a/.github/workflows/test-browser.yml +++ b/.github/workflows/test-browser.yml @@ -30,11 +30,15 @@ jobs: - env: PLAYWRIGHT_BROWSER: ${{ matrix.browser }} run: npm run test:playwright + - name: Generate private API usage reports + run: npm run process-private-api-data private-api-usage/*.json - name: Save private API usage data uses: actions/upload-artifact@v4 with: name: private-api-usage-${{ matrix.browser }} - path: private-api-usage + path: | + private-api-usage + private-api-usage-reports - name: Upload test results if: always() uses: ably/test-observability-action@v1 diff --git a/.github/workflows/test-node.yml b/.github/workflows/test-node.yml index db3ba7b7d..d74d0b3db 100644 --- a/.github/workflows/test-node.yml +++ b/.github/workflows/test-node.yml @@ -28,11 +28,15 @@ jobs: - run: npm run test:node env: CI: true + - name: Generate private API usage reports + run: npm run process-private-api-data private-api-usage/*.json - name: Save private API usage data uses: actions/upload-artifact@v4 with: name: private-api-usage-${{ matrix.node-version }} - path: private-api-usage + path: | + private-api-usage + private-api-usage-reports - name: Upload test results if: always() uses: ably/test-observability-action@v1 diff --git a/.gitignore b/.gitignore index 3a3066ac2..f98f1bc7b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ react/ typedoc/generated/ junit/ private-api-usage/ +private-api-usage-reports/ test/support/mocha_junit_reporter/build/ diff --git a/package-lock.json b/package-lock.json index 2cee606de..e587bde1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "chai": "^4.2.0", "cli-table": "^0.3.11", "cors": "^2.8.5", + "csv": "^6.3.9", "esbuild": "^0.18.10", "esbuild-plugin-umd-wrapper": "ably-forks/esbuild-plugin-umd-wrapper#1.0.7-optional-amd-named-module", "esbuild-runner": "^2.2.2", @@ -3048,6 +3049,39 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "dev": true }, + "node_modules/csv": { + "version": "6.3.9", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.9.tgz", + "integrity": "sha512-eiN+Qu8NwSLxZYia6WzB8xlX/rAQ/8EgK5A4dIF7Bz96mzcr5dW1jlcNmjG0QWySWKfPdCerH3RQ96ZqqsE8cA==", + "dev": true, + "dependencies": { + "csv-generate": "^4.4.1", + "csv-parse": "^5.5.6", + "csv-stringify": "^6.5.0", + "stream-transform": "^3.3.2" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.4.1.tgz", + "integrity": "sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg==", + "dev": true + }, + "node_modules/csv-parse": { + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.6.tgz", + "integrity": "sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==", + "dev": true + }, + "node_modules/csv-stringify": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.0.tgz", + "integrity": "sha512-edlXFVKcUx7r8Vx5zQucsuMg4wb/xT6qyz+Sr1vnLrdXqlLD1+UKyWNyZ9zn6mUW1ewmGxrpVwAcChGF0HQ/2Q==", + "dev": true + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -8986,6 +9020,12 @@ "readable-stream": "^3.5.0" } }, + "node_modules/stream-transform": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.2.tgz", + "integrity": "sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ==", + "dev": true + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -12932,6 +12972,36 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "dev": true }, + "csv": { + "version": "6.3.9", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.9.tgz", + "integrity": "sha512-eiN+Qu8NwSLxZYia6WzB8xlX/rAQ/8EgK5A4dIF7Bz96mzcr5dW1jlcNmjG0QWySWKfPdCerH3RQ96ZqqsE8cA==", + "dev": true, + "requires": { + "csv-generate": "^4.4.1", + "csv-parse": "^5.5.6", + "csv-stringify": "^6.5.0", + "stream-transform": "^3.3.2" + } + }, + "csv-generate": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.4.1.tgz", + "integrity": "sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg==", + "dev": true + }, + "csv-parse": { + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.6.tgz", + "integrity": "sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==", + "dev": true + }, + "csv-stringify": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.0.tgz", + "integrity": "sha512-edlXFVKcUx7r8Vx5zQucsuMg4wb/xT6qyz+Sr1vnLrdXqlLD1+UKyWNyZ9zn6mUW1ewmGxrpVwAcChGF0HQ/2Q==", + "dev": true + }, "data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -17286,6 +17356,12 @@ "readable-stream": "^3.5.0" } }, + "stream-transform": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.2.tgz", + "integrity": "sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ==", + "dev": true + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/package.json b/package.json index dc41b0465..da6e53714 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "chai": "^4.2.0", "cli-table": "^0.3.11", "cors": "^2.8.5", + "csv": "^6.3.9", "esbuild": "^0.18.10", "esbuild-plugin-umd-wrapper": "ably-forks/esbuild-plugin-umd-wrapper#1.0.7-optional-amd-named-module", "esbuild-runner": "^2.2.2", @@ -154,10 +155,11 @@ "lint": "eslint .", "lint:fix": "eslint --fix .", "prepare": "npm run build", - "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt", - "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt", + "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt", + "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt", "sourcemap": "source-map-explorer build/ably.min.js", "modulereport": "tsc --noEmit --esModuleInterop scripts/moduleReport.ts && esr scripts/moduleReport.ts", + "process-private-api-data": "tsc --noEmit --esModuleInterop --strictNullChecks scripts/processPrivateApiData/run.ts && esr scripts/processPrivateApiData/run.ts", "docs": "typedoc" } } diff --git a/scripts/processPrivateApiData/csv.ts b/scripts/processPrivateApiData/csv.ts new file mode 100644 index 000000000..fcb7fb563 --- /dev/null +++ b/scripts/processPrivateApiData/csv.ts @@ -0,0 +1,121 @@ +import { stringify as csvStringify } from 'csv-stringify/sync'; +import { writeFileSync, existsSync, mkdirSync } from 'fs'; +import { PrivateApiContextDto, TestStartRecord } from './dto'; +import { ContextGroup } from './grouping'; +import { RuntimeContext } from './runtimeContext'; +import { StaticContext } from './staticContext'; +import path from 'path'; + +function suiteAtLevelForCSV(suites: string[] | null, level: number) { + return suites?.[level] ?? ''; +} + +function suitesColumnsForCSV(suites: string[] | null, maxSuiteLevel: number) { + const result: string[] = []; + for (let i = 0; i < maxSuiteLevel; i++) { + result.push(suiteAtLevelForCSV(suites, i)); + } + return result; +} + +function suitesHeaders(maxSuiteLevel: number) { + const result: string[] = []; + for (let i = 0; i < maxSuiteLevel; i++) { + result.push(`Suite (level ${i + 1})`); + } + return result; +} + +function commonHeaders(maxSuiteLevel: number) { + return ['File', ...suitesHeaders(maxSuiteLevel), 'Description'].filter((val) => val !== null) as string[]; +} + +function writeCSVData(rows: string[][], name: string) { + const outputDirectoryPath = path.join(__dirname, '..', '..', 'private-api-usage-reports'); + + if (!existsSync(outputDirectoryPath)) { + mkdirSync(outputDirectoryPath); + } + + const result = csvStringify(rows); + writeFileSync(path.join(outputDirectoryPath, `${name}.csv`), result); +} + +export function writeRuntimePrivateAPIUsageCSV(contextGroups: ContextGroup[]) { + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array + const maxSuiteLevel = contextGroups + .map((group) => (group.key.suite ?? []).length) + .reduce((a, b) => Math.max(a, b), -Infinity); + + const columnHeaders = [ + 'Context', + ...commonHeaders(maxSuiteLevel), + 'Via parameterised test helper', + 'Via misc. helpers', + 'Private API called', + ]; + + const csvRows = contextGroups + .map((contextGroup) => { + const runtimeContext = contextGroup.key; + + const contextColumns = [ + runtimeContext.type, + runtimeContext.file ?? '', + ...suitesColumnsForCSV(runtimeContext.suite, maxSuiteLevel), + runtimeContext.type === 'definition' ? runtimeContext.label : runtimeContext.title, + ]; + + return contextGroup.usages.map((usage) => [ + ...contextColumns, + (usage.context.type === 'test' ? usage.context.parameterisedTestTitle : null) ?? '', + [...usage.context.helperStack].reverse().join(' -> '), + usage.privateAPIIdentifier, + ]); + }) + .flat(); + + writeCSVData([columnHeaders, ...csvRows], 'runtime-private-api-usage'); +} + +export function writeStaticPrivateAPIUsageCSV(contextGroups: ContextGroup[]) { + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array + const maxSuiteLevel = contextGroups + .map((group) => ('suite' in group.key ? group.key.suite : []).length) + .reduce((a, b) => Math.max(a, b), -Infinity); + + const columnHeaders = ['Context', ...commonHeaders(maxSuiteLevel), 'Private API called']; + + const csvRows = contextGroups + .map((contextGroup) => { + const staticContext = contextGroup.key; + + const contextColumns = [ + staticContext.type, + 'file' in staticContext ? staticContext.file : '', + ...suitesColumnsForCSV('suite' in staticContext ? staticContext.suite : null, maxSuiteLevel), + staticContext.title, + ]; + + return contextGroup.usages.map((usage) => [...contextColumns, usage.privateAPIIdentifier]); + }) + .flat(); + + writeCSVData([columnHeaders, ...csvRows], 'static-private-api-usage'); +} + +export function writeNoPrivateAPIUsageCSV(testStartRecords: TestStartRecord[]) { + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array + const maxSuiteLevel = testStartRecords + .map((record) => record.context.suite.length) + .reduce((a, b) => Math.max(a, b), -Infinity); + + const columnHeaders = commonHeaders(maxSuiteLevel); + + const csvRows = testStartRecords.map((record) => { + const context = record.context; + return [context.file, ...suitesColumnsForCSV(context.suite, maxSuiteLevel), context.title]; + }); + + writeCSVData([columnHeaders, ...csvRows], 'tests-that-do-not-require-private-api'); +} diff --git a/scripts/processPrivateApiData/dto.ts b/scripts/processPrivateApiData/dto.ts new file mode 100644 index 000000000..acbb272eb --- /dev/null +++ b/scripts/processPrivateApiData/dto.ts @@ -0,0 +1,53 @@ +export type TestPrivateApiContextDto = { + type: 'test'; + title: string; + /** + * null means that either the test isn’t parameterised or that this usage is unique to the specific parameter + */ + parameterisedTestTitle: string | null; + helperStack: string[]; + file: string; + suite: string[]; +}; + +export type HookPrivateApiContextDto = { + type: 'hook'; + title: string; + helperStack: string[]; + file: string; + suite: string[]; +}; + +export type RootHookPrivateApiContextDto = { + type: 'hook'; + title: string; + helperStack: string[]; + file: null; + suite: null; +}; + +export type TestDefinitionPrivateApiContextDto = { + type: 'definition'; + label: string; + helperStack: string[]; + file: string; + suite: string[]; +}; + +export type PrivateApiContextDto = + | TestPrivateApiContextDto + | HookPrivateApiContextDto + | RootHookPrivateApiContextDto + | TestDefinitionPrivateApiContextDto; + +export type PrivateApiUsageDto = { + context: PrivateApiContextDto; + privateAPIIdentifier: string; +}; + +export type TestStartRecord = { + context: TestPrivateApiContextDto; + privateAPIIdentifier: null; +}; + +export type Record = PrivateApiUsageDto | TestStartRecord; diff --git a/scripts/processPrivateApiData/exclusions.ts b/scripts/processPrivateApiData/exclusions.ts new file mode 100644 index 000000000..98791cf82 --- /dev/null +++ b/scripts/processPrivateApiData/exclusions.ts @@ -0,0 +1,40 @@ +import { PrivateApiUsageDto } from './dto'; + +type ExclusionRule = { + privateAPIIdentifier: string; + // i.e. only ignore when called from within this helper + helper?: string; +}; + +export function applyingExclusions(usageDtos: PrivateApiUsageDto[]) { + const exclusionRules: ExclusionRule[] = [ + // This is all helper stuff that we could pull into the test suite, and which for now we could just continue using the version privately exposed by ably-js, even in the UTS. + { privateAPIIdentifier: 'call.BufferUtils.areBuffersEqual' }, + { privateAPIIdentifier: 'call.BufferUtils.base64Decode' }, + { privateAPIIdentifier: 'call.BufferUtils.base64Encode' }, + { privateAPIIdentifier: 'call.BufferUtils.hexEncode' }, + { privateAPIIdentifier: 'call.BufferUtils.isBuffer' }, + { privateAPIIdentifier: 'call.BufferUtils.toArrayBuffer' }, + { privateAPIIdentifier: 'call.BufferUtils.utf8Encode' }, + { privateAPIIdentifier: 'call.Utils.copy' }, + { privateAPIIdentifier: 'call.Utils.inspectError' }, + { privateAPIIdentifier: 'call.Utils.keysArray' }, + { privateAPIIdentifier: 'call.Utils.mixin' }, + { privateAPIIdentifier: 'call.Utils.toQueryString' }, + { privateAPIIdentifier: 'call.msgpack.decode' }, + { privateAPIIdentifier: 'call.msgpack.encode' }, + { privateAPIIdentifier: 'call.http.doUri', helper: 'getJWT' }, + + // I’m going to exclude this use of nextTick because I think it’s one where maybe the logic applies cross-platform (but it could turn out I’m wrong about that one, in which case we’ll need to re-assess) + { privateAPIIdentifier: 'call.Platform.nextTick', helper: 'callbackOnClose' }, + ]; + + return usageDtos.filter( + (usageDto) => + !exclusionRules.some( + (exclusionRule) => + exclusionRule.privateAPIIdentifier === usageDto.privateAPIIdentifier && + (!('helper' in exclusionRule) || usageDto.context.helperStack.includes(exclusionRule.helper!)), + ), + ); +} diff --git a/scripts/processPrivateApiData/grouping.ts b/scripts/processPrivateApiData/grouping.ts new file mode 100644 index 000000000..07f5ce989 --- /dev/null +++ b/scripts/processPrivateApiData/grouping.ts @@ -0,0 +1,49 @@ +import { PrivateApiUsageDto } from './dto'; + +export type ContextGroup = { + key: Key; + usages: PrivateApiUsageDto[]; +}; + +/** + * Makes sure that each private API is only listed once in a given context. + */ +function dedupeUsages(contextGroups: ContextGroup[]) { + for (const contextGroup of contextGroups) { + const newUsages: typeof contextGroup.usages = []; + + for (const usage of contextGroup.usages) { + const existing = newUsages.find((otherUsage) => otherUsage.privateAPIIdentifier === usage.privateAPIIdentifier); + if (existing === undefined) { + newUsages.push(usage); + } + } + + contextGroup.usages = newUsages; + } +} + +export function groupedAndDeduped( + usages: PrivateApiUsageDto[], + keyForUsage: (usage: PrivateApiUsageDto) => Key, + areKeysEqual: (key1: Key, key2: Key) => boolean, +) { + const result: ContextGroup[] = []; + + for (const usage of usages) { + const key = keyForUsage(usage); + + let existingGroup = result.find((group) => areKeysEqual(group.key, key)); + + if (existingGroup === undefined) { + existingGroup = { key, usages: [] }; + result.push(existingGroup); + } + + existingGroup.usages.push(usage); + } + + dedupeUsages(result); + + return result; +} diff --git a/scripts/processPrivateApiData/load.ts b/scripts/processPrivateApiData/load.ts new file mode 100644 index 000000000..bfcadc63f --- /dev/null +++ b/scripts/processPrivateApiData/load.ts @@ -0,0 +1,16 @@ +import { readFileSync } from 'fs'; +import { applyingExclusions } from './exclusions'; +import { splittingRecords, stripFilePrefix } from './utils'; +import { Record } from './dto'; + +export function load(jsonFilePath: string) { + let records = JSON.parse(readFileSync(jsonFilePath).toString('utf-8')) as Record[]; + + stripFilePrefix(records); + + let { usageDtos, testStartRecords } = splittingRecords(records); + + usageDtos = applyingExclusions(usageDtos); + + return { usageDtos, testStartRecords }; +} diff --git a/scripts/processPrivateApiData/run.ts b/scripts/processPrivateApiData/run.ts new file mode 100644 index 000000000..65101e2b7 --- /dev/null +++ b/scripts/processPrivateApiData/run.ts @@ -0,0 +1,49 @@ +import { writeNoPrivateAPIUsageCSV, writeRuntimePrivateAPIUsageCSV, writeStaticPrivateAPIUsageCSV } from './csv'; +import { groupedAndDedupedByRuntimeContext } from './runtimeContext'; +import { groupedAndDedupedByStaticContext } from './staticContext'; +import { load } from './load'; +import { percentageString, sortStaticContextUsages } from './utils'; +import { splitTestsByPrivateAPIUsageRequirement } from './withoutPrivateAPIUsage'; + +if (process.argv.length > 3) { + throw new Error('Expected a single argument (path to private API usages JSON file'); +} + +const jsonFilePath = process.argv[2]; + +if (!jsonFilePath) { + throw new Error('Path to private API usages JSON file not specified'); +} + +const { usageDtos, testStartRecords } = load(jsonFilePath); + +const usagesGroupedByRuntimeContext = groupedAndDedupedByRuntimeContext(usageDtos); + +const usagesGroupedByStaticContext = groupedAndDedupedByStaticContext(usageDtos); +sortStaticContextUsages(usagesGroupedByStaticContext); + +const { + requiringPrivateAPIUsage: testsThatRequirePrivateAPIUsage, + notRequiringPrivateAPIUsage: testsThatDoNotRequirePrivateAPIUsage, +} = splitTestsByPrivateAPIUsageRequirement(testStartRecords, usagesGroupedByRuntimeContext); + +const totalNumberOfTests = testStartRecords.length; +const numberOfTestsThatRequirePrivateApiUsage = testsThatRequirePrivateAPIUsage.length; +const numberOfTestsThatDoNotRequirePrivateAPIUsage = testsThatDoNotRequirePrivateAPIUsage.length; + +console.log( + `Total number of tests: ${totalNumberOfTests} +Number of tests that require private API usage: ${numberOfTestsThatRequirePrivateApiUsage} (${percentageString( + numberOfTestsThatRequirePrivateApiUsage, + totalNumberOfTests, + )}) +Number of tests that do not require private API usage: ${numberOfTestsThatDoNotRequirePrivateAPIUsage} (${percentageString( + numberOfTestsThatDoNotRequirePrivateAPIUsage, + totalNumberOfTests, + )}) +`, +); + +writeRuntimePrivateAPIUsageCSV(usagesGroupedByRuntimeContext); +writeStaticPrivateAPIUsageCSV(usagesGroupedByStaticContext); +writeNoPrivateAPIUsageCSV(testsThatDoNotRequirePrivateAPIUsage); diff --git a/scripts/processPrivateApiData/runtimeContext.ts b/scripts/processPrivateApiData/runtimeContext.ts new file mode 100644 index 000000000..db882c9af --- /dev/null +++ b/scripts/processPrivateApiData/runtimeContext.ts @@ -0,0 +1,30 @@ +import { PrivateApiContextDto, PrivateApiUsageDto } from './dto'; +import { ContextGroup, groupedAndDeduped } from './grouping'; +import { joinComponents } from './utils'; + +/** + * Used for determining whether two contexts are equal. + */ +export function runtimeContextIdentifier(context: RuntimeContext) { + return joinComponents([ + { key: 'type', value: context.type }, + { key: 'file', value: context.file ?? 'null' }, + { key: 'suite', value: context.suite?.join(',') ?? 'null' }, + { key: 'title', value: context.type === 'definition' ? context.label : context.title }, + { key: 'helperStack', value: 'helperStack' in context ? context.helperStack.join(',') : 'null' }, + { + key: 'parameterisedTestTitle', + value: ('parameterisedTestTitle' in context ? context.parameterisedTestTitle : null) ?? 'null', + }, + ]); +} + +export type RuntimeContext = PrivateApiContextDto; + +export function groupedAndDedupedByRuntimeContext(usages: PrivateApiUsageDto[]): ContextGroup[] { + return groupedAndDeduped( + usages, + (usage) => usage.context, + (context1, context2) => runtimeContextIdentifier(context1) === runtimeContextIdentifier(context2), + ); +} diff --git a/scripts/processPrivateApiData/staticContext.ts b/scripts/processPrivateApiData/staticContext.ts new file mode 100644 index 000000000..07d7ca3d2 --- /dev/null +++ b/scripts/processPrivateApiData/staticContext.ts @@ -0,0 +1,115 @@ +import { PrivateApiUsageDto } from './dto'; +import { joinComponents } from './utils'; +import { ContextGroup, groupedAndDeduped } from './grouping'; + +export type MiscHelperStaticContext = { + type: 'miscHelper'; + title: string; +}; + +export type TestDefinitionStaticContext = { + type: 'testDefinition'; + title: string; + file: string; + suite: string[]; +}; + +export type RootHookStaticContext = { + type: 'rootHook'; + title: string; +}; + +export type HookStaticContext = { + type: 'hook'; + title: string; + file: string; + suite: string[]; +}; + +export type ParameterisedTestHelperStaticContext = { + type: 'parameterisedTestHelper'; + title: string; + file: string; + suite: string[]; +}; + +export type TestStaticContext = { + type: 'test'; + title: string; + file: string; + suite: string[]; +}; + +export type StaticContext = + | MiscHelperStaticContext + | TestDefinitionStaticContext + | RootHookStaticContext + | HookStaticContext + | ParameterisedTestHelperStaticContext + | TestStaticContext; + +/** + * Used for determining whether two contexts are equal. + */ +export function staticContextIdentifier(context: StaticContext) { + return joinComponents([ + { key: 'type', value: context.type }, + { key: 'file', value: 'file' in context ? context.file : 'null' }, + { key: 'suite', value: 'suite' in context ? context.suite.join(',') : 'null' }, + { key: 'title', value: context.title }, + ]); +} + +export function staticContextForUsage(usage: PrivateApiUsageDto): StaticContext { + if (usage.context.helperStack.length > 0) { + return { + type: 'miscHelper', + title: usage.context.helperStack[0], + }; + } else if (usage.context.type === 'definition') { + return { + type: 'testDefinition', + file: usage.context.file, + suite: usage.context.suite, + title: usage.context.label, + }; + } else if (usage.context.type === 'hook') { + if (usage.context.file === null) { + return { + type: 'rootHook', + title: usage.context.title, + }; + } else { + return { + type: 'hook', + file: usage.context.file, + suite: usage.context.suite, + title: usage.context.title, + }; + } + } else { + if (usage.context.parameterisedTestTitle !== null) { + return { + type: 'parameterisedTestHelper', + file: usage.context.file, + suite: usage.context.suite, + title: usage.context.parameterisedTestTitle, + }; + } else { + return { + type: 'test', + file: usage.context.file, + suite: usage.context.suite, + title: usage.context.title, + }; + } + } +} + +export function groupedAndDedupedByStaticContext(usages: PrivateApiUsageDto[]): ContextGroup[] { + return groupedAndDeduped( + usages, + (usage) => staticContextForUsage(usage), + (context1, context2) => staticContextIdentifier(context1) === staticContextIdentifier(context2), + ); +} diff --git a/scripts/processPrivateApiData/utils.ts b/scripts/processPrivateApiData/utils.ts new file mode 100644 index 000000000..81b3162cd --- /dev/null +++ b/scripts/processPrivateApiData/utils.ts @@ -0,0 +1,45 @@ +import { PrivateApiUsageDto, Record, TestStartRecord } from './dto'; +import { ContextGroup } from './grouping'; +import { StaticContext } from './staticContext'; + +export function stripFilePrefix(records: Record[]) { + for (const record of records) { + if (record.context.file !== null) { + record.context.file = record.context.file.replace('/home/runner/work/ably-js/ably-js/', ''); + } + } +} + +export function splittingRecords(records: Record[]) { + return { + testStartRecords: records.filter((record) => record.privateAPIIdentifier == null) as TestStartRecord[], + usageDtos: records.filter((record) => record.privateAPIIdentifier !== null) as PrivateApiUsageDto[], + }; +} + +export function percentageString(value: number, total: number) { + return `${((100 * value) / total).toLocaleString(undefined, { maximumFractionDigits: 1 })}%`; +} + +/** + * Puts the miscHelper usages first. + */ +export function sortStaticContextUsages(contextGroups: ContextGroup[]) { + const original = [...contextGroups]; + + contextGroups.sort((a, b) => { + if (a.key.type === 'miscHelper' && b.key.type !== 'miscHelper') { + return -1; + } + + if (a.key.type !== 'miscHelper' && b.key.type === 'miscHelper') { + return 1; + } + + return original.indexOf(a) - original.indexOf(b); + }); +} + +export function joinComponents(components: { key: string; value: string }[]) { + return components.map((pair) => `${pair.key}=${pair.value}`).join(';'); +} diff --git a/scripts/processPrivateApiData/withoutPrivateAPIUsage.ts b/scripts/processPrivateApiData/withoutPrivateAPIUsage.ts new file mode 100644 index 000000000..b4826c09e --- /dev/null +++ b/scripts/processPrivateApiData/withoutPrivateAPIUsage.ts @@ -0,0 +1,71 @@ +import { TestStartRecord } from './dto'; +import { ContextGroup } from './grouping'; +import { RuntimeContext } from './runtimeContext'; + +function areStringArraysEqual(arr1: string[], arr2: string[]) { + if (arr1.length !== arr2.length) { + return false; + } + + for (let i = 0; i < arr1.length; i++) { + if (arr1[i] !== arr2[i]) { + return false; + } + } + + return true; +} + +function mustSuiteHierarchyBeExecutedToRunTest(test: TestStartRecord, suites: string[]) { + // i.e. is `suites` a prefix of `test.context.suite`? + return areStringArraysEqual(test.context.suite.slice(0, suites.length), suites); +} + +function mustRuntimeContextBeExecutedToRunTest(test: TestStartRecord, runtimeContext: RuntimeContext) { + if (runtimeContext.type === 'hook') { + if (runtimeContext.file === null) { + // root hook; must be executed to run _any_ test + return true; + } + if ( + runtimeContext.file === test.context.file && + mustSuiteHierarchyBeExecutedToRunTest(test, runtimeContext.suite) + ) { + // the hook must be executed to run this test + return true; + } + } + + // otherwise, return true iff it’s the same test + return ( + runtimeContext.type === 'test' && + runtimeContext.file === test.context.file && + areStringArraysEqual(runtimeContext.suite, test.context.suite) && + test.context.title === runtimeContext.title + ); +} + +/** + * This extracts all of the test start records for the tests that can be run without any private API usage. That is, neither the test itself, nor any of the hooks that the test requires, call a private API. It does not consider whether private APIs are required in order to define the test (that is, contexts of type `testDefinition`). + */ +export function splitTestsByPrivateAPIUsageRequirement( + testStartRecords: TestStartRecord[], + groupedUsages: ContextGroup[], +): { requiringPrivateAPIUsage: TestStartRecord[]; notRequiringPrivateAPIUsage: TestStartRecord[] } { + const result: { requiringPrivateAPIUsage: TestStartRecord[]; notRequiringPrivateAPIUsage: TestStartRecord[] } = { + requiringPrivateAPIUsage: [], + notRequiringPrivateAPIUsage: [], + }; + + for (const testStartRecord of testStartRecords) { + if ( + groupedUsages.some((contextGroup) => mustRuntimeContextBeExecutedToRunTest(testStartRecord, contextGroup.key)) + ) { + result.requiringPrivateAPIUsage.push(testStartRecord); + } else { + result.notRequiringPrivateAPIUsage.push(testStartRecord); + } + } + + return result; +}