WIP Generate reports on private API usage

(and upload them)

TODO if this is going to go into main, then it needs to handle hooks and
test definition still

Resolves ECO-4834.

.github/workflows/test-browser.yml

@@ -30,11 +30,15 @@ jobs:
       - env:
           PLAYWRIGHT_BROWSER: ${{ matrix.browser }}
         run: npm run test:playwright
+      - name: Generate private API usage reports
+        run: npm run process-private-api-data private-api-usage/*.json
       - name: Save private API usage data
         uses: actions/upload-artifact@v4
         with:
           name: private-api-usage-${{ matrix.browser }}
-          path: private-api-usage
+          path: |
+            private-api-usage
+            private-api-usage-reports
       - name: Upload test results
         if: always()
         uses: ably/test-observability-action@v1

.github/workflows/test-node.yml

@@ -28,11 +28,15 @@ jobs:
       - run: npm run test:node
         env:
           CI: true
+      - name: Generate private API usage reports
+        run: npm run process-private-api-data private-api-usage/*.json
       - name: Save private API usage data
         uses: actions/upload-artifact@v4
         with:
           name: private-api-usage-${{ matrix.node-version }}
-          path: private-api-usage
+          path: |
+            private-api-usage
+            private-api-usage-reports
       - name: Upload test results
         if: always()
         uses: ably/test-observability-action@v1

.gitignore

@@ -9,4 +9,5 @@ react/
 typedoc/generated/
 junit/
 private-api-usage/
+private-api-usage-reports/
 test/support/mocha_junit_reporter/build/

package.json

@@ -79,6 +79,7 @@
     "chai": "^4.2.0",
     "cli-table": "^0.3.11",
     "cors": "^2.8.5",
+    "csv": "^6.3.9",
     "esbuild": "^0.18.10",
     "esbuild-plugin-umd-wrapper": "ably-forks/esbuild-plugin-umd-wrapper#1.0.7-optional-amd-named-module",
     "esbuild-runner": "^2.2.2",
@@ -154,10 +155,11 @@
     "lint": "eslint .",
     "lint:fix": "eslint --fix .",
     "prepare": "npm run build",
-    "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt",
-    "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt",
+    "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt",
+    "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt",
     "sourcemap": "source-map-explorer build/ably.min.js",
     "modulereport": "tsc --noEmit --esModuleInterop scripts/moduleReport.ts && esr scripts/moduleReport.ts",
+    "process-private-api-data": "tsc --noEmit --esModuleInterop --strictNullChecks scripts/processPrivateApiData/run.ts && esr scripts/processPrivateApiData/run.ts",
     "docs": "typedoc"
   }
 }

scripts/processPrivateApiData/csv.ts

@@ -0,0 +1,121 @@
+import { stringify as csvStringify } from 'csv-stringify/sync';
+import { writeFileSync, existsSync, mkdirSync } from 'fs';
+import { PrivateApiContextDto, TestStartRecord } from './dto';
+import { ContextGroup } from './grouping';
+import { RuntimeContext } from './runtimeContext';
+import { StaticContext } from './staticContext';
+import path from 'path';
+
+function suiteAtLevelForCSV(suites: string[] | null, level: number) {
+  return suites?.[level] ?? '';
+}
+
+function suitesColumnsForCSV(suites: string[] | null, maxSuiteLevel: number) {
+  const result: string[] = [];
+  for (let i = 0; i < maxSuiteLevel; i++) {
+    result.push(suiteAtLevelForCSV(suites, i));
+  }
+  return result;
+}
+
+function suitesHeaders(maxSuiteLevel: number) {
+  const result: string[] = [];
+  for (let i = 0; i < maxSuiteLevel; i++) {
+    result.push(`Suite (level ${i + 1})`);
+  }
+  return result;
+}
+
+function commonHeaders(maxSuiteLevel: number) {
+  return ['File', ...suitesHeaders(maxSuiteLevel), 'Description'].filter((val) => val !== null) as string[];
+}
+
+function writeCSVData(rows: string[][], name: string) {
+  const outputDirectoryPath = path.join(__dirname, '..', '..', 'private-api-usage-reports');
+
+  if (!existsSync(outputDirectoryPath)) {
+    mkdirSync(outputDirectoryPath);
+  }
+
+  const result = csvStringify(rows);
+  writeFileSync(path.join(outputDirectoryPath, `${name}.csv`), result);
+}
+
+export function writeRuntimePrivateAPIUsageCSV(contextGroups: ContextGroup<RuntimeContext>[]) {
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array
+  const maxSuiteLevel = contextGroups
+    .map((group) => (group.key.suite ?? []).length)
+    .reduce((a, b) => Math.max(a, b), -Infinity);
+
+  const columnHeaders = [
+    'Context',
+    ...commonHeaders(maxSuiteLevel),
+    'Via parameterised test helper',
+    'Via misc. helpers',
+    'Private API called',
+  ];
+
+  const csvRows = contextGroups
+    .map((contextGroup) => {
+      const runtimeContext = contextGroup.key;
+
+      const contextColumns = [
+        runtimeContext.type,
+        runtimeContext.file ?? '',
+        ...suitesColumnsForCSV(runtimeContext.suite, maxSuiteLevel),
+        runtimeContext.type === 'definition' ? runtimeContext.label : runtimeContext.title,
+      ];
+
+      return contextGroup.usages.map((usage) => [
+        ...contextColumns,
+        (usage.context.type === 'test' ? usage.context.parameterisedTestTitle : null) ?? '',
+        [...usage.context.helperStack].reverse().join(' -> '),
+        usage.privateAPIIdentifier,
+      ]);
+    })
+    .flat();
+
+  writeCSVData([columnHeaders, ...csvRows], 'runtime-private-api-usage');
+}
+
+export function writeStaticPrivateAPIUsageCSV(contextGroups: ContextGroup<StaticContext>[]) {
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array
+  const maxSuiteLevel = contextGroups
+    .map((group) => (group.key.type === 'miscHelper' ? [] : group.key.suite).length)
+    .reduce((a, b) => Math.max(a, b), -Infinity);
+
+  const columnHeaders = ['Context', ...commonHeaders(maxSuiteLevel), 'Private API called'];
+
+  const csvRows = contextGroups
+    .map((contextGroup) => {
+      const staticContext = contextGroup.key;
+
+      const contextColumns = [
+        staticContext.type,
+        staticContext.type === 'miscHelper' ? '' : staticContext.file,
+        ...suitesColumnsForCSV(staticContext.type === 'miscHelper' ? null : staticContext.suite, maxSuiteLevel),
+        staticContext.type === 'test' ? staticContext.title : staticContext.helperName,
+      ];
+
+      return contextGroup.usages.map((usage) => [...contextColumns, usage.privateAPIIdentifier]);
+    })
+    .flat();
+
+  writeCSVData([columnHeaders, ...csvRows], 'static-private-api-usage');
+}
+
+export function writeNoPrivateAPIUsageCSV(testStartRecords: TestStartRecord[]) {
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array
+  const maxSuiteLevel = testStartRecords
+    .map((record) => record.context.suite.length)
+    .reduce((a, b) => Math.max(a, b), -Infinity);
+
+  const columnHeaders = commonHeaders(maxSuiteLevel);
+
+  const csvRows = testStartRecords.map((record) => {
+    const context = record.context;
+    return [context.file, ...suitesColumnsForCSV(context.suite, maxSuiteLevel), context.title];
+  });
+
+  writeCSVData([columnHeaders, ...csvRows], 'tests-that-do-not-use-private-api');
+}

scripts/processPrivateApiData/dto.ts

@@ -0,0 +1,53 @@
+export type TestPrivateApiContextDto = {
+  type: 'test';
+  title: string;
+  /**
+   * null means that either the test isn’t parameterised or that this usage is unique to the specific parameter
+   */
+  parameterisedTestTitle: string | null;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type HookPrivateApiContextDto = {
+  type: 'hook';
+  title: string;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type RootHookPrivateApiContextDto = {
+  type: 'hook';
+  title: string;
+  helperStack: string[];
+  file: null;
+  suite: null;
+};
+
+export type TestDefinitionPrivateApiContextDto = {
+  type: 'definition';
+  label: string;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type PrivateApiContextDto =
+  | TestPrivateApiContextDto
+  | HookPrivateApiContextDto
+  | RootHookPrivateApiContextDto
+  | TestDefinitionPrivateApiContextDto;
+
+export type PrivateApiUsageDto = {
+  context: PrivateApiContextDto;
+  privateAPIIdentifier: string;
+};
+
+export type TestStartRecord = {
+  context: TestPrivateApiContextDto;
+  privateAPIIdentifier: null;
+};
+
+export type Record = PrivateApiUsageDto | TestStartRecord;

scripts/processPrivateApiData/exclusions.ts

@@ -0,0 +1,40 @@
+import { PrivateApiUsageDto } from './dto';
+
+type ExclusionRule = {
+  privateAPIIdentifier: string;
+  // i.e. only ignore when called from within this helper
+  helper?: string;
+};
+
+export function applyingExclusions(usageDtos: PrivateApiUsageDto[]) {
+  const exclusionRules: ExclusionRule[] = [
+    // This is all helper stuff that we could pull into the test suite, and which for now we could just continue using the version privately exposed by ably-js, even in the UTS.
+    { privateAPIIdentifier: 'call.BufferUtils.areBuffersEqual' },
+    { privateAPIIdentifier: 'call.BufferUtils.base64Decode' },
+    { privateAPIIdentifier: 'call.BufferUtils.base64Encode' },
+    { privateAPIIdentifier: 'call.BufferUtils.hexEncode' },
+    { privateAPIIdentifier: 'call.BufferUtils.isBuffer' },
+    { privateAPIIdentifier: 'call.BufferUtils.toArrayBuffer' },
+    { privateAPIIdentifier: 'call.BufferUtils.utf8Encode' },
+    { privateAPIIdentifier: 'call.Utils.copy' },
+    { privateAPIIdentifier: 'call.Utils.inspectError' },
+    { privateAPIIdentifier: 'call.Utils.keysArray' },
+    { privateAPIIdentifier: 'call.Utils.mixin' },
+    { privateAPIIdentifier: 'call.Utils.toQueryString' },
+    { privateAPIIdentifier: 'call.msgpack.decode' },
+    { privateAPIIdentifier: 'call.msgpack.encode' },
+    { privateAPIIdentifier: 'call.http.doUri', helper: 'getJWT' },
+
+    // I’m going to exclude this use of nextTick because I think it’s one where maybe the logic applies cross-platform (but it could turn out I’m wrong about that one, in which case we’ll need to re-assess)
+    { privateAPIIdentifier: 'call.Platform.nextTick', helper: 'callbackOnClose' },
+  ];
+
+  return usageDtos.filter(
+    (usageDto) =>
+      !exclusionRules.some(
+        (exclusionRule) =>
+          exclusionRule.privateAPIIdentifier === usageDto.privateAPIIdentifier &&
+          (!('helper' in exclusionRule) || usageDto.context.helperStack.includes(exclusionRule.helper!)),
+      ),
+  );
+}