WIP Generate reports on private API usage

(and upload them)

TODO if this is going to go into main, then it needs to handle hooks and
test definition still

.github/workflows/test-browser.yml

@@ -30,11 +30,15 @@ jobs:
       - env:
           PLAYWRIGHT_BROWSER: ${{ matrix.browser }}
         run: npm run test:playwright
+      - name: Generate private API usage reports
+        run: npm run process-private-api-data private-api-usage/*.json
       - name: Save private API usage data
         uses: actions/upload-artifact@v4
         with:
           name: private-api-usage-${{ matrix.browser }}
-          path: private-api-usage
+          path: |
+            private-api-usage
+            private-api-usage-reports
       - name: Upload test results
         if: always()
         uses: ably/test-observability-action@v1

.github/workflows/test-node.yml

@@ -28,11 +28,15 @@ jobs:
       - run: npm run test:node
         env:
           CI: true
+      - name: Generate private API usage reports
+        run: npm run process-private-api-data private-api-usage/*.json
       - name: Save private API usage data
         uses: actions/upload-artifact@v4
         with:
           name: private-api-usage-${{ matrix.node-version }}
-          path: private-api-usage
+          path: |
+            private-api-usage
+            private-api-usage-reports
       - name: Upload test results
         if: always()
         uses: ably/test-observability-action@v1

.gitignore

@@ -9,4 +9,5 @@ react/
 typedoc/generated/
 junit/
 private-api-usage/
+private-api-usage-reports/
 test/support/mocha_junit_reporter/build/

package.json

@@ -79,6 +79,7 @@
     "chai": "^4.2.0",
     "cli-table": "^0.3.11",
     "cors": "^2.8.5",
+    "csv": "^6.3.9",
     "esbuild": "^0.18.10",
     "esbuild-plugin-umd-wrapper": "ably-forks/esbuild-plugin-umd-wrapper#1.0.7-optional-amd-named-module",
     "esbuild-runner": "^2.2.2",
@@ -154,10 +155,11 @@
     "lint": "eslint .",
     "lint:fix": "eslint --fix .",
     "prepare": "npm run build",
-    "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt",
-    "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/*.[jt]s docs/**/*.md grunt",
+    "format": "prettier --write --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt",
+    "format:check": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore src test ably.d.ts modular.d.ts webpack.config.js Gruntfile.js scripts/**/*.[jt]s docs/**/*.md grunt",
     "sourcemap": "source-map-explorer build/ably.min.js",
     "modulereport": "tsc --noEmit --esModuleInterop scripts/moduleReport.ts && esr scripts/moduleReport.ts",
+    "process-private-api-data": "tsc --noEmit --esModuleInterop --strictNullChecks scripts/processPrivateApiData/run.ts && esr scripts/processPrivateApiData/run.ts",
     "docs": "typedoc"
   }
 }

scripts/processPrivateApiData/csv.ts

@@ -0,0 +1,121 @@
+import { stringify as csvStringify } from 'csv-stringify/sync';
+import { writeFileSync, existsSync, mkdirSync } from 'fs';
+import { PrivateApiContextDto, TestStartRecord } from './dto';
+import { ContextGroup } from './grouping';
+import { RuntimeContext } from './runtimeContext';
+import { StaticContext } from './staticContext';
+import path from 'path';
+
+function suiteAtLevelForCSV(suites: string[] | null, level: number) {
+  return suites?.[level] ?? '';
+}
+
+function suitesColumnsForCSV(suites: string[] | null, maxSuiteLevel: number) {
+  const result: string[] = [];
+  for (let i = 0; i < maxSuiteLevel; i++) {
+    result.push(suiteAtLevelForCSV(suites, i));
+  }
+  return result;
+}
+
+function suitesHeaders(maxSuiteLevel: number) {
+  const result: string[] = [];
+  for (let i = 0; i < maxSuiteLevel; i++) {
+    result.push(`Suite (level ${i + 1})`);
+  }
+  return result;
+}
+
+function commonHeaders(maxSuiteLevel: number) {
+  return ['File', ...suitesHeaders(maxSuiteLevel), 'Description'].filter((val) => val !== null) as string[];
+}
+
+function writeCSVData(rows: string[][], name: string) {
+  const outputDirectoryPath = path.join(__dirname, '..', '..', 'private-api-usage-reports');
+
+  if (!existsSync(outputDirectoryPath)) {
+    mkdirSync(outputDirectoryPath);
+  }
+
+  const result = csvStringify(rows);
+  writeFileSync(path.join(outputDirectoryPath, `${name}.csv`), result);
+}
+
+export function writeRuntimePrivateAPIUsageCSV(contextGroups: ContextGroup<RuntimeContext>[]) {
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array
+  const maxSuiteLevel = contextGroups
+    .map((group) => (group.key.suite ?? []).length)
+    .reduce((a, b) => Math.max(a, b), -Infinity);
+
+  const columnHeaders = [
+    'Context',
+    ...commonHeaders(maxSuiteLevel),
+    'Via parameterised test helper',
+    'Via misc. helpers',
+    'Private API called',
+  ];
+
+  const csvRows = contextGroups
+    .map((contextGroup) => {
+      const runtimeContext = contextGroup.key;
+
+      const contextColumns = [
+        runtimeContext.type,
+        runtimeContext.file ?? '',
+        ...suitesColumnsForCSV(runtimeContext.suite, maxSuiteLevel),
+        runtimeContext.type === 'definition' ? runtimeContext.label : runtimeContext.title,
+      ];
+
+      return contextGroup.usages.map((usage) => [
+        ...contextColumns,
+        (usage.context.type === 'test' ? usage.context.parameterisedTestTitle : null) ?? '',
+        [...usage.context.helperStack].reverse().join(' -> '),
+        usage.privateAPIIdentifier,
+      ]);
+    })
+    .flat();
+
+  writeCSVData([columnHeaders, ...csvRows], 'runtime-private-api-usage');
+}
+
+export function writeStaticPrivateAPIUsageCSV(contextGroups: ContextGroup<StaticContext>[]) {
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array
+  const maxSuiteLevel = contextGroups
+    .map((group) => (group.key.type === 'miscHelper' ? [] : group.key.suite).length)
+    .reduce((a, b) => Math.max(a, b), -Infinity);
+
+  const columnHeaders = ['Context', ...commonHeaders(maxSuiteLevel), 'Private API called'];
+
+  const csvRows = contextGroups
+    .map((contextGroup) => {
+      const staticContext = contextGroup.key;
+
+      const contextColumns = [
+        staticContext.type,
+        staticContext.type === 'miscHelper' ? '' : staticContext.file,
+        ...suitesColumnsForCSV(staticContext.type === 'miscHelper' ? null : staticContext.suite, maxSuiteLevel),
+        staticContext.type === 'test' ? staticContext.title : staticContext.helperName,
+      ];
+
+      return contextGroup.usages.map((usage) => [...contextColumns, usage.privateAPIIdentifier]);
+    })
+    .flat();
+
+  writeCSVData([columnHeaders, ...csvRows], 'static-private-api-usage');
+}
+
+export function writeNoPrivateAPIUsageCSV(testStartRecords: TestStartRecord[]) {
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max#getting_the_maximum_element_of_an_array
+  const maxSuiteLevel = testStartRecords
+    .map((record) => record.context.suite.length)
+    .reduce((a, b) => Math.max(a, b), -Infinity);
+
+  const columnHeaders = commonHeaders(maxSuiteLevel);
+
+  const csvRows = testStartRecords.map((record) => {
+    const context = record.context;
+    return [context.file, ...suitesColumnsForCSV(context.suite, maxSuiteLevel), context.title];
+  });
+
+  writeCSVData([columnHeaders, ...csvRows], 'tests-that-do-not-use-private-api');
+}

scripts/processPrivateApiData/dto.ts

@@ -0,0 +1,53 @@
+export type TestPrivateApiContextDto = {
+  type: 'test';
+  title: string;
+  /**
+   * null means that either the test isn’t parameterised or that this usage is unique to the specific parameter
+   */
+  parameterisedTestTitle: string | null;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type HookPrivateApiContextDto = {
+  type: 'hook';
+  title: string;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type RootHookPrivateApiContextDto = {
+  type: 'hook';
+  title: string;
+  helperStack: string[];
+  file: null;
+  suite: null;
+};
+
+export type TestDefinitionPrivateApiContextDto = {
+  type: 'definition';
+  label: string;
+  helperStack: string[];
+  file: string;
+  suite: string[];
+};
+
+export type PrivateApiContextDto =
+  | TestPrivateApiContextDto
+  | HookPrivateApiContextDto
+  | RootHookPrivateApiContextDto
+  | TestDefinitionPrivateApiContextDto;
+
+export type PrivateApiUsageDto = {
+  context: PrivateApiContextDto;
+  privateAPIIdentifier: string;
+};
+
+export type TestStartRecord = {
+  context: TestPrivateApiContextDto;
+  privateAPIIdentifier: null;
+};
+
+export type Record = PrivateApiUsageDto | TestStartRecord;

scripts/processPrivateApiData/exclusions.ts

@@ -0,0 +1,40 @@
+import { PrivateApiUsageDto } from './dto';
+
+type ExclusionRule = {
+  privateAPIIdentifier: string;
+  // i.e. only ignore when called from within this helper
+  helper?: string;
+};
+
+export function applyingExclusions(usageDtos: PrivateApiUsageDto[]) {
+  const exclusionRules: ExclusionRule[] = [
+    // This is all helper stuff that we could pull into the test suite, and which for now we could just continue using the version privately exposed by ably-js, even in the UTS.
+    { privateAPIIdentifier: 'call.BufferUtils.areBuffersEqual' },
+    { privateAPIIdentifier: 'call.BufferUtils.base64Decode' },
+    { privateAPIIdentifier: 'call.BufferUtils.base64Encode' },
+    { privateAPIIdentifier: 'call.BufferUtils.hexEncode' },
+    { privateAPIIdentifier: 'call.BufferUtils.isBuffer' },
+    { privateAPIIdentifier: 'call.BufferUtils.toArrayBuffer' },
+    { privateAPIIdentifier: 'call.BufferUtils.utf8Encode' },
+    { privateAPIIdentifier: 'call.Utils.copy' },
+    { privateAPIIdentifier: 'call.Utils.inspectError' },
+    { privateAPIIdentifier: 'call.Utils.keysArray' },
+    { privateAPIIdentifier: 'call.Utils.mixin' },
+    { privateAPIIdentifier: 'call.Utils.toQueryString' },
+    { privateAPIIdentifier: 'call.msgpack.decode' },
+    { privateAPIIdentifier: 'call.msgpack.encode' },
+    { privateAPIIdentifier: 'call.http.doUri', helper: 'getJWT' },
+
+    // I’m going to exclude this use of nextTick because I think it’s one where maybe the logic applies cross-platform (but it could turn out I’m wrong about that one, in which case we’ll need to re-assess)
+    { privateAPIIdentifier: 'call.Platform.nextTick', helper: 'callbackOnClose' },
+  ];
+
+  return usageDtos.filter(
+    (usageDto) =>
+      !exclusionRules.some(
+        (exclusionRule) =>
+          exclusionRule.privateAPIIdentifier === usageDto.privateAPIIdentifier &&
+          (!('helper' in exclusionRule) || usageDto.context.helperStack.includes(exclusionRule.helper!)),
+      ),
+  );
+}