chore: improve test run speed

.github/workflows/build-test.yml

@@ -97,4 +97,4 @@ jobs:
               run: pnpm install --frozen-lockfile
 
             - name: Test
-              run: pnpm run test-ci
+              run: pnpm run test-scaffold && pnpm run test-ci

.gitignore

@@ -7,3 +7,4 @@ dist
 .npmcache
 coverage
 .build
+.test

CONTRIBUTING.md

@@ -35,6 +35,14 @@ I want to think you first for considering contributing to ZenStack 🙏🏻. It'
     pnpm build
     ```
 
+1. Scaffold the project used for testing
+
+    ```bash
+    pnpm test-scaffold
+    ```
+
+    You only need to run this command once.
+
 1. Run tests
 
     ```bash

jest.config.ts

@@ -3,15 +3,19 @@
  * https://jestjs.io/docs/configuration
  */
 
+import path from 'path';
+
 export default {
     // Automatically clear mock calls, instances, contexts and results before every test
     clearMocks: true,
 
+    globalSetup: path.join(__dirname, './test-setup.ts'),
+
     // Indicates whether the coverage information should be collected while executing the test
     collectCoverage: true,
 
     // The directory where Jest should output its coverage files
-    coverageDirectory: 'tests/coverage',
+    coverageDirectory: path.join(__dirname, '.test/coverage'),
 
     // An array of regexp pattern strings used to skip coverage collection
     coveragePathIgnorePatterns: ['/node_modules/', '/tests/'],

package.json

@@ -5,8 +5,9 @@
     "scripts": {
         "build": "pnpm -r build",
         "lint": "pnpm -r lint",
-        "test": "ZENSTACK_TEST=1 pnpm -r run test --silent --forceExit",
-        "test-ci": "ZENSTACK_TEST=1 pnpm -r run test --silent --forceExit",
+        "test": "ZENSTACK_TEST=1 pnpm -r --parallel run test --silent --forceExit",
+        "test-ci": "ZENSTACK_TEST=1 pnpm -r --parallel run test --silent --forceExit",
+        "test-scaffold": "tsx script/test-scaffold.ts",
         "publish-all": "pnpm --filter \"./packages/**\" -r publish --access public",
         "publish-preview": "pnpm --filter \"./packages/**\" -r publish --force --registry https://preview.registry.zenstack.dev/",
         "unpublish-preview": "pnpm --recursive --shell-mode exec -- npm unpublish -f --registry https://preview.registry.zenstack.dev/ \"\\$PNPM_PACKAGE_NAME\""

packages/testtools/package.json

@@ -11,7 +11,7 @@
   "scripts": {
     "clean": "rimraf dist",
     "lint": "eslint src --ext ts",
-    "build": "pnpm lint && pnpm clean && tsc && copyfiles ./package.json ./LICENSE ./README.md dist && copyfiles -u 1 src/package.template.json src/.npmrc.template dist && pnpm pack dist --pack-destination '../../../.build'",
+    "build": "pnpm lint && pnpm clean && tsc && copyfiles ./package.json ./LICENSE ./README.md dist && pnpm pack dist --pack-destination '../../../.build'",
     "watch": "tsc --watch",
     "prepublishOnly": "pnpm build"
   },

packages/testtools/src/schema.ts

@@ -35,14 +35,23 @@
 };
 
 export function run(cmd: string, env?: Record<string, string>, cwd?: string) {
-    const start = Date.now();
-    execSync(cmd, {
-        stdio: 'pipe',
-        encoding: 'utf-8',
-        env: { ...process.env, DO_NOT_TRACK: '1', ...env },
-        cwd,
-    });
-    console.log('Execution took', Date.now() - start, 'ms', '-', cmd);
+    try {
+        const start = Date.now();
+        execSync(cmd, {
+            stdio: 'pipe',
+            encoding: 'utf-8',
+            env: { ...process.env, DO_NOT_TRACK: '1', ...env },
+            cwd,
+        });
+        console.log('Execution took', Date.now() - start, 'ms', '-', cmd);
+    } catch (err) {
+        console.error('Command failed:', cmd, err);
+        throw err;
+    }
+}
+
+export function installPackage(pkg: string, dev = false) {
+    run(`npm install ${dev ? '-D' : ''} --no-audit --no-fund ${pkg}`);
 }
 
 function normalizePath(p: string) {
@@ -86,17 +95,17 @@
 
 plugin meta {
     provider = '@core/model-meta'
-    preserveTsFiles = true
+    // preserveTsFiles = true
 }
 
 plugin policy {
     provider = '@core/access-policy'
-    preserveTsFiles = true
+    // preserveTsFiles = true
 }
 
 plugin zod {
     provider = '@core/zod'
-    preserveTsFiles = true
+    // preserveTsFiles = true
     modelOnly = ${!options.fullZod}
 }
 `;
@@ -138,21 +147,29 @@
 
     const { name: projectRoot } = tmp.dirSync({ unsafeCleanup: true });
 
-    const root = getWorkspaceRoot(__dirname);
+    const workspaceRoot = getWorkspaceRoot(__dirname);
 
-    if (!root) {
+    if (!workspaceRoot) {
         throw new Error('Could not find workspace root');
     }
 
-    const pkgContent = fs.readFileSync(path.join(__dirname, 'package.template.json'), { encoding: 'utf-8' });
-    fs.writeFileSync(path.join(projectRoot, 'package.json'), pkgContent.replaceAll('<root>', root));
-
-    const npmrcContent = fs.readFileSync(path.join(__dirname, '.npmrc.template'), { encoding: 'utf-8' });
-    fs.writeFileSync(path.join(projectRoot, '.npmrc'), npmrcContent.replaceAll('<root>', root));
-
     console.log('Workdir:', projectRoot);
     process.chdir(projectRoot);
 
+    // copy project structure from scaffold (prepared by test-setup.ts)
+    fs.cpSync(path.join(workspaceRoot, '.test/scaffold'), projectRoot, { recursive: true, force: true });
+
+    // install local deps
+    const localInstallDeps = [
+        'packages/schema/dist',
+        'packages/runtime/dist',
+        'packages/plugins/swr/dist',
+        'packages/plugins/trpc/dist',
+        'packages/plugins/openapi/dist',
+    ];
+
+    run(`npm i --no-audit --no-fund ${localInstallDeps.map((d) => path.join(workspaceRoot, d)).join(' ')}`);
+
     let zmodelPath = path.join(projectRoot, 'schema.zmodel');
 
     const files = schema.split(FILE_SPLITTER);
@@ -166,7 +183,7 @@
             let fileContent = file.substring(firstLine + 1);
             if (index === 0) {
                 // The first file is the main schema file
                 zmodelPath = path.join(projectRoot, fileName);
                 if (opt.addPrelude) {
                     // plugin need to be added after import statement
                     fileContent = `${fileContent}\n${makePrelude(opt)}`;
@@ -181,7 +198,7 @@
         schema = schema.replaceAll('$projectRoot', projectRoot);
         const content = opt.addPrelude ? `${makePrelude(opt)}\n${schema}` : schema;
         if (opt.customSchemaFilePath) {
             zmodelPath = path.join(projectRoot, opt.customSchemaFilePath);
             fs.mkdirSync(path.dirname(zmodelPath), { recursive: true });
             fs.writeFileSync(zmodelPath, content);
         } else {
@@ -189,16 +206,16 @@
         }
     }
 
-    run('npm install');
-
     const outputArg = opt.output ? ` --output ${opt.output}` : '';
 
     if (opt.customSchemaFilePath) {
-        run(`npx zenstack generate --schema ${zmodelPath} --no-dependency-check${outputArg}`, {
+        run(`npx zenstack generate --no-version-check --schema ${zmodelPath} --no-dependency-check${outputArg}`, {
             NODE_PATH: './node_modules',
         });
     } else {
-        run(`npx zenstack generate --no-dependency-check${outputArg}`, { NODE_PATH: './node_modules' });
+        run(`npx zenstack generate --no-version-check --no-dependency-check${outputArg}`, {
+            NODE_PATH: './node_modules',
+        });
     }
 
     if (opt.pushDb) {
@@ -209,10 +226,10 @@
         opt.extraDependencies?.push('@prisma/extension-pulse');
     }
 
-    opt.extraDependencies?.forEach((dep) => {
-        console.log(`Installing dependency ${dep}`);
-        run(`npm install ${dep}`);
-    });
+    if (opt.extraDependencies) {
+        console.log(`Installing dependency ${opt.extraDependencies.join(' ')}`);
+        installPackage(opt.extraDependencies.join(' '));
+    }
 
     opt.copyDependencies?.forEach((dep) => {
         const pkgJson = JSON.parse(fs.readFileSync(path.join(dep, 'package.json'), { encoding: 'utf-8' }));

script/test-scaffold.ts

@@ -0,0 +1,24 @@
+import path from 'path';
+import fs from 'fs';
+import { execSync } from 'child_process';
+
+const scaffoldPath = path.join(__dirname, '../.test/scaffold');
+if (fs.existsSync(scaffoldPath)) {
+    fs.rmSync(scaffoldPath, { recursive: true, force: true });
+}
+fs.mkdirSync(scaffoldPath, { recursive: true });
+
+function run(cmd: string) {
+    console.log(`Running: ${cmd}, in ${scaffoldPath}`);
+    try {
+        execSync(cmd, { cwd: scaffoldPath, stdio: 'ignore' });
+    } catch (err) {
+        console.error(`Test project scaffolding cmd error: ${err}`);
+        throw err;
+    }
+}
+
+run('npm init -y');
+run('npm i --no-audit --no-fund typescript prisma @prisma/client zod decimal.js');
+
+console.log('Test scaffold setup complete.');

test-setup.ts

@@ -0,0 +1,9 @@
+import fs from 'fs';
+import path from 'path';
+
+export default function globalSetup() {
+    if (!fs.existsSync(path.join(__dirname, '.test/scaffold/package-lock.json'))) {
+        console.error(`Test scaffold not found. Please run \`pnpm test-scaffold\` first.`);
+        process.exit(1);
+    }
+}

tests/integration/jest.config.ts

@@ -1,30 +1,10 @@
+import baseConfig from '../../jest.config';
+
 /*
  * For a detailed explanation regarding each configuration property and type check, visit:
  * https://jestjs.io/docs/configuration
  */
 export default {
-    // Automatically clear mock calls, instances, contexts and results before every test
-    clearMocks: true,
-
-    // A map from regular expressions to paths to transformers
-    transform: { '^.+\\.tsx?$': 'ts-jest' },
-
-    testTimeout: 300000,
-
+    ...baseConfig,
     setupFilesAfterEnv: ['./test-setup.ts'],
-
-    // Indicates whether the coverage information should be collected while executing the test
-    collectCoverage: true,
-
-    // The directory where Jest should output its coverage files
-    coverageDirectory: 'tests/coverage',
-
-    // An array of regexp pattern strings used to skip coverage collection
-    coveragePathIgnorePatterns: ['/node_modules/', '/tests/'],
-
-    // Indicates which provider should be used to instrument code for coverage
-    coverageProvider: 'v8',
-
-    // A list of reporter names that Jest uses when writing coverage reports
-    coverageReporters: ['json', 'text', 'lcov', 'clover'],
 };

tests/integration/tests/cli/generate.test.ts

@@ -1,11 +1,11 @@
 /* eslint-disable @typescript-eslint/no-var-requires */
 /// <reference types="@types/jest" />
 
+import { installPackage } from '@zenstackhq/testtools';
 import * as fs from 'fs';
 import path from 'path';
 import * as tmp from 'tmp';
 import { createProgram } from '../../../../packages/schema/src/cli';
-import { execSync } from '../../../../packages/schema/src/utils/exec-utils';
 import { createNpmrc } from './share';
 
 describe('CLI generate command tests', () => {
@@ -43,8 +43,8 @@ model Post {
         // set up project
         fs.writeFileSync('package.json', JSON.stringify({ name: 'my app', version: '1.0.0' }));
         createNpmrc();
-        execSync('npm install prisma @prisma/client zod');
-        execSync(`npm install ${path.join(__dirname, '../../../../packages/runtime/dist')}`);
+        installPackage('prisma @prisma/client zod');
+        installPackage(path.join(__dirname, '../../../../packages/runtime/dist'));
 
         // set up schema
         fs.writeFileSync('schema.zmodel', MODEL, 'utf-8');

tests/integration/tests/cli/plugins.test.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-var-requires */
 /// <reference types="@types/jest" />
 
-import { getWorkspaceNpmCacheFolder, run } from '@zenstackhq/testtools';
+import { getWorkspaceNpmCacheFolder, installPackage, run } from '@zenstackhq/testtools';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as tmp from 'tmp';
@@ -94,8 +94,8 @@ describe('CLI Plugins Tests', () => {
 
         switch (pm) {
             case 'npm':
-                run('npm install ' + deps);
-                run('npm install -D ' + devDeps);
+                installPackage(deps);
+                installPackage(devDeps, true);
                 break;
             // case 'yarn':
             //     run('yarn add ' + deps);

tests/integration/tests/schema/todo.zmodel

@@ -14,7 +14,7 @@ generator js {
 
 plugin zod {
     provider = '@core/zod'
-    preserveTsFiles = true
+    // preserveTsFiles = true
 }
 
 /*