Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: [cxx-parser] Support cached build #14

Merged
merged 3 commits into from
Sep 19, 2023
Merged

feat: [cxx-parser] Support cached build #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9772053
feat: [cxx-parser] Support cached build
littleGnAl Sep 19, 2023
60e27af
++
littleGnAl Sep 19, 2023
7912661
++
littleGnAl Sep 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 235 additions & 0 deletions cxx-parser/__tests__/unit_test/cxx_parser.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
import fs from "fs";
import path from "path";
import os from 'os';

import { execSync } from "child_process";

import { dumpCXXAstJson, generateChecksum } from "../../src/cxx_parser";
import { TerraContext } from "@agoraio-extensions/terra-core";

jest.mock('child_process');

describe("cxx_parser", () => {
let tmpDir: string = "";
let cppastBackendBuildDir: string = "";

beforeEach(() => {
(execSync as jest.Mock).mockReset();

// Since the test run on the root of the `cxx-parser`, so we can concat the path
// as relative path here.
cppastBackendBuildDir = path.resolve("cxx", "cppast_backend", "build");
// Clean the build dir before each test case.
if (fs.existsSync(cppastBackendBuildDir)) {
fs.rmSync(cppastBackendBuildDir, { recursive: true, force: true });
}

tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'terra-ut-'));
});

afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});

describe("dumpCXXAstJson", () => {
it("generate ast json by default", () => {
let file1Path = path.join(tmpDir, "file1.h");
let file2Path = path.join(tmpDir, "file2.h");

fs.writeFileSync(file1Path, "void file1_main() {}");
fs.writeFileSync(file2Path, "void file2_main() {}");

const expectedJson = JSON.stringify([
{
file_path: '/my/path/IAgoraRtcEngine.h',
nodes: [
{
__TYPE: 'Clazz',
name: 'TestClazz',
methods: [
{
__TYPE: 'MemberFunction',
name: 'test',
parameters: [
{
__TYPE: 'Variable',
name: 'test',
type: {
__TYPE: 'SimpleType',
is_builtin_type: false,
is_const: false,
kind: 101,
name: 'Test',
source: 'Test *',
},
},
],
parent_name: 'TestClazz',
namespaces: ['test'],
},
],
namespaces: ['test'],
},
],
},
]);

let checkSum = generateChecksum([file1Path, file2Path]);
let jsonFilePath = path.join(cppastBackendBuildDir, `dump_json_${checkSum}.json`);

(execSync as jest.Mock).mockImplementationOnce(() => {
// Simulate generate the ast json file after run the bash script
fs.mkdirSync(cppastBackendBuildDir, { recursive: true });
fs.writeFileSync(jsonFilePath, expectedJson);
return "";
});

let json = dumpCXXAstJson(new TerraContext(), [], [], [file1Path, file2Path], []);

let cppastBackendBuildBashPath = path.join(path.resolve(cppastBackendBuildDir, ".."), "build.sh");
let expectedBashScript = `bash ${cppastBackendBuildBashPath} "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --dump-json"`;
expect(execSync).toHaveBeenCalledWith(expectedBashScript, { encoding: "utf8", stdio: "inherit" });
expect(json).toEqual(expectedJson);
});

it("generate ast json with clean", () => {
let file1Path = path.join(tmpDir, "file1.h");
let file2Path = path.join(tmpDir, "file2.h");

fs.writeFileSync(file1Path, "void file1_main() {}");
fs.writeFileSync(file2Path, "void file2_main() {}");

const expectedJson = JSON.stringify([
{
file_path: '/my/path/IAgoraRtcEngine.h',
nodes: [
{
__TYPE: 'Clazz',
name: 'TestClazz',
methods: [
{
__TYPE: 'MemberFunction',
name: 'test',
parameters: [
{
__TYPE: 'Variable',
name: 'test',
type: {
__TYPE: 'SimpleType',
is_builtin_type: false,
is_const: false,
kind: 101,
name: 'Test',
source: 'Test *',
},
},
],
parent_name: 'TestClazz',
namespaces: ['test'],
},
],
namespaces: ['test'],
},
],
},
]);

let checkSum = generateChecksum([file1Path, file2Path]);
let jsonFilePath = path.join(cppastBackendBuildDir, `dump_json_${checkSum}.json`);

let isBuildDirExists = false;

(execSync as jest.Mock).mockImplementationOnce(() => {
// When passing the `TerraContext.clean` as true, the build dir should be removed.
isBuildDirExists = fs.existsSync(cppastBackendBuildDir);

// Simulate generate the ast json file after run the bash script
fs.mkdirSync(cppastBackendBuildDir, { recursive: true });
fs.writeFileSync(jsonFilePath, expectedJson);
return "";
});

let json = dumpCXXAstJson(new TerraContext("", "", true, false), [], [], [file1Path, file2Path], []);

let cppastBackendBuildBashPath = path.join(path.resolve(cppastBackendBuildDir, ".."), "build.sh");
let expectedBashScript = `bash ${cppastBackendBuildBashPath} "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --dump-json"`;
expect(execSync).toHaveBeenCalledWith(expectedBashScript, { encoding: "utf8", stdio: "inherit" });
expect(json).toEqual(expectedJson);
expect(isBuildDirExists).toBe(false);
});

it("generate ast json with cached ast json", () => {
let file1Path = path.join(tmpDir, "file1.h");
let file2Path = path.join(tmpDir, "file2.h");

fs.writeFileSync(file1Path, "void file1_main() {}");
fs.writeFileSync(file2Path, "void file2_main() {}");

const expectedJson = JSON.stringify([
{
file_path: '/my/path/IAgoraRtcEngine.h',
nodes: [
{
__TYPE: 'Clazz',
name: 'TestClazz',
methods: [
{
__TYPE: 'MemberFunction',
name: 'test',
parameters: [
{
__TYPE: 'Variable',
name: 'test',
type: {
__TYPE: 'SimpleType',
is_builtin_type: false,
is_const: false,
kind: 101,
name: 'Test',
source: 'Test *',
},
},
],
parent_name: 'TestClazz',
namespaces: ['test'],
},
],
namespaces: ['test'],
},
],
},
]);

fs.mkdirSync(cppastBackendBuildDir, { recursive: true });
let checkSum = generateChecksum([file1Path, file2Path]);
let jsonFilePath = path.join(cppastBackendBuildDir, `dump_json_${checkSum}.json`);
// Simulate cached ast json file exists
fs.writeFileSync(jsonFilePath, expectedJson);

(execSync as jest.Mock).mockImplementationOnce(() => {
return "";
});

let json = dumpCXXAstJson(new TerraContext(), [], [], [file1Path, file2Path], []);

expect(execSync).not.toHaveBeenCalled();
expect(json).toEqual(expectedJson);
});
});

it("generateChecksum can generate checksum", () => {
let file1Path = path.join(tmpDir, "file1.h");
let file2Path = path.join(tmpDir, "file2.h");

fs.writeFileSync(file1Path, "void file1_main() {}");
fs.writeFileSync(file2Path, "void file2_main() {}");

let res = generateChecksum([file1Path, file2Path]);

// The md5 of the string "void file1_main() {}\nvoid file2_main() {}", can
// be easily calculated by online tools.
let expectedChecksum = "43180edcbadc7e88ed2d6255c65ab9d2";

expect(res).toEqual(expectedChecksum);
});
});
6 changes: 6 additions & 0 deletions cxx-parser/jest.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testPathIgnorePatterns: ['node_modules/', '.dist/'],
testRegex: '.*\\.test\\.ts$',
};
9 changes: 7 additions & 2 deletions cxx-parser/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@agoraio-extensions/cxx-parser",
"version": "0.1.6",
"version": "0.1.7",
"description": "",
"main": "src/index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
"test": "jest"
},
"author": "",
"license": "ISC",
Expand All @@ -17,5 +17,10 @@
"ts-node": "^10.9.1",
"yaml": "^2.1.3",
"glob": "^10.3.4"
},
"devDependencies": {
"@types/jest": "^29.5.1",
"jest": "^29.5.0",
"ts-jest": "^29.1.0"
}
}
26 changes: 25 additions & 1 deletion cxx-parser/src/cxx_parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,19 @@ import {
} from "@agoraio-extensions/terra-core";
import { CXXFile, CXXTYPE, cast } from "./cxx_terra_node";
import { CXXParserConfigs } from "./cxx_parser_configs";
import crypto from "crypto";

export function generateChecksum(files: string[]) {
let allFileContents = files.map((it) => {
return fs.readFileSync(it, "utf-8");
}).join("\n");

return crypto
.createHash('md5')
.update(allFileContents)
.digest('hex')
.toString();
}

export function dumpCXXAstJson(
terraContext: TerraContext,
Expand All @@ -16,6 +29,9 @@ export function dumpCXXAstJson(
parseFiles: string[],
defines: string[]
): string {

let parseFilesChecksum = generateChecksum(parseFiles);

let agora_rtc_ast_dir_path = path.join(
__dirname,
"..",
Expand All @@ -25,12 +41,20 @@ export function dumpCXXAstJson(

let build_shell_path = path.join(agora_rtc_ast_dir_path, "build.sh");
let build_cache_dir_path = path.join(agora_rtc_ast_dir_path, "build");
let outputJsonPath = path.join(build_cache_dir_path, "dump_json.json");
let outputJsonFileName = `dump_json_${parseFilesChecksum}.json`;
let outputJsonPath = path.join(build_cache_dir_path, outputJsonFileName);

if (terraContext.clean && fs.existsSync(build_cache_dir_path)) {
fs.rmSync(build_cache_dir_path, { recursive: true, force: true });
}

// If the previous output json cache exists, skip the process of cppast parser
if (fs.existsSync(outputJsonPath)) {
console.log(`Skip the process of cppast parser, use the cached ast json file: ${outputJsonPath}`);
let ast_json_file_content = fs.readFileSync(outputJsonPath, "utf-8");
return ast_json_file_content;
}

let include_header_dirs_arg = includeHeaderDirs.join(",");
let visit_headers_arg = parseFiles.join(",");

Expand Down
4 changes: 2 additions & 2 deletions cxx-parser/tsconfig.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@

/* Modules */
"module": "commonjs", /* Specify what module code is generated. */
"rootDir": "src", /* Specify the root folder within your source files. */
// "rootDir": "src", /* Specify the root folder within your source files. */
// "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
"rootDirs": ["./src", "__tests__/unit_test"], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
Expand Down