From f5721264c61445291b6bf592a5bc442e28f539ef Mon Sep 17 00:00:00 2001 From: Littlegnal <8847263+littleGnAl@users.noreply.github.com> Date: Tue, 10 Oct 2023 19:08:03 +0800 Subject: [PATCH] feat: Parse struct constructor initializer list using clang CLI (#19) At this time, only parse the struct constructors, but not the assign constructors, and move constructors. --- .../cxx_parser.integration.test.ts | 58 +- .../constructor_initializer_parser.test.ts | 774 ++++++++++++++++++ cxx-parser/cxx/cppast_backend/build.sh | 9 +- .../src/constructor_initializer_parser.ts | 495 +++++++++++ cxx-parser/src/cxx_parser.ts | 40 +- cxx-parser/src/cxx_parser_configs.ts | 8 +- cxx-parser/src/cxx_terra_node.ts | 27 +- package.json | 2 +- run_ast.sh | 5 + terra-core/src/index.ts | 1 + terra-core/src/testing_utils.ts | 37 + tsconfig.json | 3 +- 12 files changed, 1443 insertions(+), 16 deletions(-) create mode 100644 cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts create mode 100644 cxx-parser/src/constructor_initializer_parser.ts create mode 100644 run_ast.sh create mode 100644 terra-core/src/testing_utils.ts diff --git a/cxx-parser/__integration_test__/cxx_parser.integration.test.ts b/cxx-parser/__integration_test__/cxx_parser.integration.test.ts index 71d4d2e..207c3aa 100644 --- a/cxx-parser/__integration_test__/cxx_parser.integration.test.ts +++ b/cxx-parser/__integration_test__/cxx_parser.integration.test.ts @@ -4,7 +4,8 @@ import path from 'path'; import { TerraContext } from '@agoraio-extensions/terra-core'; -import { dumpCXXAstJson } from '../src/cxx_parser'; +import { CXXParser, dumpCXXAstJson } from '../src/cxx_parser'; +import { CXXFile, Struct } from '../src/cxx_terra_node'; describe('cxx_parser', () => { let tmpDir: string = ''; @@ -27,8 +28,8 @@ describe('cxx_parser', () => { file1Path, ` struct AAA { - int a; -} + int a; +}; ` ); @@ -85,4 +86,55 @@ struct AAA { expect(JSON.parse(json)).toEqual(JSON.parse(expectedJson)); }); }); + + describe('CXXParser', () => { + it('parse Struct with constructor initializer list', () => { + let file1Name = 'file1.h'; + let file1Path = path.join(tmpDir, file1Name); + + fs.writeFileSync( + file1Path, + ` +#pragma once + +namespace ns1 { + struct AAA { + int aaa_; + + AAA(): aaa_(0) {} + AAA(int aaa): aaa_(aaa) {} + }; +} +` + ); + + let args = { + includeHeaderDirs: [], + definesMacros: [], + parseFiles: { include: [file1Name] }, + customHeaders: [], + }; + + let parseResult = CXXParser( + new TerraContext(tmpDir, tmpDir), + args, + undefined + )!; + + let s = (parseResult.nodes[0] as CXXFile).nodes[0] as Struct; + expect(s.constructors.length).toBe(2); + + expect(s.constructors[0].initializerList.length).toBe(1); + expect(s.constructors[0].initializerList[0].kind).toEqual('Value'); + expect(s.constructors[0].initializerList[0].name).toEqual('aaa_'); + expect(s.constructors[0].initializerList[0].type).toEqual('int'); + expect(s.constructors[0].initializerList[0].values).toEqual(['0']); + + expect(s.constructors[1].initializerList.length).toBe(1); + expect(s.constructors[1].initializerList[0].kind).toEqual('Parameter'); + expect(s.constructors[1].initializerList[0].name).toEqual('aaa_'); + expect(s.constructors[1].initializerList[0].type).toEqual('int'); + expect(s.constructors[1].initializerList[0].values).toEqual(['aaa']); + }); + }); }); diff --git a/cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts b/cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts new file mode 100644 index 0000000..07ae505 --- /dev/null +++ b/cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts @@ -0,0 +1,774 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { ClangASTStructConstructorParser } from '../../src/constructor_initializer_parser'; + +import { genParseResultFromJson } from '../../src/cxx_parser'; +import { + CXXFile, + ConstructorInitializerKind, + Struct, +} from '../../src/cxx_terra_node'; + +describe('constructor_initializer_parser', () => { + let tmpDir: string = ''; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'terra-ut-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + describe('fillCXXTerraNodeConstructor', () => { + it('can fill the struct constructor initializer list', () => { + let cppastBackendPath = path.join( + __dirname, + '..', + '..', + 'cxx', + 'cppast_backend' + ); + + let file1Path = path.join(tmpDir, 'file1.h'); + + fs.writeFileSync( + file1Path, + ` +#pragma once + +namespace ns1 { + struct AAA { + int aaa_; + + AAA(): aaa_(0) {} + AAA(int aaa): aaa_(aaa) {} + }; +} +` + ); + + let cppastJSON = ` + [ + { + "__TYPE":"CXXFile", + "file_path":"${cppastBackendPath}/tmp/file1.h", + "nodes":[ + { + "__TYPE":"Struct", + "attributes":[], + "base_clazzs":[], + "comment":"", + "constructors":[ + { + "__TYPE":"Constructor", + "name":"AAA", + "parameters":[] + }, + { + "__TYPE":"Constructor", + "name":"AAA", + "parameters":[ + { + "__TYPE":"Variable", + "default_value":"", + "is_output":false, + "name":"aaa", + "type":{ + "__TYPE":"SimpleType", + "is_builtin_type":true, + "is_const":false, + "kind":100, + "name":"int", + "source":"int" + } + } + ] + } + ], + "file_path":"${cppastBackendPath}/tmp/file1.h", + "member_variables":[ + { + "__TYPE":"MemberVariable", + "access_specifier":"", + "is_mutable":false, + "name":"aaa_", + "type":{ + "__TYPE":"SimpleType", + "is_builtin_type":true, + "is_const":false, + "kind":100, + "name":"int", + "source":"int" + } + } + ], + "methods":[], + "name":"AAA", + "namespaces":["ns1"], + "parent_name":"ns1", + "source":"" + } + ] + } + ]`; + + let parseResult = genParseResultFromJson(cppastJSON); + + ClangASTStructConstructorParser(tmpDir, [], [file1Path], parseResult); + + let s = (parseResult.nodes[0] as CXXFile).nodes[0] as Struct; + expect(s.constructors.length).toBe(2); + + expect(s.constructors[0].initializerList.length).toBe(1); + expect(s.constructors[0].initializerList[0].kind).toEqual('Value'); + expect(s.constructors[0].initializerList[0].name).toEqual('aaa_'); + expect(s.constructors[0].initializerList[0].type).toEqual('int'); + expect(s.constructors[0].initializerList[0].values).toEqual(['0']); + + expect(s.constructors[1].initializerList.length).toBe(1); + expect(s.constructors[1].initializerList[0].kind).toEqual('Parameter'); + expect(s.constructors[1].initializerList[0].name).toEqual('aaa_'); + expect(s.constructors[1].initializerList[0].type).toEqual('int'); + expect(s.constructors[1].initializerList[0].values).toEqual(['aaa']); + }); + + describe('parseStructConstructors', () => { + const testingUsed = require('../../src/constructor_initializer_parser'); + // Aim to test the private funtion + const parseStructConstructors = testingUsed.parseStructConstructors; + + it('constructor assign enum', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once + +namespace ns1 { + enum AAA_ENUM { + ERR_OK = 0, + ERR_NOT_READY = 3 + }; + + struct AAA { + AAA_ENUM aaa_enum_; + + AAA(): aaa_enum_(ERR_NOT_READY) {} + + AAA(AAA_ENUM aaa_enum): aaa_enum_(aaa_enum) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_enum_', + type: 'ns1::AAA_ENUM', + values: ['ns1::AAA_ENUM::ERR_NOT_READY'], + }, + ], + }, + { + name: 'AAA', + signature: 'void (ns1::AAA_ENUM)', + parameterList: [ + { + kind: ConstructorInitializerKind.Parameter, + name: 'aaa_enum', + type: 'ns1::AAA_ENUM', + values: ['aaa_enum'], + }, + ], + initializerList: [ + { + kind: ConstructorInitializerKind.Parameter, + name: 'aaa_enum_', + type: 'ns1::AAA_ENUM', + values: ['aaa_enum'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('fill constructor assign enum to int', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once + +namespace ns1 { + enum AAA_ENUM { + ERR_OK = 0, + ERR_NOT_READY = 3 + }; + + struct AAA { + int aaa_; + + AAA(): aaa_(-ERR_NOT_READY) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'int', + values: ['-ns1::AAA_ENUM::ERR_NOT_READY'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor assign NULL to typedef type', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once +#include + +namespace ns1 { + typedef void* view_t; + + struct AAA { + view_t aaa_; + + AAA(): aaa_(NULL) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'ns1::view_t', + values: ['NULL'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor assign pointer with std nullptr', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once +#include + +namespace ns1 { + struct AAA { + void *aaa_; + + AAA(): aaa_(nullptr) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'void *', + values: ['std::nullptr_t'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor assign float value', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once +#include + +namespace ns1 { + struct AAA { + float aaa_; + + AAA(): aaa_(0.0) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'float', + values: ['0'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor assign double value', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once +#include + +namespace ns1 { + struct AAA { + double aaa_; + + AAA(): aaa_(0.0) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'double', + values: ['0'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor assign int with negative value', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once +#include + +namespace ns1 { + struct AAA { + int aaa_; + + AAA(): aaa_(-1) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'int', + values: ['-1'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor assign bool value', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once +#include + +namespace ns1 { + struct AAA { + bool aaa_; + + AAA(): aaa_(false) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'bool', + values: ['false'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor assign uint32_t with hex value', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once +#include + +namespace ns1 { + struct AAA { + uint32_t aaa_; + + AAA(): aaa_(0x00000000) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'uint32_t', + values: ['0'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor assign with struct construct', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once + +namespace ns1 { + struct Rectangle { + int x; + int y; + int width; + int height; + + Rectangle(int xx, int yy, int ww, int hh) : x(xx), y(yy), width(ww), height(hh) {} + }; + + struct AAA { + Rectangle aaa_; + + AAA(): aaa_(0, 0, 0, 0) {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::Rectangle', + constructors: [ + { + name: 'Rectangle', + signature: 'void (int, int, int, int)', + parameterList: [ + { + kind: ConstructorInitializerKind.Parameter, + name: 'xx', + type: 'int', + values: ['xx'], + }, + { + kind: ConstructorInitializerKind.Parameter, + name: 'yy', + type: 'int', + values: ['yy'], + }, + { + kind: ConstructorInitializerKind.Parameter, + name: 'ww', + type: 'int', + values: ['ww'], + }, + { + kind: ConstructorInitializerKind.Parameter, + name: 'hh', + type: 'int', + values: ['hh'], + }, + ], + initializerList: [ + { + kind: ConstructorInitializerKind.Parameter, + name: 'x', + type: 'int', + values: ['xx'], + }, + { + kind: ConstructorInitializerKind.Parameter, + name: 'y', + type: 'int', + values: ['yy'], + }, + { + kind: ConstructorInitializerKind.Parameter, + name: 'width', + type: 'int', + values: ['ww'], + }, + { + kind: ConstructorInitializerKind.Parameter, + name: 'height', + type: 'int', + values: ['hh'], + }, + ], + }, + ], + }, + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Construct, + name: 'aaa_', + type: 'ns1::Rectangle', + values: ['0', '0', '0', '0'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('constructor with multiple nested namespaces', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once + +namespace ns1 { +namespace ns2 { + struct AAA { + int aaa_; + + AAA(): aaa_(0) {} + }; +} +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::ns2::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [ + { + kind: ConstructorInitializerKind.Value, + name: 'aaa_', + type: 'int', + values: ['0'], + }, + ], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + }); + }); +}); diff --git a/cxx-parser/cxx/cppast_backend/build.sh b/cxx-parser/cxx/cppast_backend/build.sh index 9f9aea8..106f6ad 100644 --- a/cxx-parser/cxx/cppast_backend/build.sh +++ b/cxx-parser/cxx/cppast_backend/build.sh @@ -14,11 +14,18 @@ fi pushd ${OUTPUT_PATH} +LLVM_CONFIG_BINARY=$(which llvm-config) +if [[ ! -z "${LLVM_DOWNLOAD_URL}" ]]; then + echo "Use the llvm from the url: ${LLVM_DOWNLOAD_URL}" + # Use the llvm from the url instead of the system installed one + LLVM_CONFIG_BINARY="" +fi + # set LLVM_DOWNLOAD_URL env like # linux: https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz # macos: https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.7/clang+llvm-15.0.7-x86_64-apple-darwin21.0.tar.xz cmake \ - -DLLVM_CONFIG_BINARY=$(which llvm-config) \ + -DLLVM_CONFIG_BINARY=${LLVM_CONFIG_BINARY} \ -DLLVM_DOWNLOAD_URL=${LLVM_DOWNLOAD_URL} \ -DRUNTIME_OUTPUT_DIRECTORY=${OUTPUT_PATH} \ ${MY_PATH} diff --git a/cxx-parser/src/constructor_initializer_parser.ts b/cxx-parser/src/constructor_initializer_parser.ts new file mode 100644 index 0000000..a08988e --- /dev/null +++ b/cxx-parser/src/constructor_initializer_parser.ts @@ -0,0 +1,495 @@ +import { strict as assert } from 'assert'; + +import { execSync } from 'child_process'; +import * as fs from 'fs'; +import path from 'path'; + +import { ParseResult, visibleForTesting } from '@agoraio-extensions/terra-core'; + +import { generateChecksum } from './cxx_parser'; +import { + CXXFile, + CXXTYPE, + Constructor, + ConstructorInitializer, + ConstructorInitializerKind, + SimpleType, + Struct, +} from './cxx_terra_node'; + +enum TagUsedType { + struct_t = 'struct', +} + +/** + * The node type: + * https://clang.llvm.org/doxygen/Decl_8h_source.html + */ +enum ClangASTNodeKind { + TranslationUnitDecl = 'TranslationUnitDecl', + NamespaceDecl = 'NamespaceDecl', + TypedefDecl = 'TypedefDecl', + CXXRecordDecl = 'CXXRecordDecl', + FieldDecl = 'FieldDecl', + FullComment = 'FullComment', + ParagraphComment = 'ParagraphComment', + TextComment = 'TextComment', + InlineCommandComment = 'InlineCommandComment', + CXXConstructorDecl = 'CXXConstructorDecl', + CXXCtorInitializer = 'CXXCtorInitializer', + CXXBoolLiteralExpr = 'CXXBoolLiteralExpr', + ParmVarDecl = 'ParmVarDecl', + ImplicitCastExpr = 'ImplicitCastExpr', + CXXConstructExpr = 'CXXConstructExpr', + GNUNullExpr = 'GNUNullExpr', + CXXNullPtrLiteralExpr = 'CXXNullPtrLiteralExpr', + IntegerLiteral = 'IntegerLiteral', + FloatingLiteral = 'FloatingLiteral', + DeclRefExpr = 'DeclRefExpr', + EnumConstantDecl = 'EnumConstantDecl', + UnaryOperator = 'UnaryOperator', +} + +/** + * Intermediate objects used internally + */ +interface _FlattenNode { + ns: string; // namespace + node: any; +} + +/** + * Intermediate objects used internally + */ +interface _NameValueTypeHolder { + kind: ConstructorInitializerKind; + name: string; + type: string; + values: string[]; +} + +/** + * Intermediate objects used internally + */ +class _ConstructorInitializer { + name: string = ''; + signature: string = ''; + parameterList: _NameValueTypeHolder[] = []; + initializerList: _NameValueTypeHolder[] = []; +} + +/** + * Intermediate objects used internally + */ +class _StructConstructors { + name: string = ''; + constructors: _ConstructorInitializer[] = []; +} + +/** + * This function mainly use the clang command line tool to dump the AST in json format, + * ``` + * clang++ -fsyntax-only -Xclang -ast-dump=json \ + * -I /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include \ + * -I /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1/ + * > output.json + * ``` + * More detail: https://clang.llvm.org/docs/IntroductionToTheClangAST.html + * + * For debug, you can dump the normal AST format which is more readable, + * ``` + * clang++ -fsyntax-only -Xclang -ast-dump -fno-color-diagnostics \ + * -I /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include \ + * -I /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1/ \ + * > output.txt + * ```` + */ +function dumpClangASTJSON( + buildDir: string, + includeHeaderDirs: string[], + parseFile: string +): string { + let fileName = path.basename(parseFile); + let checksum = generateChecksum([parseFile]); + + // dump_clang_ast__.json + let clangAstJsonPath = path.join( + buildDir, + `dump_clang_ast_${fileName}_${checksum}.json` + ); + + // If the cache found, just return it. + // Note that we are no need to handle the `TerraContext.clean` in this function, since if the + // clean flag passed from the command line, the `CXXParser` will handle it first, the build dir + // will be cleaned. + if (fs.existsSync(clangAstJsonPath)) { + return fs.readFileSync(clangAstJsonPath, 'utf-8'); + } + + let includeHeaderDirsNew = [ + // For macos + '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include', + '/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include', + // For linux + '/usr/local/include', + ...includeHeaderDirs, + ]; + let includeHeaderDirsArgs = includeHeaderDirsNew.map((it) => { + return `-I ${it}`; + }); + + let args = ['-Xclang', '-ast-dump=json', '-fsyntax-only']; + let _args = [...args, ...includeHeaderDirsArgs, parseFile].join(' '); + + let bashScript = `clang++ ${_args} > ${clangAstJsonPath}`; + console.log(`Running command: \n${bashScript}`); + + execSync(bashScript, { encoding: 'utf8', stdio: 'inherit' }); + + let ast_json_file_content = fs.readFileSync(clangAstJsonPath, 'utf-8'); + return ast_json_file_content; +} + +function _parseStructConstructors( + buildDir: string, + includeHeaderDirs: string[], + parseFile: string +): _StructConstructors[] { + let astJson = dumpClangASTJSON(buildDir, includeHeaderDirs, parseFile); + let nodes = filterAndFlattenNodes(parseFile, astJson); + + // Find all `CXXRecordDecl` nodes in "struct" tag + let allStructs = nodes.filter((node: _FlattenNode) => { + return ( + node.node.kind == ClangASTNodeKind.CXXRecordDecl && + node.node.tagUsed == TagUsedType.struct_t + ); + }); + + let _structConstructors: _StructConstructors[] = []; + + for (let s of allStructs) { + let structConstructor: _StructConstructors = {} as _StructConstructors; + structConstructor.name = s.ns ? `${s.ns}::${s.node.name}` : s.node.name; + + // Find all `CXXConstructorDecl` nodes in `cxxRecordDecls`'s `inner` + let cxxConstructorDecls = s.node.inner.filter((node: any) => { + return ( + node.kind == ClangASTNodeKind.CXXConstructorDecl && + !node.isImplicit /* filter the implicited generated constructors */ + ); + }); + + let constructorInitializers: _ConstructorInitializer[] = []; + + for (let cxxConstructorDecl of cxxConstructorDecls) { + let constructorInitializer = new _ConstructorInitializer(); + constructorInitializer.name = cxxConstructorDecl.name; + constructorInitializer.signature = cxxConstructorDecl.type.qualType; + + // Find all `ParmVarDecl` nodes in `cxxConstructorDecl`'s `inner` + let parmVarDecls = cxxConstructorDecl.inner.filter((node: any) => { + return node.kind == ClangASTNodeKind.ParmVarDecl; + }); + parmVarDecls.forEach((parmVarDecl: any) => { + let nameValueTypeHolder = {} as _NameValueTypeHolder; + nameValueTypeHolder.kind = ConstructorInitializerKind.Parameter; + nameValueTypeHolder.name = parmVarDecl.name; + nameValueTypeHolder.type = parmVarDecl.type.qualType; + // TODO(littlegnal): Maybe parse the default value + nameValueTypeHolder.values = [parmVarDecl.name]; + constructorInitializer.parameterList.push(nameValueTypeHolder); + }); + + // Find all `CXXCtorInitializer` nodes in `cxxConstructorDecl`'s `inner` + let cxxCtorInitializers = cxxConstructorDecl.inner.filter((node: any) => { + return node.kind == ClangASTNodeKind.CXXCtorInitializer; + }); + cxxCtorInitializers.forEach((cxxCtorInitializer: any) => { + let nameValueTypeHolder = {} as _NameValueTypeHolder; + const [kind, values] = parseInnerValues(cxxCtorInitializer.inner); + + nameValueTypeHolder.kind = parseConstructorInitializerKind( + cxxCtorInitializer.kind, + kind + ); + nameValueTypeHolder.name = cxxCtorInitializer.anyInit.name; + nameValueTypeHolder.type = cxxCtorInitializer.anyInit.type.qualType; + nameValueTypeHolder.values = values; + constructorInitializer.initializerList.push(nameValueTypeHolder); + }); + + constructorInitializers.push(constructorInitializer); + } + + structConstructor.constructors = constructorInitializers; + + _structConstructors.push(structConstructor); + } + + return _structConstructors; +} + +function parseStructConstructors( + buildDir: string, + includeHeaderDirs: string[], + parseFiles: string[] +): _StructConstructors[] { + return parseFiles + .map((it) => { + return _parseStructConstructors(buildDir, includeHeaderDirs, it); + }) + .flat(1); +} + +function parseConstructorInitializerKind( + kind: string, + valueKind: ConstructorInitializerKind +) { + switch (kind) { + case ClangASTNodeKind.CXXConstructExpr: + return ConstructorInitializerKind.Construct; + default: + return valueKind; + } +} + +function parseReferencedDeclValue( + referencedDecl: any +): [ConstructorInitializerKind, string] { + let kind = ConstructorInitializerKind.Value; + switch (referencedDecl.kind) { + case ClangASTNodeKind.EnumConstantDecl: + // e.g., + // "referencedDecl": { + // "id": "0x7fe0510ca138", + // "kind": "EnumConstantDecl", + // "name": "RENDER_MODE_HIDDEN", + // "type": { + // "qualType": "agora::media::base::RENDER_MODE_TYPE" + // } + // } + return [kind, `${referencedDecl.type.qualType}::${referencedDecl.name}`]; + case ClangASTNodeKind.ParmVarDecl: + // e.g., + // "referencedDecl": { + // "id": "0x7fe0549e5218", + // "kind": "ParmVarDecl", + // "name": "v", + // "type": { + // "desugaredQualType": "void *", + // "qualType": "agora::view_t", + // "typeAliasDeclId": "0x7fe0505c0de8" + // } + // } + kind = ConstructorInitializerKind.Parameter; + return [kind, referencedDecl.name]; + default: + return [kind, referencedDecl.name]; + } +} + +// Parse the values of the `inner` nodes +function parseInnerValues(inner: any): [ConstructorInitializerKind, string[]] { + let values: string[] = []; + let kind = ConstructorInitializerKind.Value; + + let nodes = inner; + for (let node of nodes) { + switch (node.kind) { + case ClangASTNodeKind.CXXBoolLiteralExpr: + case ClangASTNodeKind.IntegerLiteral: + // The `float/double` is `FloatingLiteral` + // TODO(littlegnal): The float value `0.0` is returned `0`, maybe substring the float/double value + // from the source, so we can get the exact value of the source. + case ClangASTNodeKind.FloatingLiteral: + // Get the value from `value` field + values.push(`${node.value}`); + break; + case ClangASTNodeKind.ImplicitCastExpr: { + let [k, vs] = parseInnerValues(node.inner); + kind = k; + values.push(...vs); + break; + } + case ClangASTNodeKind.GNUNullExpr: + values.push('NULL'); + break; + case ClangASTNodeKind.CXXNullPtrLiteralExpr: + // std::nullptr_t + values.push('std::nullptr_t'); + break; + case ClangASTNodeKind.DeclRefExpr: { + // e.g., + // "referencedDecl": { + // "id": "0x7fe0510ca138", + // "kind": "EnumConstantDecl", + // "name": "RENDER_MODE_HIDDEN", + // "type": { + // "qualType": "agora::media::base::RENDER_MODE_TYPE" + // } + // } + // + // "referencedDecl": { + // "id": "0x7fe0549e5218", + // "kind": "ParmVarDecl", + // "name": "v", + // "type": { + // "desugaredQualType": "void *", + // "qualType": "agora::view_t", + // "typeAliasDeclId": "0x7fe0505c0de8" + // } + // } + const [k, v] = parseReferencedDeclValue(node.referencedDecl); + kind = k; + values.push(v); + break; + } + case ClangASTNodeKind.UnaryOperator: { + let opcode = node.opcode; + let [_, vs] = parseInnerValues(node.inner); + let finalV = `${opcode}${vs[0]}`; + values.push(finalV); + break; + } + case ClangASTNodeKind.CXXConstructExpr: { + if (!node.isImplicit) { + const [_, v] = parseInnerValues(node.inner); + kind = ConstructorInitializerKind.Construct; + values.push(...v); + } + + break; + } + default: + break; + } + } + + return [kind, values]; +} + +export function ClangASTStructConstructorParser( + buildDir: string, + includeHeaderDirs: string[], + parseFiles: string[], + parseResult: ParseResult +) { + function _signature(constructor: Constructor): string { + let parameterTypeList = constructor.parameters + .map((it) => { + function _type2String(type: SimpleType): string { + if (type.namespaces?.length > 0) { + return `${type.namespace}::${type.realName}`; + } + return type.name; + } + return _type2String(it.type); + }) + .join(', '); + + return `void (${parameterTypeList})`; + } + + let constructorInitializers = parseStructConstructors( + buildDir, + includeHeaderDirs, + parseFiles + ); + let constructorInitializersMap = constructorInitializers.reduce( + (acc, item) => acc.set(item.name, item), + new Map() + ); + + parseResult.nodes.forEach((f: any) => { + let cxxFile = f as CXXFile; + + for (let n of cxxFile.nodes) { + if (n.__TYPE == CXXTYPE.Struct) { + let node = n as Struct; + let fullName = node.fullName; + let structConstructor = constructorInitializersMap.get(fullName); + if (!structConstructor) { + continue; + } + + for (let c of node.constructors) { + let constructorSignature = _signature(c); + + let foundStructConstructor = structConstructor?.constructors.find( + (it) => { + return it.signature == constructorSignature; + } + ); + + if (foundStructConstructor) { + let initializerList = foundStructConstructor.initializerList.map( + (it) => { + let ci = new ConstructorInitializer(); + ci.kind = it.kind; + ci.name = it.name; + ci.type = it.type; + ci.values = it.values; + + return ci; + } + ); + + c.initializerList = initializerList; + } + } + } + } + }); +} + +function filterAndFlattenNodes( + parseFiles: string, + astJson: string +): Array<_FlattenNode> { + let jsonObj = JSON.parse(astJson); + // The first object kind is `TranslationUnitDecl` + assert(jsonObj.kind == ClangASTNodeKind.TranslationUnitDecl); + // `inner` should have values + assert(jsonObj.inner && Array.isArray(jsonObj.inner)); + + function _flattenNodes( + inner: any, + nsStack: string[], + filterByFile: boolean = true + ): Array<_FlattenNode> { + let res = new Array<_FlattenNode>(); + + for (let n of inner) { + if (filterByFile) { + if (!n.loc?.file) { + continue; + } + + if (!parseFiles.includes(n.loc.file)) { + continue; + } + } + + switch (n.kind) { + case ClangASTNodeKind.NamespaceDecl: + nsStack.push(n.name); + res.push(..._flattenNodes(n.inner, nsStack, false)); + nsStack.pop(); + break; + default: + res.push({ ns: nsStack.join('::'), node: n }); + break; + } + } + + return res; + } + + let res = _flattenNodes(jsonObj.inner, []); + + return res; +} + +// Add functions that visible for testing here. +visibleForTesting(module, parseStructConstructors); diff --git a/cxx-parser/src/cxx_parser.ts b/cxx-parser/src/cxx_parser.ts index a017fa6..e99fa3b 100644 --- a/cxx-parser/src/cxx_parser.ts +++ b/cxx-parser/src/cxx_parser.ts @@ -5,6 +5,7 @@ import path from 'path'; import { ParseResult, TerraContext } from '@agoraio-extensions/terra-core'; +import { ClangASTStructConstructorParser } from './constructor_initializer_parser'; import { CXXParserConfigs } from './cxx_parser_configs'; import { CXXFile, CXXTYPE, cast } from './cxx_terra_node'; @@ -22,6 +23,11 @@ export function generateChecksum(files: string[]) { .toString(); } +function getBuildDir(terraContext: TerraContext) { + // /.terra/cxx_parser + return path.join(terraContext.buildDir, 'cxx_parser'); +} + export function dumpCXXAstJson( terraContext: TerraContext, includeHeaderDirs: string[], @@ -31,8 +37,7 @@ export function dumpCXXAstJson( ): string { let parseFilesChecksum = generateChecksum(parseFiles); - // /.terra/cxx_parser - let buildDir = path.join(terraContext.buildDir, 'cxx_parser'); + let buildDir = getBuildDir(terraContext); let agora_rtc_ast_dir_path = path.join( __dirname, @@ -83,7 +88,17 @@ export function dumpCXXAstJson( let buildScript = `bash ${build_shell_path} \"${buildDir}\" \"${bashArgs}\"`; console.log(`Running command: \n${buildScript}`); - execSync(buildScript, { encoding: 'utf8', stdio: 'inherit' }); + try { + execSync(buildScript, { encoding: 'utf8', stdio: 'inherit' }); + } catch (e: any) { + console.log(`Failed to run command: \n${buildScript}`); + console.log(`status: ${e.status}`); + console.log(`stderr: ${e.stderr}`); + console.log(`stdout: ${e.stdout}`); + console.log(`message: ${e.message}`); + // console.error(e); + throw e; + } let ast_json_file_content = fs.readFileSync(outputJsonPath, 'utf-8'); return ast_json_file_content; @@ -108,7 +123,7 @@ export function genParseResultFromJson(astJsonContent: string): ParseResult { export function CXXParser( terraContext: TerraContext, args: any, - parseResult?: ParseResult + _?: ParseResult ): ParseResult | undefined { let cxxParserConfigs = CXXParserConfigs.resolve(terraContext.configDir, args); @@ -123,7 +138,22 @@ export function CXXParser( cxxParserConfigs.definesMacros ); - return genParseResultFromJson(jsonContent); + let newParseResult = genParseResultFromJson(jsonContent); + + // Use the parsed file path from cppast parser to avoid additional operations for the file, + // e.g., the macros operations + let cppastParsedFiles = newParseResult.nodes.map((it) => { + return (it as CXXFile).file_path; + }); + + ClangASTStructConstructorParser( + getBuildDir(terraContext), + cxxParserConfigs.includeHeaderDirs, + cppastParsedFiles, + newParseResult + ); + + return newParseResult; } function fillParentNode(cxxFiles: CXXFile[]) { diff --git a/cxx-parser/src/cxx_parser_configs.ts b/cxx-parser/src/cxx_parser_configs.ts index 2ec2bd2..b06ed77 100644 --- a/cxx-parser/src/cxx_parser_configs.ts +++ b/cxx-parser/src/cxx_parser_configs.ts @@ -24,25 +24,25 @@ export interface CXXParserConfigs { export class CXXParserConfigs { static resolve(configDir: any, original: CXXParserConfigs): CXXParserConfigs { return { - includeHeaderDirs: original.includeHeaderDirs + includeHeaderDirs: (original.includeHeaderDirs ?? []) .map((it) => { return globSync(resolvePath(it, configDir)); }) .flat(1), definesMacros: original.definesMacros ?? [], parseFiles: { - include: original.parseFiles.include + include: (original.parseFiles.include ?? []) .map((it) => { return globSync(resolvePath(it, configDir)); }) .flat(1), - exclude: original.parseFiles.exclude + exclude: (original.parseFiles.exclude ?? []) .map((it) => { return globSync(resolvePath(it, configDir)); }) .flat(1), }, - customHeaders: original.customHeaders + customHeaders: (original.customHeaders ?? []) .map((it) => { return globSync(resolvePath(it, configDir)); }) diff --git a/cxx-parser/src/cxx_terra_node.ts b/cxx-parser/src/cxx_terra_node.ts index 200de62..c9ef8fd 100644 --- a/cxx-parser/src/cxx_terra_node.ts +++ b/cxx-parser/src/cxx_terra_node.ts @@ -1,6 +1,7 @@ import path from 'path'; import { TerraNode } from '@agoraio-extensions/terra-core'; +import './cxx_terra_node_ext'; function getAllClazzs(cxxfiles: CXXFile[]): Clazz[] { return cxxfiles.flatMap((file) => @@ -46,7 +47,7 @@ export abstract class CXXTerraNode implements TerraNode { source: string = ''; user_data?: any = undefined; - get fullName(): string { + public get fullName(): string { if (this.namespaces?.length > 0) { return `${this.namespace}::${this.realName}`; } @@ -160,9 +161,31 @@ export class TypeAlias extends CXXTerraNode { underlyingType: SimpleType = new SimpleType(); } +export enum ConstructorInitializerKind { + Parameter = 'Parameter', + Value = 'Value', + Construct = 'Construct', +} + +export class ConstructorInitializer { + kind: ConstructorInitializerKind = ConstructorInitializerKind.Value; + name: string = ''; + type: string = ''; // Maybe change the type to the `SimpleType` in the furture + // If the kind is `ConstructorInitializerKind.Parameter`, the `values`'s length is 1, + // the `values[0]` is the parameter name of the constructor + // + // If the kind is `ConstructorInitializerKind.Value`, the `values`'s length is 1, + // the `values[0]` is the value of the initializer + // + // If the kind is `ConstructorInitializerKind.Construct`, the `values`'s length is the + // constructor parameter's length of the `type`. + values: string[] = []; +} + export class Constructor extends CXXTerraNode { override __TYPE: CXXTYPE = CXXTYPE.Constructor; parameters: Variable[] = []; + initializerList: ConstructorInitializer[] = []; } export class Clazz extends CXXTerraNode { @@ -238,6 +261,8 @@ export class SimpleType extends CXXTerraNode { return this.source?.trimNamespace(); } + // TODO(lxh): Remove this custom logic, this function should return the common full name + // in C++ way: :: override get fullName(): string { if (this.parent?.__TYPE === CXXTYPE.MemberFunction) { return `${this.parent?.fullName}@return_type`; diff --git a/package.json b/package.json index 7cc98ee..bce7d5f 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "build": "tsc", "lint": "eslint \"**/*.{js,ts,tsx}\"", "test": "jest --coverage --selectProjects=test", - "test:integration": "jest --coverage --selectProjects=integration", + "test:integration": "jest --selectProjects=integration --verbose --useStderr", "test:terra": "jest --selectProjects=terra", "test:terra-core": "jest --selectProjects=terra-core", "test:cxx-parser": "jest --selectProjects=cxx-parser", diff --git a/run_ast.sh b/run_ast.sh new file mode 100644 index 0000000..c03341d --- /dev/null +++ b/run_ast.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e +set -x + +clang++ -fsyntax-only -Xclang -ast-dump=json -o /Users/fenglang/codes/aw/terra_shared_configs/aa.json -I /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include -I /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1/ /Users/fenglang/codes/aw/terra/aaa.h > ast.json \ No newline at end of file diff --git a/terra-core/src/index.ts b/terra-core/src/index.ts index d330142..9bc68e0 100644 --- a/terra-core/src/index.ts +++ b/terra-core/src/index.ts @@ -1,5 +1,6 @@ // export * from "./app_root_path"; export * from './path_resolver'; +export * from './testing_utils'; export interface TerraNode {} diff --git a/terra-core/src/testing_utils.ts b/terra-core/src/testing_utils.ts new file mode 100644 index 0000000..96f7f2e --- /dev/null +++ b/terra-core/src/testing_utils.ts @@ -0,0 +1,37 @@ +/** + * Exports a list of functions from the current module for testing purposes when in a test environment. + * + * This function dynamically exports the given functions to the specified module's exports, making them + * available for import in other files. This is particularly useful for exposing functions for + * testing without making them part of the module's public API. + * + * @example + * // In your-module.ts + * import { visibleForTesting } from '@agoraio-extensions/terra-core'; + * + * function privateFunction1() { + * // ... + * } + * + * function privateFunction2() { + * // ... + * } + * + * visibleForTesting(module, privateFunction1, privateFunction2); + * + * // Now privateFunction1 and privateFunction2 can be used from your-module.ts in a test environment. + * + * @param {any} module - The module object where the functions will be exported. + * @param {...Function[]} func - The functions to be exported for testing. + */ +export function visibleForTesting(module: any, ...func: Function[]): void { + if (process.env.NODE_ENV === 'test') { + for (let f of func) { + if (typeof f === 'function' && f.name) { + (module as any).exports[f.name] = f; + } else { + console.error('Invalid function passed to visibleForTesting:', f); + } + } + } +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 89bec60..b7b7b78 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ "module": "commonjs", "strict": true, "sourceMap": true, - "esModuleInterop": true + "esModuleInterop": true, + "rootDirs": ["**/src", "**/__tests__"] } }