From 016edf9cd207de6fe9ac9df8defe4ad570ac2214 Mon Sep 17 00:00:00 2001 From: Littlegnal <8847263+littleGnAl@users.noreply.github.com> Date: Thu, 9 Nov 2023 16:12:48 +0800 Subject: [PATCH] fix: Fix can not parse the c++ std headers when parsing the constructor initializers (#31) --- .../constructor_initializer_parser.test.ts | 102 +++++++++++++++ .../src/constructor_initializer_parser.ts | 120 ++++++++++++++++-- cxx-parser/src/cxx_parser.ts | 11 +- 3 files changed, 215 insertions(+), 18 deletions(-) diff --git a/cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts b/cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts index 07ae505..b5a977d 100644 --- a/cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts +++ b/cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts @@ -140,6 +140,108 @@ namespace ns1 { // Aim to test the private funtion const parseStructConstructors = testingUsed.parseStructConstructors; + it('empty constructor', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once + +namespace ns1 { + struct AAA { + AAA() {} + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [ + { + name: 'AAA', + signature: 'void ()', + parameterList: [], + initializerList: [], + }, + ], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('default constructor', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once + +namespace ns1 { + struct AAA { + AAA() = default; + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + + it('declare struct only', () => { + let filePath = path.join(tmpDir, 'file.h'); + let fileContent = ` +#pragma once + +namespace ns1 { + struct AAANameOnly; + + struct AAA { + AAA() = default; + }; +} +`; + fs.writeFileSync(filePath, fileContent); + + let constructorInitializers = parseStructConstructors( + tmpDir, + [], + [filePath] + ); + + let expectedRes = [ + { + name: 'ns1::AAA', + constructors: [], + }, + ]; + + expect(JSON.stringify(constructorInitializers)).toEqual( + JSON.stringify(expectedRes) + ); + }); + it('constructor assign enum', () => { let filePath = path.join(tmpDir, 'file.h'); let fileContent = ` diff --git a/cxx-parser/src/constructor_initializer_parser.ts b/cxx-parser/src/constructor_initializer_parser.ts index a08988e..f355c1d 100644 --- a/cxx-parser/src/constructor_initializer_parser.ts +++ b/cxx-parser/src/constructor_initializer_parser.ts @@ -6,7 +6,7 @@ import path from 'path'; import { ParseResult, visibleForTesting } from '@agoraio-extensions/terra-core'; -import { generateChecksum } from './cxx_parser'; +import { generateChecksum, getCppAstBackendDir } from './cxx_parser'; import { CXXFile, CXXTYPE, @@ -86,6 +86,66 @@ class _StructConstructors { constructors: _ConstructorInitializer[] = []; } +/** + * + * @returns The default include dirs for clang command line tool + */ +function getDefaultIncludeDirs(): string[] { + // TODO(littlegnal): This implementation is borrowed from cppast to retrive the default include dirs + // https://github.com/foonathan/cppast/blob/f00df6675d87c6983033d270728c57a55cd3db22/src/libclang/libclang_parser.cpp#L287 + // But it seems it's not necessary to add these when running the clang command line tool, so we just keep the code here, and + // see if we need to add these in the future. + // function _findIncludeDirsFromClang(): string[] { + // let verbose_output: string = ''; + // let verboseCommand = 'clang++ -xc++ -v -'; + // try { + // execSync(verboseCommand, { + // input: '', + // stdio: ['pipe', 'pipe', 'pipe'], // Pipe stdin, stdout, and stderr + // encoding: 'utf-8', + // }).toString(); + // } catch (error: any) { + // // We need to use the tricky wayt to get the verbose output from clang + // verbose_output = error.message ?? ''; + // } + + // // If the command failed, return the empty include header dirs + // if (!verbose_output) { + // return []; + // } + + // let verboseOutputInLines = verbose_output.split('\n'); + + // verboseOutputInLines = verboseOutputInLines.slice( + // verboseOutputInLines.findIndex((it) => { + // return it.startsWith('#include <...>'); + // }) + 1, // Do not include the index of '#include <...>' + // verboseOutputInLines.findIndex((it) => { + // return it.startsWith('End of search list.'); + // }) + // ); + // let includeDirs = verboseOutputInLines + // .filter((it) => { + // return it.startsWith(' '); + // }) + // .map((it) => { + // if (it.includes(' (')) { + // // /Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/System/Library/Frameworks (framework directory) + // return it.trim().split(' (')[0]; + // } + + // return it.trim(); + // }); + + // return includeDirs; + // } + + return [ + // Add cxx-parser/cxx/cppast_backend/include/system_fake + path.join(getCppAstBackendDir(), 'include', 'system_fake'), + ]; +} + /** * This function mainly use the clang command line tool to dump the AST in json format, * ``` @@ -126,25 +186,42 @@ function dumpClangASTJSON( return fs.readFileSync(clangAstJsonPath, 'utf-8'); } - let includeHeaderDirsNew = [ - // For macos - '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include', - '/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include', - // For linux - '/usr/local/include', - ...includeHeaderDirs, - ]; + let includeHeaderDirsNew = [...getDefaultIncludeDirs(), ...includeHeaderDirs]; let includeHeaderDirsArgs = includeHeaderDirsNew.map((it) => { return `-I ${it}`; }); - let args = ['-Xclang', '-ast-dump=json', '-fsyntax-only']; + let args = ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-std=c++11']; let _args = [...args, ...includeHeaderDirsArgs, parseFile].join(' '); let bashScript = `clang++ ${_args} > ${clangAstJsonPath}`; console.log(`Running command: \n${bashScript}`); - execSync(bashScript, { encoding: 'utf8', stdio: 'inherit' }); + try { + execSync(bashScript, { + stdio: ['pipe', 'pipe', 'pipe'], // Pipe stdin, stdout, and stderr, + encoding: 'utf8', + }); + } catch (err: any) { + let errMessage = err.message; + // Eliminate the fatal error summary, and then see if there's any other fatal error message, like + // /usr/local/Cellar/llvm@15/15.0.7/lib/clang/15.0.7/include/arm64intr.h:12:15: fatal error: 'arm64intr.h' file not found + errMessage = errMessage.replace( + 'fatal error: too many errors emitted, stopping now [-ferror-limit=]', + '' + ); + if (errMessage.includes('fatal error:')) { + // The file in path `clangAstJsonPath` will be created no matter the clang command failed or not, + // so we need to remove it if the clang command failed. + if (fs.existsSync(clangAstJsonPath)) { + fs.rmSync(clangAstJsonPath); + } + console.error(err.message); + process.exit(err.status); + } + + console.log(errMessage); + } let ast_json_file_content = fs.readFileSync(clangAstJsonPath, 'utf-8'); return ast_json_file_content; @@ -172,6 +249,15 @@ function _parseStructConstructors( let structConstructor: _StructConstructors = {} as _StructConstructors; structConstructor.name = s.ns ? `${s.ns}::${s.node.name}` : s.node.name; + // If only declare the struct with name, e.g., + // ```c++ + // struct EncodedVideoFrameInfo; + // ``` + // There's no `s.node.inner` of it, skip it. + if (!s.node.inner) { + continue; + } + // Find all `CXXConstructorDecl` nodes in `cxxRecordDecls`'s `inner` let cxxConstructorDecls = s.node.inner.filter((node: any) => { return ( @@ -187,6 +273,16 @@ function _parseStructConstructors( constructorInitializer.name = cxxConstructorDecl.name; constructorInitializer.signature = cxxConstructorDecl.type.qualType; + // If the constructor is explicitly defaulted, skip it. e.g., + // ```c++ + //struct RemoteVoicePositionInfo { + // RemoteVoicePositionInfo() = default; + // }; + // ``` + if (!cxxConstructorDecl.inner) { + continue; + } + // Find all `ParmVarDecl` nodes in `cxxConstructorDecl`'s `inner` let parmVarDecls = cxxConstructorDecl.inner.filter((node: any) => { return node.kind == ClangASTNodeKind.ParmVarDecl; @@ -353,7 +449,7 @@ function parseInnerValues(inner: any): [ConstructorInitializerKind, string[]] { break; } case ClangASTNodeKind.CXXConstructExpr: { - if (!node.isImplicit) { + if (!node.isImplicit && node.inner) { const [_, v] = parseInnerValues(node.inner); kind = ConstructorInitializerKind.Construct; values.push(...v); diff --git a/cxx-parser/src/cxx_parser.ts b/cxx-parser/src/cxx_parser.ts index d6409e9..fe2b320 100644 --- a/cxx-parser/src/cxx_parser.ts +++ b/cxx-parser/src/cxx_parser.ts @@ -28,6 +28,10 @@ function getBuildDir(terraContext: TerraContext) { return path.join(terraContext.buildDir, 'cxx_parser'); } +export function getCppAstBackendDir() { + return path.join(__dirname, '..', 'cxx', 'cppast_backend'); +} + export function dumpCXXAstJson( terraContext: TerraContext, includeHeaderDirs: string[], @@ -39,12 +43,7 @@ export function dumpCXXAstJson( let buildDir = getBuildDir(terraContext); - let agora_rtc_ast_dir_path = path.join( - __dirname, - '..', - 'cxx', - 'cppast_backend' - ); + let agora_rtc_ast_dir_path = getCppAstBackendDir(); let build_shell_path = path.join(agora_rtc_ast_dir_path, 'build.sh'); let build_cache_dir_path = buildDir;