Skip to content

Commit

Permalink
fix: Fix can not parse the c++ std headers when parsing the construct…
Browse files Browse the repository at this point in the history 
…or initializers
  • Loading branch information
@littleGnAl
littleGnAl committed Nov 9, 2023
1 parent 1935eee commit c7f72fd
Show file tree
Hide file tree
Showing 3 changed files with 215 additions and 18 deletions.
102 changes: 102 additions & 0 deletions cxx-parser/__tests__/unit_test/constructor_initializer_parser.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,108 @@ namespace ns1 {
// Aim to test the private funtion
const parseStructConstructors = testingUsed.parseStructConstructors;

it('empty constructor', () => {
let filePath = path.join(tmpDir, 'file.h');
let fileContent = `
#pragma once
namespace ns1 {
struct AAA {
AAA() {}
};
}
`;
fs.writeFileSync(filePath, fileContent);

let constructorInitializers = parseStructConstructors(
tmpDir,
[],
[filePath]
);

let expectedRes = [
{
name: 'ns1::AAA',
constructors: [
{
name: 'AAA',
signature: 'void ()',
parameterList: [],
initializerList: [],
},
],
},
];

expect(JSON.stringify(constructorInitializers)).toEqual(
JSON.stringify(expectedRes)
);
});

it('default constructor', () => {
let filePath = path.join(tmpDir, 'file.h');
let fileContent = `
#pragma once
namespace ns1 {
struct AAA {
AAA() = default;
};
}
`;
fs.writeFileSync(filePath, fileContent);

let constructorInitializers = parseStructConstructors(
tmpDir,
[],
[filePath]
);

let expectedRes = [
{
name: 'ns1::AAA',
constructors: [],
},
];

expect(JSON.stringify(constructorInitializers)).toEqual(
JSON.stringify(expectedRes)
);
});

it('declare struct only', () => {
let filePath = path.join(tmpDir, 'file.h');
let fileContent = `
#pragma once
namespace ns1 {
struct AAANameOnly;
struct AAA {
AAA() = default;
};
}
`;
fs.writeFileSync(filePath, fileContent);

let constructorInitializers = parseStructConstructors(
tmpDir,
[],
[filePath]
);

let expectedRes = [
{
name: 'ns1::AAA',
constructors: [],
},
];

expect(JSON.stringify(constructorInitializers)).toEqual(
JSON.stringify(expectedRes)
);
});

it('constructor assign enum', () => {
let filePath = path.join(tmpDir, 'file.h');
let fileContent = `
Expand Down
120 changes: 108 additions & 12 deletions cxx-parser/src/constructor_initializer_parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import path from 'path';

import { ParseResult, visibleForTesting } from '@agoraio-extensions/terra-core';

import { generateChecksum } from './cxx_parser';
import { generateChecksum, getCppAstBackendDir } from './cxx_parser';
import {
CXXFile,
CXXTYPE,
Expand Down Expand Up @@ -86,6 +86,66 @@ class _StructConstructors {
constructors: _ConstructorInitializer[] = [];
}

/**
*
* @returns The default include dirs for clang command line tool
*/
function getDefaultIncludeDirs(): string[] {
// TODO(littlegnal): This implementation is borrowed from cppast to retrive the default include dirs
// https://github.com/foonathan/cppast/blob/f00df6675d87c6983033d270728c57a55cd3db22/src/libclang/libclang_parser.cpp#L287
// But it seems it's not necessary to add these when running the clang command line tool, so we just keep the code here, and
// see if we need to add these in the future.
// function _findIncludeDirsFromClang(): string[] {
// let verbose_output: string = '';
// let verboseCommand = 'clang++ -xc++ -v -';
// try {
// execSync(verboseCommand, {
// input: '',
// stdio: ['pipe', 'pipe', 'pipe'], // Pipe stdin, stdout, and stderr
// encoding: 'utf-8',
// }).toString();
// } catch (error: any) {
// // We need to use the tricky wayt to get the verbose output from clang
// verbose_output = error.message ?? '';
// }

// // If the command failed, return the empty include header dirs
// if (!verbose_output) {
// return [];
// }

// let verboseOutputInLines = verbose_output.split('\n');

// verboseOutputInLines = verboseOutputInLines.slice(
// verboseOutputInLines.findIndex((it) => {
// return it.startsWith('#include <...>');
// }) + 1, // Do not include the index of '#include <...>'
// verboseOutputInLines.findIndex((it) => {
// return it.startsWith('End of search list.');
// })
// );
// let includeDirs = verboseOutputInLines
// .filter((it) => {
// return it.startsWith(' ');
// })
// .map((it) => {
// if (it.includes(' (')) {
// // /Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/System/Library/Frameworks (framework directory)
// return it.trim().split(' (')[0];
// }

// return it.trim();
// });

// return includeDirs;
// }

return [
// Add cxx-parser/cxx/cppast_backend/include/system_fake
path.join(getCppAstBackendDir(), 'include', 'system_fake'),
];
}

/**
* This function mainly use the clang command line tool to dump the AST in json format,
* ```
Expand Down Expand Up @@ -126,25 +186,42 @@ function dumpClangASTJSON(
return fs.readFileSync(clangAstJsonPath, 'utf-8');
}

let includeHeaderDirsNew = [
// For macos
'/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include',
'/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include',
// For linux
'/usr/local/include',
...includeHeaderDirs,
];
let includeHeaderDirsNew = [...getDefaultIncludeDirs(), ...includeHeaderDirs];
let includeHeaderDirsArgs = includeHeaderDirsNew.map((it) => {
return `-I ${it}`;
});

let args = ['-Xclang', '-ast-dump=json', '-fsyntax-only'];
let args = ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-std=c++11'];
let _args = [...args, ...includeHeaderDirsArgs, parseFile].join(' ');

let bashScript = `clang++ ${_args} > ${clangAstJsonPath}`;
console.log(`Running command: \n${bashScript}`);

execSync(bashScript, { encoding: 'utf8', stdio: 'inherit' });
try {
execSync(bashScript, {
stdio: ['pipe', 'pipe', 'pipe'], // Pipe stdin, stdout, and stderr,
encoding: 'utf8',
});
} catch (err: any) {
let errMessage = err.message;
// Eliminate the fatal error summary, and then see if there's any other fatal error message, like
// /usr/local/Cellar/llvm@15/15.0.7/lib/clang/15.0.7/include/arm64intr.h:12:15: fatal error: 'arm64intr.h' file not found
errMessage = errMessage.replace(
'fatal error: too many errors emitted, stopping now [-ferror-limit=]',
''
);
if (errMessage.includes('fatal error:')) {
// The file in path `clangAstJsonPath` will be created no matter the clang command failed or not,
// so we need to remove it if the clang command failed.
if (fs.existsSync(clangAstJsonPath)) {
fs.rmSync(clangAstJsonPath);
}
console.error(err.message);
process.exit(err.status);
}

console.log(errMessage);
}

let ast_json_file_content = fs.readFileSync(clangAstJsonPath, 'utf-8');
return ast_json_file_content;
Expand Down Expand Up @@ -172,6 +249,15 @@ function _parseStructConstructors(
let structConstructor: _StructConstructors = {} as _StructConstructors;
structConstructor.name = s.ns ? `${s.ns}::${s.node.name}` : s.node.name;

// If only declare the struct with name, e.g.,
// ```c++
// struct EncodedVideoFrameInfo;
// ```
// There's no `s.node.inner` of it, skip it.
if (!s.node.inner) {
continue;
}

// Find all `CXXConstructorDecl` nodes in `cxxRecordDecls`'s `inner`
let cxxConstructorDecls = s.node.inner.filter((node: any) => {
return (
Expand All @@ -187,6 +273,16 @@ function _parseStructConstructors(
constructorInitializer.name = cxxConstructorDecl.name;
constructorInitializer.signature = cxxConstructorDecl.type.qualType;

// If the constructor is explicitly defaulted, skip it. e.g.,
// ```c++
//struct RemoteVoicePositionInfo {
// RemoteVoicePositionInfo() = default;
// };
// ```
if (!cxxConstructorDecl.inner) {
continue;
}

// Find all `ParmVarDecl` nodes in `cxxConstructorDecl`'s `inner`
let parmVarDecls = cxxConstructorDecl.inner.filter((node: any) => {
return node.kind == ClangASTNodeKind.ParmVarDecl;
Expand Down Expand Up @@ -353,7 +449,7 @@ function parseInnerValues(inner: any): [ConstructorInitializerKind, string[]] {
break;
}
case ClangASTNodeKind.CXXConstructExpr: {
if (!node.isImplicit) {
if (!node.isImplicit && node.inner) {
const [_, v] = parseInnerValues(node.inner);
kind = ConstructorInitializerKind.Construct;
values.push(...v);
Expand Down
11 changes: 5 additions & 6 deletions cxx-parser/src/cxx_parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ function getBuildDir(terraContext: TerraContext) {
return path.join(terraContext.buildDir, 'cxx_parser');
}

export function getCppAstBackendDir() {
return path.join(__dirname, '..', 'cxx', 'cppast_backend');
}

export function dumpCXXAstJson(
terraContext: TerraContext,
includeHeaderDirs: string[],
Expand All @@ -39,12 +43,7 @@ export function dumpCXXAstJson(

let buildDir = getBuildDir(terraContext);

let agora_rtc_ast_dir_path = path.join(
__dirname,
'..',
'cxx',
'cppast_backend'
);
let agora_rtc_ast_dir_path = getCppAstBackendDir();

let build_shell_path = path.join(agora_rtc_ast_dir_path, 'build.sh');
let build_cache_dir_path = buildDir;
Expand Down

0 comments on commit c7f72fd

Please sign in to comment.