Merge pull request #22 from Solo-steven/steven/issu-10/enhance-web-pa…

…rser

[Feat]: Refactor Parser structure with different scope manager

web-infras/parser/README.md

@@ -11,7 +11,6 @@ JS/JSX parser written in Typescript.
     - `estree/typescript`: TS not in current scope.
     - `jsx/errors/_no-plugin-ts-*`: TS not in current scope.
     - `flow`: Flow is not support.
-    - `core/scope`: Sematic Scope Not in current scope.
     - `esprima`: Duplicate to esprima.
     - `tokens`: should move to token folder.
     - `es2022/top-level-await-unambiguous`: Will not support.

web-infras/parser/src/dev.ts

@@ -1,6 +1,6 @@
 import { createLexer } from "@/src/lexer";
 import { SyntaxKinds, SytaxKindsMapLexicalLiteral } from "web-infra-common";
-import { createParser } from "@/src/parser";
+import { parse } from "@/src/index";
 import { transformSyntaxKindToLiteral } from "../tests/parserRunner/helpers/transform";
 import fs from "fs";
 import path from "path";
@@ -20,9 +20,10 @@ const code = fs.readFileSync(path.join(__dirname, "test.js")).toString();
 // // Combine styled and normal strings
 // log(chalk.blue('Hello') + ' World' + chalk.red('!'));
 
-printLexer(code);
+// printLexer(code);
 printParser(code);
 
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 function printLexer(code: string) {
   console.log("=================================");
   console.log("=================================");
@@ -48,7 +49,7 @@ function printLexer(code: string) {
   );
 }
 function printParser(code: string) {
-  const ast = createParser(code, { sourceType: "script", plugins: ["jsx"] }).parse();
+  const ast = parse(code, { sourceType: "script", plugins: ["jsx"] });
   transformSyntaxKindToLiteral(ast);
   console.log(JSON.stringify(ast, null, 4));
   return 0;

web-infras/parser/src/errorHandler/index.ts

@@ -0,0 +1,54 @@
+/**
+ * Error handler to record a recoverable error and print it
+ * as a format message
+ */
+import { SourcePosition } from "web-infra-common";
+import { SyntaxError, SyntaxErrorHandler } from "./type";
+
+export function createErrorHandler(code: string): SyntaxErrorHandler {
+  const errors: Array<SyntaxError> = [];
+  function pushSyntaxErrors(...syntaxErrors: Array<SyntaxError>) {
+    errors.push(...syntaxErrors);
+  }
+  function haveError() {
+    return errors.length > 0;
+  }
+  function formatErrorString() {
+    let formatString = "";
+    for (const err of errors) {
+      formatString += printError(err);
+    }
+    return formatString;
+  }
+  function printError(error: SyntaxError) {
+    const [lineStart, lineEnd] = getLineStartAndEnd(error.position);
+    const arrayLen = error.position.index - lineStart - 1 >= 0 ? error.position.index - lineStart - 1 : 0;
+    return `[SyntaxError]: ${error.message} (${error.position.row},${error.position.col})\n${error.position.row}|${code.slice(lineStart, lineEnd)}\n |${printSpace(arrayLen)}^\n`;
+  }
+  function getLineStartAndEnd(position: SourcePosition) {
+    let lineStart = 0;
+    for (lineStart = position.index - 1; lineStart > 0; --lineStart) {
+      if (code[lineStart] === "\n") {
+        break;
+      }
+    }
+    let lineEnd = 0;
+    for (lineEnd = position.index + 1; lineEnd < code.length; ++lineEnd) {
+      if (code[lineEnd] === "\n") {
+        break;
+      }
+    }
+    return [lineStart === 0 ? lineStart : lineStart + 1, lineEnd];
+  }
+  function printSpace(len: number): string {
+    return new Array(len)
+      .fill(0)
+      .map(() => " ")
+      .join("");
+  }
+  return {
+    pushSyntaxErrors,
+    haveError,
+    formatErrorString,
+  };
+}

web-infras/parser/src/errorHandler/type.ts

@@ -0,0 +1,11 @@
+import { SourcePosition } from "web-infra-common";
+export interface SyntaxError {
+  message: string;
+  position: SourcePosition;
+}
+
+export type SyntaxErrorHandler = {
+  pushSyntaxErrors: (...syntaxErrors: Array<SyntaxError>) => void;
+  haveError: () => boolean;
+  formatErrorString: () => string;
+};

web-infras/parser/src/index.ts

@@ -2,11 +2,17 @@ import { createParser } from "@/src/parser";
 import { createLexer } from "@/src/lexer";
 import { Token } from "@/src/lexer/type";
 import { SyntaxKinds } from "web-infra-common";
-import { ParserConfig } from "./parser/config";
+import { ParserUserConfig } from "./parser/config";
+import { createErrorHandler } from "./errorHandler";
 
-export function parse(code: string, config?: ParserConfig) {
-  const parser = createParser(code, config);
-  return parser.parse();
+export function parse(code: string, config?: ParserUserConfig) {
+  const errorHandler = createErrorHandler(code);
+  const parser = createParser(code, errorHandler, config);
+  const program = parser.parse();
+  if (errorHandler.haveError()) {
+    throw new Error(errorHandler.formatErrorString());
+  }
+  return program;
 }
 export function tokenize(code: string): Array<Token> {
   const lexer = createLexer(code);

web-infras/parser/src/lexer/error.ts

@@ -1,16 +1,18 @@
 /**
  * Format error from MDN, using v8 based as error message value, some error is come from babel and
- * v8 engine
+ * v8 engine.
+ * If MDN message exsit for certain error, if not, using v8 error, if v8 still not, go back to use
+ * babel.
  */
 export const ErrorMessageMap = {
   // reference: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Bad_regexp_flag
-  syntax_error_invalid_regular_expression_flags: "SyntaxError: Invalid regular expression flags",
+  syntax_error_invalid_regular_expression_flags: "Invalid regular expression flags",
   // reference: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Deprecated_octal_escape_sequence
   syntax_error_Octal_escape_sequences_are_not_allowed_in_strict_mode:
-    "SyntaxError: Octal escape sequences are not allowed in strict mode.",
-  v8_error_invalid_unicode_escape_sequence: "SyntaxError: Invalid Unicode escape sequence",
+    "Octal escape sequences are not allowed in strict mode.",
+  v8_error_invalid_unicode_escape_sequence: "Invalid Unicode escape sequence",
   // try `"\xfG"` in v8 runtime
-  v8_error_invalid_hexadecimal_escape_sequence: "SyntaxError: Invalid hexadecimal escape sequence",
+  v8_error_invalid_hexadecimal_escape_sequence: "Invalid hexadecimal escape sequence",
   // try: `const a = ';` in babel playground
   babel_error_unterminated_string_constant: "Unterminated string constant",
   // try: `const a = `;` in babel playground
@@ -22,9 +24,9 @@ export const ErrorMessageMap = {
   babel_error_floating_point_numbers_require_a_valid_exponent_after_the_e:
     "Floating-point numbers require a valid exponent after the 'e'",
   // try: `1.3n` in babel playground
-  babel_error_invalid_bigIntLiteral: "SyntaxError: Invalid hexadecimal escape sequence",
+  babel_error_invalid_bigIntLiteral: "Invalid hexadecimal escape sequence",
   // try `3in []` in babel playground
-  babel_error_Identifier_directly_after_number: "SyntaxError: Identifier directly after number.",
+  babel_error_Identifier_directly_after_number: "Identifier directly after number.",
   // error: `001_1`
   error_legacy_octal_literals_contain_numeric_seperator:
     "Legacy octal literals can not contain any numeric seperator",

web-infras/parser/src/lexer/lexer.ts

@@ -85,6 +85,12 @@ export function createLexer(code: string) {
   function getEndPosition(): SourcePosition {
     return state.token.endPosition;
   }
+  /**
+   * Public API for get end position of last token
+   */
+  function getLastTokenEndPositon(): SourcePosition {
+    return state.token.lastTokenEndPosition;
+  }
   /**
    * Public API for get current token kind.
    * @returns {SyntaxKinds}
@@ -331,6 +337,7 @@ export function createLexer(code: string) {
     getSourceValueByIndex,
     getStartPosition,
     getEndPosition,
+    getLastTokenEndPositon,
     getLineTerminatorFlag,
     getEscFlag,
     getTemplateLiteralTag,
@@ -1072,9 +1079,14 @@ export function createLexer(code: string) {
   function readQuestionStart(): SyntaxKinds {
     eatChar();
     switch (getCharCodePoint()) {
-      case UnicodePoints.Dot:
+      case UnicodePoints.Dot: {
+        const next = getNextCharCodePoint();
+        if (next && next >= UnicodePoints.Digital0 && next <= UnicodePoints.Digital9) {
+          return finishToken(SyntaxKinds.QustionOperator);
+        }
         eatChar();
         return finishToken(SyntaxKinds.QustionDotOperator);
+      }
       case UnicodePoints.Question:
         eatChar();
         switch (getCharCodePoint()) {

web-infras/parser/src/parser/config.ts

@@ -2,28 +2,26 @@
  * Config from babel parser
  */
 export type ParserConfig = {
-  sourceType?: "script" | "module";
-  allowReturnOutsideFunction?: boolean;
-  allowAwaitOutsideFunction?: boolean;
-  // createImportExpressions?: boolean;
-  allowNewTargetOutsideFunction?: boolean;
-  plugins?: Array<string>;
-};
-export type ParserMergedConfig = {
   sourceType: "script" | "module";
   allowReturnOutsideFunction: boolean;
   allowAwaitOutsideFunction: boolean;
   allowNewTargetOutsideFunction: boolean;
+  allowUndeclaredExports: boolean;
   plugins: Array<string>;
+  // errorRecovery: boolean;
+  // createImportExpressions: boolean;
+  // allowSuperOutsideMethod: boolean;
 };
-const defaultConfig: ParserMergedConfig = {
+export type ParserUserConfig = Partial<ParserConfig>;
+const defaultConfig: ParserConfig = {
   sourceType: "script",
   allowReturnOutsideFunction: false,
   allowAwaitOutsideFunction: false,
   allowNewTargetOutsideFunction: false,
+  allowUndeclaredExports: false,
   plugins: [],
 };
-export function getConfigFromUserInput(config?: ParserConfig): ParserMergedConfig {
+export function getConfigFromUserInput(config?: ParserUserConfig): ParserConfig {
   return {
     ...defaultConfig,
     ...(config || {}),