From 4b904206d6913c03abaa2533769ee21a19af0183 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Tue, 16 May 2017 15:21:49 -0500 Subject: [PATCH 1/6] Moving to TypeScript target --- grammars-v4 | 2 +- language_configs/java8.compile.js | 1 + language_configs/python3.compile.js | 1 + package.json | 4 ++ src/compile.js | 16 +++++-- src/error_listener.js | 46 +++++++++---------- src/runtime.js | 4 +- src/transformers/simplify_node.js | 10 ++-- src/tree_matcher.js | 10 ++-- .../lang_config.compile.js | 1 + src/utils/run_antlr.js | 39 ++++++++++------ src/utils/run_tsc.js | 27 +++++++++++ 12 files changed, 106 insertions(+), 55 deletions(-) create mode 100644 src/utils/run_tsc.js diff --git a/grammars-v4 b/grammars-v4 index 96255d4..9c16398 160000 --- a/grammars-v4 +++ b/grammars-v4 @@ -1 +1 @@ -Subproject commit 96255d4c1bfcc377263842d35cbba92620154275 +Subproject commit 9c16398f5af1274cdc00a2bc78730901dcda5f27 diff --git a/language_configs/java8.compile.js b/language_configs/java8.compile.js index 051210c..d343cff 100644 --- a/language_configs/java8.compile.js +++ b/language_configs/java8.compile.js @@ -4,6 +4,7 @@ module.exports = { 'grammar_files': { 'Java': 'Java8.g4', 'JavaScript': 'Java8.JavaScriptTarget.g4', + 'TypeScript': 'Java8.TypeScriptTarget.g4', }, 'needs_java_func_data': true, 'tree_matcher_specs': require('./java8.tree_matcher_specs.js'), diff --git a/language_configs/python3.compile.js b/language_configs/python3.compile.js index c3b92b9..caa5aa8 100644 --- a/language_configs/python3.compile.js +++ b/language_configs/python3.compile.js @@ -4,6 +4,7 @@ module.exports = { 'grammar_files': { 'Java': 'Python3.g4', 'JavaScript': 'Python3.JavaScriptTarget.g4', + 'TypeScript': 'Python3.TypeScriptTarget.g4', }, 'tree_matcher_specs': require('./python3.tree_matcher_specs.js'), }; diff --git a/package.json b/package.json index e51daea..8b97e80 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,12 @@ "author": "Maryville Techonologies", "license": "ISC", "dependencies": { + "@types/node": "^7.0.18", + "antlr4ts": "^0.4.0-alpha.4", + "antlr4ts-cli": "^0.4.0-alpha.4", "child-process-promise": "^2.2.0", "fs-promise": "^2.0.0", + "typescript": "^2.3.2", "webpack": "^2.2.1", "webpack-closure-compiler": "^2.1.4" }, diff --git a/src/compile.js b/src/compile.js index 055696d..c0e330d 100644 --- a/src/compile.js +++ b/src/compile.js @@ -1,11 +1,11 @@ const path = require('path'); const fs = require('fs-promise'); const child_process = require('child-process-promise'); -const antlr = require('antlr4'); const config = require('../config'); const expect_error = require('./utils/expect_error.js'); const run_antlr = require('./utils/run_antlr.js'); +const run_tsc = require('./utils/run_tsc.js'); const tree_matcher = require('./tree_matcher.js'); const java_func_data_generator = require('./java_func_data_generator.js'); @@ -31,6 +31,7 @@ parser. */ const build_tasks = (lang_compile_config, lang_runtime_config) => { const { + use_typescript_target, tree_matcher_specs, needs_java_func_data, } = lang_compile_config; @@ -41,8 +42,8 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { let result; - const make_lexer_and_parser = async () => { - result = await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); + const make_typescript_target = async () => { + result = await run_antlr(lang_compile_config, lang_runtime_config, 'TypeScript'); } /* --- Generates java func data, if this lang needs any --- */ @@ -53,6 +54,10 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { } } + const make_javascript_target = async() => { + await run_tsc(result.build_dir); + } + /* --- Creates and returns the parser and symbol_name_map --- */ const make_parser = async () => { // Create instance of the parser @@ -62,7 +67,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { // Create an array of symbol (terminal) names const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] - .concat(parser.symbolicNames.slice(1)) + .concat(parser.vocabulary.symbolicNames.slice(1)) .map((val) => val ? '.' + val : undefined); // Create the lists of rule names (both terminals and non-terminals) @@ -130,8 +135,9 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { // Return the set of async closures return [ - make_lexer_and_parser, + make_typescript_target, make_java_func_data, + make_javascript_target, make_parser, make_runtime_config_modifier, write_runtime_config_modifier, diff --git a/src/error_listener.js b/src/error_listener.js index 28732f4..52bbe07 100644 --- a/src/error_listener.js +++ b/src/error_listener.js @@ -1,30 +1,26 @@ -let antlr = require('antlr4'); +"use strict"; +class ConsoleErrorListener { + constructor(callback) { + this.callback = callback; + } -function ErrorListener(callback) { - antlr.error.ErrorListener.call(this); - this.callback = callback; - return this; -} - -ErrorListener.prototype = Object.create(antlr.error.ErrorListener.prototype); -ErrorListener.prototype.constructor = ErrorListener; + syntaxError(recognizer, offendingSymbol, line, charPositionInLine, msg, e) { + this.callback({ + 'type': 'syntaxError', + 'begin': offendingSymbol.start, + 'end': offendingSymbol.stop + 1, + 'msg': msg, + }); + } -ErrorListener.prototype.syntaxError = function(recognizer, offendingSymbol, line, column, msg, e) { - this.callback({ - 'type': 'syntaxError', - 'begin': offendingSymbol.start, - 'end': offendingSymbol.stop + 1, - 'msg': msg, - }); -}; + reportAmbiguity(recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs) { + } -ErrorListener.prototype.reportAmbiguity = function(recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs) { -}; + reportAttemptingFullContext(recognizer, dfa, startIndex, stopIndex, conflictingAlts, configs) { + } -ErrorListener.prototype.reportAttemptingFullContext = function(recognizer, dfa, startIndex, stopIndex, conflictingAlts, configs) { -}; - -ErrorListener.prototype.reportContextSensitivity = function(recognizer, dfa, startIndex, stopIndex, prediction, configs) { -}; + reportContextSensitivity(recognizer, dfa, startIndex, stopIndex, prediction, configs) { + } +} -module.exports = ErrorListener; +module.exports = ConsoleErrorListener; diff --git a/src/runtime.js b/src/runtime.js index 816002d..1d673bd 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -1,5 +1,5 @@ let path = require('path'); -let antlr = require('antlr4'); +let antlr = require('antlr4ts'); let lang_runtime_config = require('LangRuntimeConfig'); @@ -32,7 +32,7 @@ module.exports = function(input, error_callback, options) { // Take the string of code, and generate a stream of tokens using the antlr lexer. // Example: ['if', '(', 'var', '==', '123', ')', '{', ...] - let chars = new antlr.InputStream(input); + let chars = new antlr.ANTLRInputStream(input); let lexer = new LexerClass(chars); let tokens = new antlr.CommonTokenStream(lexer); diff --git a/src/transformers/simplify_node.js b/src/transformers/simplify_node.js index 3138c7c..c58d731 100644 --- a/src/transformers/simplify_node.js +++ b/src/transformers/simplify_node.js @@ -1,12 +1,14 @@ -const { TerminalNodeImpl } = require('antlr4/tree/Tree.js'); - +const { TerminalNode } = require('antlr4ts/tree/TerminalNode.js'); +// BLABLA module.exports = function(lang_runtime_config, root) { // Takes an antlr node generated by the antlr parser, and // outputs our simplified node. + + const { symbol_name_map, rule_name_map } = lang_runtime_config; + const simplify_node = function(node) { - const { symbol_name_map, rule_name_map } = lang_runtime_config; - if (node instanceof TerminalNodeImpl) { + if (node instanceof TerminalNode) { const { type, start, stop, text } = node.symbol; return { 'type': symbol_name_map[type + 2], diff --git a/src/tree_matcher.js b/src/tree_matcher.js index a34085b..aab7eca 100644 --- a/src/tree_matcher.js +++ b/src/tree_matcher.js @@ -1,6 +1,6 @@ let path = require('path'); -let antlr = require('antlr4'); -let TerminalNodeImpl = require('antlr4/tree/Tree.js').TerminalNodeImpl; +let antlr = require('antlr4ts'); +let TerminalNode = require('antlr4ts/tree/TerminalNode.js').TerminalNode; let tree_matcher_compile_config = require('./tree_matcher_parser/lang_config.compile.js'); let tree_matcher_runtime_config = require('./tree_matcher_parser/lang_config.runtime.js'); @@ -51,7 +51,7 @@ module.exports.make_generator = async function(lang_compile_config, lang_runtime return function(matcher_spec) { // Take the string of a matcher specification pattern, and generate a stream of tokens using the antlr lexer. - let chars = new antlr.InputStream(matcher_spec.pattern); + let chars = new antlr.ANTLRInputStream(matcher_spec.pattern); let lexer = new LexerClass(chars); let tokens = new antlr.CommonTokenStream(lexer); @@ -171,7 +171,7 @@ module.exports.make_generator = async function(lang_compile_config, lang_runtime case 'mod_fixed_children': { let child_index = 0; node.children[1].children.forEach(function(child) { - if (child instanceof TerminalNodeImpl) { + if (child instanceof TerminalNode) { return ''; } else { let child_var = create_var(); @@ -200,7 +200,7 @@ module.exports.make_generator = async function(lang_compile_config, lang_runtime } case 'expr': { return node.children.map(function(child) { - if (child instanceof TerminalNodeImpl) { + if (child instanceof TerminalNode) { return ''; } else { return build_tester(child, node_var, pass_stmt); diff --git a/src/tree_matcher_parser/lang_config.compile.js b/src/tree_matcher_parser/lang_config.compile.js index 4c98987..16e68a2 100644 --- a/src/tree_matcher_parser/lang_config.compile.js +++ b/src/tree_matcher_parser/lang_config.compile.js @@ -4,5 +4,6 @@ module.exports = { 'grammar_dir': __dirname, 'grammar_files': { 'JavaScript': 'TreeMatcher.g4', + 'TypeScript': 'TreeMatcher.g4', }, }; diff --git a/src/utils/run_antlr.js b/src/utils/run_antlr.js index cd3668d..ab0eaa9 100644 --- a/src/utils/run_antlr.js +++ b/src/utils/run_antlr.js @@ -21,6 +21,10 @@ module.exports = async (lang_compile_config, lang_runtime_config, target_languag // Figure out the path to the grammar file const g4_dir = grammar_dir ? grammar_dir : path.resolve(__dirname, '..', '..', 'grammars-v4', language_key); + + if (typeof grammar_files[target_language] !== 'string') { + throw new Error('No grammar file specified for target ' + target_language); + } const g4_path = path.resolve(g4_dir, grammar_files[target_language]); const build_dir = config.resolve_build_dir(lang_runtime_config, target_language); @@ -45,20 +49,29 @@ module.exports = async (lang_compile_config, lang_runtime_config, target_languag const invoke_antlr = async () => { // Prepare options to the antlr compiler that generates // the antlr lexer and antlr parser - const cmd = 'java'; - const args = [ - '-Xmx500M', - '-cp', path.resolve(__dirname, '../../bin/antlr-4.6-complete.jar'), - 'org.antlr.v4.Tool', - '-long-messages', - generate_listener ? '-listener' : '-no-listener', - generate_visitor ? '-visitor' : '-no-visitor', - '-Dlanguage=' + target_language, - language + '.g4', - ]; + + let cmd; + let args; + + if (target_language === 'TypeScript') { + // To generate a TypeScript target, run the node antlr4ts-cli module + cmd = process.argv[0]; // This should be an absolute path to the node executable + args = [require.resolve('antlr4ts-cli')]; + } else { + cmd = 'java'; + args = ['-jar', path.resolve(__dirname, '../../bin/antlr-4.6-complete.jar')]; + } + + args = args.concat([ + '-long-messages', + generate_listener ? '-listener' : '-no-listener', + generate_visitor ? '-visitor' : '-no-visitor', + '-Dlanguage=' + target_language, + language + '.g4', + ]); const opts = { - 'cwd': build_dir, - 'stdio': ['ignore', process.stdout, process.stderr], + 'cwd': build_dir, + 'stdio': ['ignore', process.stdout, process.stderr], }; await child_process.spawn(cmd, args, opts); diff --git a/src/utils/run_tsc.js b/src/utils/run_tsc.js new file mode 100644 index 0000000..e3f0605 --- /dev/null +++ b/src/utils/run_tsc.js @@ -0,0 +1,27 @@ +const path = require('path'); +const fs = require('fs-promise'); +const child_process = require('child-process-promise'); + +module.exports = async (build_dir) => { + const tsconfig = { + 'compilerOptions': { + 'module': 'commonjs', + 'target': 'es6', + 'experimentalDecorators': true, + 'noImplicitAny': true, + 'strictNullChecks': true, + } + }; + + const tsconfig_path = path.resolve(build_dir, 'tsconfig.json'); + await fs.writeFile(tsconfig_path, JSON.stringify(tsconfig)); + + const cmd = process.argv[0]; + const args = [require.resolve('typescript/lib/tsc')]; + const opts = { + 'cwd': build_dir, + 'stdio': ['ignore', process.stdout, process.stderr], + }; + + await child_process.spawn(cmd, args, opts); +} From 623781619c6dd69de3f461f35a130ff128f9ad20 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 18 May 2017 14:40:23 -0500 Subject: [PATCH 2/6] Fixed test failures with TypeScript target --- __tests__/transformers/simplify_node.test.js | 7 ++++++- __tests__/tree/code/snippet.kt29xnfw.py | 1 - __tests__/tree/compare.test.js | 13 ++++++------- grammars-v4 | 2 +- src/runtime.js | 4 ++-- src/transformers/simplify_node.js | 2 +- src/utils/utils.js | 8 +++----- tools/index.html | 6 ++++-- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/__tests__/transformers/simplify_node.test.js b/__tests__/transformers/simplify_node.test.js index 987a18a..c867e25 100644 --- a/__tests__/transformers/simplify_node.test.js +++ b/__tests__/transformers/simplify_node.test.js @@ -29,11 +29,12 @@ describe(`simplify_node.js`, () => { • end • tags • children`, () => { + // Different forms for starts and stops are intentionally // used here, as both can be found in real ANTLR nodes. const input_node = ( makeAntlrNode(0, { start: 0 }, { stop: 5 }, [ - makeAntlrNode(1, { start: 0, stop: 4 }, undefined, undefined), + makeAntlrNode(1, { start: 0, stop: 4 }, undefined, []), ])); const { rule_name_map } = lang_runtime_config; @@ -41,6 +42,7 @@ describe(`simplify_node.js`, () => { makeNode(rule_name_map[0], 0, 5 + 1, [], [ makeNode(rule_name_map[1], 0, 4 + 1, [], []), ])); + expect(simplify_node(lang_runtime_config, input_node)).toEqual(expected); }); @@ -51,13 +53,16 @@ describe(`simplify_node.js`, () => { • tags • children • text`, () => { + const input_terminal = ( makeAntlrTerminal(0, 0, 5, 'foobar') ); + const { symbol_name_map } = lang_runtime_config; const expected = ( makeTerminal(symbol_name_map[0 + 2], 0, 6, 'foobar', []) ); + expect(simplify_node(lang_runtime_config, input_terminal)).toEqual(expected); }); }); diff --git a/__tests__/tree/code/snippet.kt29xnfw.py b/__tests__/tree/code/snippet.kt29xnfw.py index 5696edc..896ca9d 100644 --- a/__tests__/tree/code/snippet.kt29xnfw.py +++ b/__tests__/tree/code/snippet.kt29xnfw.py @@ -4,4 +4,3 @@ print('sum =', total) except ValueError: print('Please supply integer arguments') - diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index 0045bd1..ee49e86 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -1,6 +1,5 @@ const path = require('path'); const fs = require('fs-promise'); -const antlr = require('antlr4'); const expect_error = require('../../src/utils/expect_error.js'); const run_antlr = require('../../src/utils/run_antlr.js'); const run_java = require('../../src/utils/run_java.js'); @@ -79,13 +78,11 @@ const prepareJs = async function(lang_compile_config, lang_runtime_config) { } const genTreeViaJs = async function(lang_runtime_config, js_parser, code) { - const tree = js_parser(code, function(err) { + return js_parser(code, function(err) { throw err; }, { - 'return_antlr_tree': true, + 'return_antlr_toStringTree': true, }); - - return tree.toStringTree(tree.parser.ruleNames); }; @@ -123,7 +120,9 @@ const compare = async function(language_name, code_file) { console.log('compare(' + code_file + ') genJava: ' + (t4 - t3) + 'ms'); console.log('compare(' + code_file + ') genJs: ' + (t5 - t4) + 'ms'); - return treeViaJava.trim() === treeViaJs.trim(); + return [treeViaJava, treeViaJs].map(function(val) { + return val.trim().replace(/(\\n|\s)+/g, ' '); + }); }; @@ -145,7 +144,7 @@ describe('grammars-v4/', () => { it(description, () => { expect.assertions(1); return compare(lang_key, __dirname + '/code/' + code_filename).then(data => { - expect(data).toBeTruthy(); + expect(data[0]).toEqual(data[1]); }).catch(err => { console.error(err); }); diff --git a/grammars-v4 b/grammars-v4 index 9c16398..9975d18 160000 --- a/grammars-v4 +++ b/grammars-v4 @@ -1 +1 @@ -Subproject commit 9c16398f5af1274cdc00a2bc78730901dcda5f27 +Subproject commit 9975d18368efe37dd4f45058affd21000493d91f diff --git a/src/runtime.js b/src/runtime.js index 1d673bd..e6d9755 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -53,8 +53,8 @@ module.exports = function(input, error_callback, options) { // This tree has complicated nodes that need to be simplified by our process_node function. let tree = parser[lang_runtime_config.entry_rule](); - if (options.return_antlr_tree) { - return tree; + if (options.return_antlr_toStringTree) { + return tree.toStringTree(parser.ruleNames); } // Transform the tree and return it diff --git a/src/transformers/simplify_node.js b/src/transformers/simplify_node.js index 2c44d0d..aee915e 100644 --- a/src/transformers/simplify_node.js +++ b/src/transformers/simplify_node.js @@ -1,5 +1,5 @@ const { TerminalNode } = require('antlr4ts/tree/TerminalNode.js'); -// BLABLA + module.exports = function(lang_runtime_config, root) { // Takes an antlr node generated by the antlr parser, and diff --git a/src/utils/utils.js b/src/utils/utils.js index 269311d..9f008c7 100644 --- a/src/utils/utils.js +++ b/src/utils/utils.js @@ -1,4 +1,4 @@ -const { TerminalNodeImpl } = require('antlr4/tree/Tree.js'); +const { TerminalNode } = require('antlr4ts/tree/TerminalNode.js'); /* Returns an object built from the given arguments. The object is @@ -49,12 +49,10 @@ Returns an object built from the given arguments. The object is a mock of terminal nodes that appear in ANTLR output ASTs. Note that these mocks are incomplete, and only contain some properties found in real ANTLR nodes. */ module.exports.makeAntlrTerminal = (type, start, stop, text) => { - const terminal = Object.create(TerminalNodeImpl.prototype, {}); - terminal.symbol = { + return new TerminalNode({ type, start, stop, text - }; - return terminal; + }); }; diff --git a/tools/index.html b/tools/index.html index da342cb..ab4c8a7 100644 --- a/tools/index.html +++ b/tools/index.html @@ -144,7 +144,7 @@ let lang_el = document.getElementById('language'); lang_el.onchange = function() { - let path = 'build/parsers/' + lang_el.value + '.js'; + let path = '../build/parsers/' + lang_el.value + '.js'; // let path = 'https://api.codesplain.io/dev/parsers/' + lang_el.value; var script_el = document.createElement('script'); @@ -161,7 +161,9 @@ if (typeof parser !== 'function') {return;} let code = code_el.value; - let ast = parser(code, function() {}, {'log_cutoffs': false}); + let ast = parser(code, function(e) { + console.error(e); + }, {'log_cutoffs': false}); console.log(ast); let html = gen_html(code, ast); formatted_el.innerHTML = html; From 219498205c4fb78719c0c37a15f8f40311ce4772 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 18 May 2017 14:52:56 -0500 Subject: [PATCH 3/6] Removed JavaScript antlr runtime dependency --- .gitmodules | 3 --- antlr4 | 1 - install | 7 ------- language_configs/java8.compile.js | 1 - language_configs/python3.compile.js | 1 - 5 files changed, 13 deletions(-) delete mode 160000 antlr4 diff --git a/.gitmodules b/.gitmodules index 09ca83d..ce39aaa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "grammars-v4"] path = grammars-v4 url = https://github.com/maryvilledev/grammars-v4.git -[submodule "antlr4"] - path = antlr4 - url = https://github.com/maryvilledev/antlr4.git diff --git a/antlr4 b/antlr4 deleted file mode 160000 index 2ebae94..0000000 --- a/antlr4 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2ebae94612110045b86ca5585c8227e7c808d192 diff --git a/install b/install index 9d42b84..125249d 100755 --- a/install +++ b/install @@ -8,10 +8,3 @@ git submodule update --init --recursive mkdir -p bin/ curl http://www.antlr.org/download/antlr-4.6-complete.jar -o bin/antlr-4.6-complete.jar -cd antlr4/runtime/JavaScript/src/antlr4/ - npm install -cd - -npm link antlr4/runtime/JavaScript/src/antlr4/ - -mkdir -p _cache/ -mkdir -p _cache/java_func_data/ diff --git a/language_configs/java8.compile.js b/language_configs/java8.compile.js index d343cff..331ced9 100644 --- a/language_configs/java8.compile.js +++ b/language_configs/java8.compile.js @@ -3,7 +3,6 @@ module.exports = { 'grammar_files': { 'Java': 'Java8.g4', - 'JavaScript': 'Java8.JavaScriptTarget.g4', 'TypeScript': 'Java8.TypeScriptTarget.g4', }, 'needs_java_func_data': true, diff --git a/language_configs/python3.compile.js b/language_configs/python3.compile.js index caa5aa8..9948351 100644 --- a/language_configs/python3.compile.js +++ b/language_configs/python3.compile.js @@ -3,7 +3,6 @@ module.exports = { 'grammar_files': { 'Java': 'Python3.g4', - 'JavaScript': 'Python3.JavaScriptTarget.g4', 'TypeScript': 'Python3.TypeScriptTarget.g4', }, 'tree_matcher_specs': require('./python3.tree_matcher_specs.js'), From 8e4729082d70e2b9145f3107de955cfc79eeffae Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Fri, 19 May 2017 11:00:51 -0500 Subject: [PATCH 4/6] Added C, Clojure, Java (not java8), and Lua --- grammars-v4 | 2 +- language_configs/c.compile.js | 9 + language_configs/c.rules.js | 194 ++++++++++++++++ language_configs/c.runtime.js | 7 + language_configs/c.tree_matcher_specs.js | 2 + language_configs/clojure.compile.js | 9 + language_configs/clojure.rules.js | 63 ++++++ language_configs/clojure.runtime.js | 7 + .../clojure.tree_matcher_specs.js | 2 + language_configs/java.compile.js | 9 + language_configs/java.rules.js | 214 ++++++++++++++++++ language_configs/java.runtime.js | 7 + language_configs/java.tree_matcher_specs.js | 2 + language_configs/java8.compile.js | 4 +- language_configs/lua.compile.js | 9 + language_configs/lua.rules.js | 55 +++++ language_configs/lua.runtime.js | 7 + language_configs/lua.tree_matcher_specs.js | 2 + language_configs/python3.compile.js | 4 +- mappings/c.csv | 190 ++++++++++++++++ mappings/clojure.csv | 59 +++++ mappings/java.csv | 210 +++++++++++++++++ mappings/lua.csv | 51 +++++ .../lang_config.compile.js | 5 +- src/utils/run_antlr.js | 9 +- 25 files changed, 1116 insertions(+), 16 deletions(-) create mode 100644 language_configs/c.compile.js create mode 100644 language_configs/c.rules.js create mode 100644 language_configs/c.runtime.js create mode 100644 language_configs/c.tree_matcher_specs.js create mode 100644 language_configs/clojure.compile.js create mode 100644 language_configs/clojure.rules.js create mode 100644 language_configs/clojure.runtime.js create mode 100644 language_configs/clojure.tree_matcher_specs.js create mode 100644 language_configs/java.compile.js create mode 100644 language_configs/java.rules.js create mode 100644 language_configs/java.runtime.js create mode 100644 language_configs/java.tree_matcher_specs.js create mode 100644 language_configs/lua.compile.js create mode 100644 language_configs/lua.rules.js create mode 100644 language_configs/lua.runtime.js create mode 100644 language_configs/lua.tree_matcher_specs.js create mode 100644 mappings/c.csv create mode 100644 mappings/clojure.csv create mode 100644 mappings/java.csv create mode 100644 mappings/lua.csv diff --git a/grammars-v4 b/grammars-v4 index 9975d18..4184d19 160000 --- a/grammars-v4 +++ b/grammars-v4 @@ -1 +1 @@ -Subproject commit 9975d18368efe37dd4f45058affd21000493d91f +Subproject commit 4184d196dd272be943fa6368d7ad2ade4198d65b diff --git a/language_configs/c.compile.js b/language_configs/c.compile.js new file mode 100644 index 0000000..b19bc74 --- /dev/null +++ b/language_configs/c.compile.js @@ -0,0 +1,9 @@ +// Configuration that can only be read by the compiler. + +module.exports = { + 'grammar_files': { + 'Java': 'grammars-v4/c/C.g4', + 'TypeScript': 'grammars-v4/c/C.g4', + }, + 'tree_matcher_specs': require('./c.tree_matcher_specs.js'), +}; diff --git a/language_configs/c.rules.js b/language_configs/c.rules.js new file mode 100644 index 0000000..3920910 --- /dev/null +++ b/language_configs/c.rules.js @@ -0,0 +1,194 @@ +// Configuration for each possible node type. +// Currently, the only option is 'collapse', which is used by the collapse transformer. + +module.exports = { + 'primaryExpression': {}, + 'genericSelection': {}, + 'genericAssocList': {}, + 'genericAssociation': {}, + 'postfixExpression': {}, + 'argumentExpressionList': {}, + 'unaryExpression': {}, + 'unaryOperator': {}, + 'castExpression': {}, + 'multiplicativeExpression': {}, + 'additiveExpression': {}, + 'shiftExpression': {}, + 'relationalExpression': {}, + 'equalityExpression': {}, + 'andExpression': {}, + 'exclusiveOrExpression': {}, + 'inclusiveOrExpression': {}, + 'logicalAndExpression': {}, + 'logicalOrExpression': {}, + 'conditionalExpression': {}, + 'assignmentExpression': {}, + 'assignmentOperator': {}, + 'expression': {}, + 'constantExpression': {}, + 'declaration': {}, + 'declarationSpecifiers': {}, + 'declarationSpecifiers2': {}, + 'declarationSpecifier': {}, + 'initDeclaratorList': {}, + 'initDeclarator': {}, + 'storageClassSpecifier': {}, + 'typeSpecifier': {}, + 'structOrUnionSpecifier': {}, + 'structOrUnion': {}, + 'structDeclarationList': {}, + 'structDeclaration': {}, + 'specifierQualifierList': {}, + 'structDeclaratorList': {}, + 'structDeclarator': {}, + 'enumSpecifier': {}, + 'enumeratorList': {}, + 'enumerator': {}, + 'enumerationConstant': {}, + 'atomicTypeSpecifier': {}, + 'typeQualifier': {}, + 'functionSpecifier': {}, + 'alignmentSpecifier': {}, + 'declarator': {}, + 'directDeclarator': {}, + 'gccDeclaratorExtension': {}, + 'gccAttributeSpecifier': {}, + 'gccAttributeList': {}, + 'gccAttribute': {}, + 'nestedParenthesesBlock': {}, + 'pointer': {}, + 'typeQualifierList': {}, + 'parameterTypeList': {}, + 'parameterList': {}, + 'parameterDeclaration': {}, + 'identifierList': {}, + 'typeName': {}, + 'abstractDeclarator': {}, + 'directAbstractDeclarator': {}, + 'typedefName': {}, + 'initializer': {}, + 'initializerList': {}, + 'designation': {}, + 'designatorList': {}, + 'designator': {}, + 'staticAssertDeclaration': {}, + 'statement': {}, + 'labeledStatement': {}, + 'compoundStatement': {}, + 'blockItemList': {}, + 'blockItem': {}, + 'expressionStatement': {}, + 'selectionStatement': {}, + 'iterationStatement': {}, + 'jumpStatement': {}, + 'compilationUnit': {}, + 'translationUnit': {}, + 'externalDeclaration': {}, + 'functionDefinition': {}, + 'declarationList': {}, + '._EPSILON': {}, + '._EOF': {}, + '._INVALID': {}, + '.Auto': {}, + '.Break': {}, + '.Case': {}, + '.Char': {}, + '.Const': {}, + '.Continue': {}, + '.Default': {}, + '.Do': {}, + '.Double': {}, + '.Else': {}, + '.Enum': {}, + '.Extern': {}, + '.Float': {}, + '.For': {}, + '.Goto': {}, + '.If': {}, + '.Inline': {}, + '.Int': {}, + '.Long': {}, + '.Register': {}, + '.Restrict': {}, + '.Return': {}, + '.Short': {}, + '.Signed': {}, + '.Sizeof': {}, + '.Static': {}, + '.Struct': {}, + '.Switch': {}, + '.Typedef': {}, + '.Union': {}, + '.Unsigned': {}, + '.Void': {}, + '.Volatile': {}, + '.While': {}, + '.Alignas': {}, + '.Alignof': {}, + '.Atomic': {}, + '.Bool': {}, + '.Complex': {}, + '.Generic': {}, + '.Imaginary': {}, + '.Noreturn': {}, + '.StaticAssert': {}, + '.ThreadLocal': {}, + '.LeftParen': {}, + '.RightParen': {}, + '.LeftBracket': {}, + '.RightBracket': {}, + '.LeftBrace': {}, + '.RightBrace': {}, + '.Less': {}, + '.LessEqual': {}, + '.Greater': {}, + '.GreaterEqual': {}, + '.LeftShift': {}, + '.RightShift': {}, + '.Plus': {}, + '.PlusPlus': {}, + '.Minus': {}, + '.MinusMinus': {}, + '.Star': {}, + '.Div': {}, + '.Mod': {}, + '.And': {}, + '.Or': {}, + '.AndAnd': {}, + '.OrOr': {}, + '.Caret': {}, + '.Not': {}, + '.Tilde': {}, + '.Question': {}, + '.Colon': {}, + '.Semi': {}, + '.Comma': {}, + '.Assign': {}, + '.StarAssign': {}, + '.DivAssign': {}, + '.ModAssign': {}, + '.PlusAssign': {}, + '.MinusAssign': {}, + '.LeftShiftAssign': {}, + '.RightShiftAssign': {}, + '.AndAssign': {}, + '.XorAssign': {}, + '.OrAssign': {}, + '.Equal': {}, + '.NotEqual': {}, + '.Arrow': {}, + '.Dot': {}, + '.Ellipsis': {}, + '.Identifier': {}, + '.Constant': {}, + '.StringLiteral': {}, + '.ComplexDefine': {}, + '.AsmBlock': {}, + '.LineAfterPreprocessing': {}, + '.LineDirective': {}, + '.PragmaDirective': {}, + '.Whitespace': {}, + '.Newline': {}, + '.BlockComment': {}, + '.LineComment': {}, +}; diff --git a/language_configs/c.runtime.js b/language_configs/c.runtime.js new file mode 100644 index 0000000..2129a9e --- /dev/null +++ b/language_configs/c.runtime.js @@ -0,0 +1,7 @@ +// Configuration that can be read by both the compiler and the runtime. + +module.exports = { + 'language': 'C', + 'entry_rule': 'compilationUnit', + 'rules': require('./c.rules.js'), +}; diff --git a/language_configs/c.tree_matcher_specs.js b/language_configs/c.tree_matcher_specs.js new file mode 100644 index 0000000..c767c65 --- /dev/null +++ b/language_configs/c.tree_matcher_specs.js @@ -0,0 +1,2 @@ +module.exports = [ +]; diff --git a/language_configs/clojure.compile.js b/language_configs/clojure.compile.js new file mode 100644 index 0000000..ea83451 --- /dev/null +++ b/language_configs/clojure.compile.js @@ -0,0 +1,9 @@ +// Configuration that can only be read by the compiler. + +module.exports = { + 'grammar_files': { + 'Java': 'grammars-v4/clojure/Clojure.g4', + 'TypeScript': 'grammars-v4/clojure/Clojure.g4', + }, + 'tree_matcher_specs': require('./clojure.tree_matcher_specs.js'), +}; diff --git a/language_configs/clojure.rules.js b/language_configs/clojure.rules.js new file mode 100644 index 0000000..88f9581 --- /dev/null +++ b/language_configs/clojure.rules.js @@ -0,0 +1,63 @@ +// Configuration for each possible node type. +// Currently, the only option is 'collapse', which is used by the collapse transformer. + +module.exports = { + 'file': {}, + 'form': {}, + 'forms': {}, + 'list': {}, + 'vector': {}, + 'map': {}, + 'set': {}, + 'reader_macro': {}, + 'quote': {}, + 'backtick': {}, + 'unquote': {}, + 'unquote_splicing': {}, + 'tag': {}, + 'deref': {}, + 'gensym': {}, + 'lambda': {}, + 'meta_data': {}, + 'var_quote': {}, + 'host_expr': {}, + 'discard': {}, + 'dispatch': {}, + 'regex': {}, + 'literal': {}, + 'string': {}, + 'hex': {}, + 'bin': {}, + 'bign': {}, + 'number': {}, + 'character': {}, + 'named_char': {}, + 'any_char': {}, + 'u_hex_quad': {}, + 'nil': {}, + 'keyword': {}, + 'simple_keyword': {}, + 'macro_keyword': {}, + 'symbol': {}, + 'simple_sym': {}, + 'ns_symbol': {}, + 'param_name': {}, + '._EPSILON': {}, + '._EOF': {}, + '._INVALID': {}, + '.STRING': {}, + '.FLOAT': {}, + '.HEX': {}, + '.BIN': {}, + '.LONG': {}, + '.BIGN': {}, + '.CHAR_U': {}, + '.CHAR_NAMED': {}, + '.CHAR_ANY': {}, + '.NIL': {}, + '.BOOLEAN': {}, + '.SYMBOL': {}, + '.NS_SYMBOL': {}, + '.PARAM_NAME': {}, + '.TRASH': {}, +}; diff --git a/language_configs/clojure.runtime.js b/language_configs/clojure.runtime.js new file mode 100644 index 0000000..c96fc08 --- /dev/null +++ b/language_configs/clojure.runtime.js @@ -0,0 +1,7 @@ +// Configuration that can be read by both the compiler and the runtime. + +module.exports = { + 'language': 'Clojure', + 'entry_rule': 'file', + 'rules': require('./clojure.rules.js'), +}; diff --git a/language_configs/clojure.tree_matcher_specs.js b/language_configs/clojure.tree_matcher_specs.js new file mode 100644 index 0000000..c767c65 --- /dev/null +++ b/language_configs/clojure.tree_matcher_specs.js @@ -0,0 +1,2 @@ +module.exports = [ +]; diff --git a/language_configs/java.compile.js b/language_configs/java.compile.js new file mode 100644 index 0000000..b4bd15d --- /dev/null +++ b/language_configs/java.compile.js @@ -0,0 +1,9 @@ +// Configuration that can only be read by the compiler. + +module.exports = { + 'grammar_files': { + 'Java': 'grammars-v4/java/Java.g4', + 'TypeScript': 'grammars-v4/java/Java.g4', + }, + 'tree_matcher_specs': require('./java.tree_matcher_specs.js'), +}; diff --git a/language_configs/java.rules.js b/language_configs/java.rules.js new file mode 100644 index 0000000..1bdfeef --- /dev/null +++ b/language_configs/java.rules.js @@ -0,0 +1,214 @@ +// Configuration for each possible node type. +// Currently, the only option is 'collapse', which is used by the collapse transformer. + +module.exports = { + 'compilationUnit': {}, + 'packageDeclaration': {}, + 'importDeclaration': {}, + 'typeDeclaration': {}, + 'modifier': {}, + 'classOrInterfaceModifier': {}, + 'variableModifier': {}, + 'classDeclaration': {}, + 'typeParameters': {}, + 'typeParameter': {}, + 'typeBound': {}, + 'enumDeclaration': {}, + 'enumConstants': {}, + 'enumConstant': {}, + 'enumBodyDeclarations': {}, + 'interfaceDeclaration': {}, + 'typeList': {}, + 'classBody': {}, + 'interfaceBody': {}, + 'classBodyDeclaration': {}, + 'memberDeclaration': {}, + 'methodDeclaration': {}, + 'genericMethodDeclaration': {}, + 'constructorDeclaration': {}, + 'genericConstructorDeclaration': {}, + 'fieldDeclaration': {}, + 'interfaceBodyDeclaration': {}, + 'interfaceMemberDeclaration': {}, + 'constDeclaration': {}, + 'constantDeclarator': {}, + 'interfaceMethodDeclaration': {}, + 'genericInterfaceMethodDeclaration': {}, + 'variableDeclarators': {}, + 'variableDeclarator': {}, + 'variableDeclaratorId': {}, + 'variableInitializer': {}, + 'arrayInitializer': {}, + 'enumConstantName': {}, + 'typeType': {}, + 'classOrInterfaceType': {}, + 'primitiveType': {}, + 'typeArguments': {}, + 'typeArgument': {}, + 'qualifiedNameList': {}, + 'formalParameters': {}, + 'formalParameterList': {}, + 'formalParameter': {}, + 'lastFormalParameter': {}, + 'methodBody': {}, + 'constructorBody': {}, + 'qualifiedName': {}, + 'literal': {}, + 'annotation': {}, + 'annotationName': {}, + 'elementValuePairs': {}, + 'elementValuePair': {}, + 'elementValue': {}, + 'elementValueArrayInitializer': {}, + 'annotationTypeDeclaration': {}, + 'annotationTypeBody': {}, + 'annotationTypeElementDeclaration': {}, + 'annotationTypeElementRest': {}, + 'annotationMethodOrConstantRest': {}, + 'annotationMethodRest': {}, + 'annotationConstantRest': {}, + 'defaultValue': {}, + 'block': {}, + 'blockStatement': {}, + 'localVariableDeclarationStatement': {}, + 'localVariableDeclaration': {}, + 'statement': {}, + 'catchClause': {}, + 'catchType': {}, + 'finallyBlock': {}, + 'resourceSpecification': {}, + 'resources': {}, + 'resource': {}, + 'switchBlockStatementGroup': {}, + 'switchLabel': {}, + 'forControl': {}, + 'forInit': {}, + 'enhancedForControl': {}, + 'forUpdate': {}, + 'parExpression': {}, + 'expressionList': {}, + 'statementExpression': {}, + 'constantExpression': {}, + 'expression': {}, + 'primary': {}, + 'creator': {}, + 'createdName': {}, + 'innerCreator': {}, + 'arrayCreatorRest': {}, + 'classCreatorRest': {}, + 'explicitGenericInvocation': {}, + 'nonWildcardTypeArguments': {}, + 'typeArgumentsOrDiamond': {}, + 'nonWildcardTypeArgumentsOrDiamond': {}, + 'superSuffix': {}, + 'explicitGenericInvocationSuffix': {}, + 'arguments': {}, + '._EPSILON': {}, + '._EOF': {}, + '._INVALID': {}, + '.ABSTRACT': {}, + '.ASSERT': {}, + '.BOOLEAN': {}, + '.BREAK': {}, + '.BYTE': {}, + '.CASE': {}, + '.CATCH': {}, + '.CHAR': {}, + '.CLASS': {}, + '.CONST': {}, + '.CONTINUE': {}, + '.DEFAULT': {}, + '.DO': {}, + '.DOUBLE': {}, + '.ELSE': {}, + '.ENUM': {}, + '.EXTENDS': {}, + '.FINAL': {}, + '.FINALLY': {}, + '.FLOAT': {}, + '.FOR': {}, + '.IF': {}, + '.GOTO': {}, + '.IMPLEMENTS': {}, + '.IMPORT': {}, + '.INSTANCEOF': {}, + '.INT': {}, + '.INTERFACE': {}, + '.LONG': {}, + '.NATIVE': {}, + '.NEW': {}, + '.PACKAGE': {}, + '.PRIVATE': {}, + '.PROTECTED': {}, + '.PUBLIC': {}, + '.RETURN': {}, + '.SHORT': {}, + '.STATIC': {}, + '.STRICTFP': {}, + '.SUPER': {}, + '.SWITCH': {}, + '.SYNCHRONIZED': {}, + '.THIS': {}, + '.THROW': {}, + '.THROWS': {}, + '.TRANSIENT': {}, + '.TRY': {}, + '.VOID': {}, + '.VOLATILE': {}, + '.WHILE': {}, + '.IntegerLiteral': {}, + '.FloatingPointLiteral': {}, + '.BooleanLiteral': {}, + '.CharacterLiteral': {}, + '.StringLiteral': {}, + '.NullLiteral': {}, + '.LPAREN': {}, + '.RPAREN': {}, + '.LBRACE': {}, + '.RBRACE': {}, + '.LBRACK': {}, + '.RBRACK': {}, + '.SEMI': {}, + '.COMMA': {}, + '.DOT': {}, + '.ASSIGN': {}, + '.GT': {}, + '.LT': {}, + '.BANG': {}, + '.TILDE': {}, + '.QUESTION': {}, + '.COLON': {}, + '.EQUAL': {}, + '.LE': {}, + '.GE': {}, + '.NOTEQUAL': {}, + '.AND': {}, + '.OR': {}, + '.INC': {}, + '.DEC': {}, + '.ADD': {}, + '.SUB': {}, + '.MUL': {}, + '.DIV': {}, + '.BITAND': {}, + '.BITOR': {}, + '.CARET': {}, + '.MOD': {}, + '.ADD_ASSIGN': {}, + '.SUB_ASSIGN': {}, + '.MUL_ASSIGN': {}, + '.DIV_ASSIGN': {}, + '.AND_ASSIGN': {}, + '.OR_ASSIGN': {}, + '.XOR_ASSIGN': {}, + '.MOD_ASSIGN': {}, + '.LSHIFT_ASSIGN': {}, + '.RSHIFT_ASSIGN': {}, + '.URSHIFT_ASSIGN': {}, + '.Identifier': {}, + '.AT': {}, + '.ELLIPSIS': {}, + '.WS': {}, + '.COMMENT': {}, + '.LINE_COMMENT': {}, +}; diff --git a/language_configs/java.runtime.js b/language_configs/java.runtime.js new file mode 100644 index 0000000..db065fd --- /dev/null +++ b/language_configs/java.runtime.js @@ -0,0 +1,7 @@ +// Configuration that can be read by both the compiler and the runtime. + +module.exports = { + 'language': 'Java', + 'entry_rule': 'compilationUnit', + 'rules': require('./java.rules.js'), +}; diff --git a/language_configs/java.tree_matcher_specs.js b/language_configs/java.tree_matcher_specs.js new file mode 100644 index 0000000..c767c65 --- /dev/null +++ b/language_configs/java.tree_matcher_specs.js @@ -0,0 +1,2 @@ +module.exports = [ +]; diff --git a/language_configs/java8.compile.js b/language_configs/java8.compile.js index 331ced9..1ee5620 100644 --- a/language_configs/java8.compile.js +++ b/language_configs/java8.compile.js @@ -2,8 +2,8 @@ module.exports = { 'grammar_files': { - 'Java': 'Java8.g4', - 'TypeScript': 'Java8.TypeScriptTarget.g4', + 'Java': 'grammars-v4/java8/Java8.g4', + 'TypeScript': 'grammars-v4/java8/Java8.TypeScriptTarget.g4', }, 'needs_java_func_data': true, 'tree_matcher_specs': require('./java8.tree_matcher_specs.js'), diff --git a/language_configs/lua.compile.js b/language_configs/lua.compile.js new file mode 100644 index 0000000..8fec671 --- /dev/null +++ b/language_configs/lua.compile.js @@ -0,0 +1,9 @@ +// Configuration that can only be read by the compiler. + +module.exports = { + 'grammar_files': { + 'Java': 'grammars-v4/lua/Lua.g4', + 'TypeScript': 'grammars-v4/lua/Lua.g4', + }, + 'tree_matcher_specs': require('./lua.tree_matcher_specs.js'), +}; diff --git a/language_configs/lua.rules.js b/language_configs/lua.rules.js new file mode 100644 index 0000000..a1458bf --- /dev/null +++ b/language_configs/lua.rules.js @@ -0,0 +1,55 @@ +// Configuration for each possible node type. +// Currently, the only option is 'collapse', which is used by the collapse transformer. + +module.exports = { + 'chunk': {}, + 'block': {}, + 'stat': {}, + 'retstat': {}, + 'label': {}, + 'funcname': {}, + 'varlist': {}, + 'namelist': {}, + 'explist': {}, + 'exp': {}, + 'prefixexp': {}, + 'functioncall': {}, + 'varOrExp': {}, + 'var': {}, + 'varSuffix': {}, + 'nameAndArgs': {}, + 'args': {}, + 'functiondef': {}, + 'funcbody': {}, + 'parlist': {}, + 'tableconstructor': {}, + 'fieldlist': {}, + 'field': {}, + 'fieldsep': {}, + 'operatorOr': {}, + 'operatorAnd': {}, + 'operatorComparison': {}, + 'operatorStrcat': {}, + 'operatorAddSub': {}, + 'operatorMulDivMod': {}, + 'operatorBitwise': {}, + 'operatorUnary': {}, + 'operatorPower': {}, + 'number': {}, + 'string': {}, + '._EPSILON': {}, + '._EOF': {}, + '._INVALID': {}, + '.NAME': {}, + '.NORMALSTRING': {}, + '.CHARSTRING': {}, + '.LONGSTRING': {}, + '.INT': {}, + '.HEX': {}, + '.FLOAT': {}, + '.HEX_FLOAT': {}, + '.COMMENT': {}, + '.LINE_COMMENT': {}, + '.WS': {}, + '.SHEBANG': {}, +}; diff --git a/language_configs/lua.runtime.js b/language_configs/lua.runtime.js new file mode 100644 index 0000000..d06831c --- /dev/null +++ b/language_configs/lua.runtime.js @@ -0,0 +1,7 @@ +// Configuration that can be read by both the compiler and the runtime. + +module.exports = { + 'language': 'Lua', + 'entry_rule': 'chunk', + 'rules': require('./lua.rules.js'), +}; diff --git a/language_configs/lua.tree_matcher_specs.js b/language_configs/lua.tree_matcher_specs.js new file mode 100644 index 0000000..c767c65 --- /dev/null +++ b/language_configs/lua.tree_matcher_specs.js @@ -0,0 +1,2 @@ +module.exports = [ +]; diff --git a/language_configs/python3.compile.js b/language_configs/python3.compile.js index 9948351..4bc3051 100644 --- a/language_configs/python3.compile.js +++ b/language_configs/python3.compile.js @@ -2,8 +2,8 @@ module.exports = { 'grammar_files': { - 'Java': 'Python3.g4', - 'TypeScript': 'Python3.TypeScriptTarget.g4', + 'Java': 'grammars-v4/python3/Python3.g4', + 'TypeScript': 'grammars-v4/python3/Python3.TypeScriptTarget.g4', }, 'tree_matcher_specs': require('./python3.tree_matcher_specs.js'), }; diff --git a/mappings/c.csv b/mappings/c.csv new file mode 100644 index 0000000..91e34a9 --- /dev/null +++ b/mappings/c.csv @@ -0,0 +1,190 @@ +RULE,ENABLED,PRETTY_NAME,COLOR +primaryExpression,0,, +genericSelection,0,, +genericAssocList,0,, +genericAssociation,0,, +postfixExpression,0,, +argumentExpressionList,0,, +unaryExpression,0,, +unaryOperator,0,, +castExpression,0,, +multiplicativeExpression,0,, +additiveExpression,0,, +shiftExpression,0,, +relationalExpression,0,, +equalityExpression,0,, +andExpression,0,, +exclusiveOrExpression,0,, +inclusiveOrExpression,0,, +logicalAndExpression,0,, +logicalOrExpression,0,, +conditionalExpression,0,, +assignmentExpression,0,, +assignmentOperator,0,, +expression,0,, +constantExpression,0,, +declaration,0,, +declarationSpecifiers,0,, +declarationSpecifiers2,0,, +declarationSpecifier,0,, +initDeclaratorList,0,, +initDeclarator,0,, +storageClassSpecifier,0,, +typeSpecifier,0,, +structOrUnionSpecifier,0,, +structOrUnion,0,, +structDeclarationList,0,, +structDeclaration,0,, +specifierQualifierList,0,, +structDeclaratorList,0,, +structDeclarator,0,, +enumSpecifier,0,, +enumeratorList,0,, +enumerator,0,, +enumerationConstant,0,, +atomicTypeSpecifier,0,, +typeQualifier,0,, +functionSpecifier,0,, +alignmentSpecifier,0,, +declarator,0,, +directDeclarator,0,, +gccDeclaratorExtension,0,, +gccAttributeSpecifier,0,, +gccAttributeList,0,, +gccAttribute,0,, +nestedParenthesesBlock,0,, +pointer,0,, +typeQualifierList,0,, +parameterTypeList,0,, +parameterList,0,, +parameterDeclaration,0,, +identifierList,0,, +typeName,0,, +abstractDeclarator,0,, +directAbstractDeclarator,0,, +typedefName,0,, +initializer,0,, +initializerList,0,, +designation,0,, +designatorList,0,, +designator,0,, +staticAssertDeclaration,0,, +statement,0,, +labeledStatement,0,, +compoundStatement,0,, +blockItemList,0,, +blockItem,0,, +expressionStatement,0,, +selectionStatement,0,, +iterationStatement,0,, +jumpStatement,0,, +compilationUnit,0,, +translationUnit,0,, +externalDeclaration,0,, +functionDefinition,0,, +declarationList,0,, +._EPSILON,0,, +._EOF,0,, +._INVALID,0,, +.Auto,0,, +.Break,0,, +.Case,0,, +.Char,0,, +.Const,0,, +.Continue,0,, +.Default,0,, +.Do,0,, +.Double,0,, +.Else,0,, +.Enum,0,, +.Extern,0,, +.Float,0,, +.For,0,, +.Goto,0,, +.If,0,, +.Inline,0,, +.Int,0,, +.Long,0,, +.Register,0,, +.Restrict,0,, +.Return,0,, +.Short,0,, +.Signed,0,, +.Sizeof,0,, +.Static,0,, +.Struct,0,, +.Switch,0,, +.Typedef,0,, +.Union,0,, +.Unsigned,0,, +.Void,0,, +.Volatile,0,, +.While,0,, +.Alignas,0,, +.Alignof,0,, +.Atomic,0,, +.Bool,0,, +.Complex,0,, +.Generic,0,, +.Imaginary,0,, +.Noreturn,0,, +.StaticAssert,0,, +.ThreadLocal,0,, +.LeftParen,0,, +.RightParen,0,, +.LeftBracket,0,, +.RightBracket,0,, +.LeftBrace,0,, +.RightBrace,0,, +.Less,0,, +.LessEqual,0,, +.Greater,0,, +.GreaterEqual,0,, +.LeftShift,0,, +.RightShift,0,, +.Plus,0,, +.PlusPlus,0,, +.Minus,0,, +.MinusMinus,0,, +.Star,0,, +.Div,0,, +.Mod,0,, +.And,0,, +.Or,0,, +.AndAnd,0,, +.OrOr,0,, +.Caret,0,, +.Not,0,, +.Tilde,0,, +.Question,0,, +.Colon,0,, +.Semi,0,, +.Comma,0,, +.Assign,0,, +.StarAssign,0,, +.DivAssign,0,, +.ModAssign,0,, +.PlusAssign,0,, +.MinusAssign,0,, +.LeftShiftAssign,0,, +.RightShiftAssign,0,, +.AndAssign,0,, +.XorAssign,0,, +.OrAssign,0,, +.Equal,0,, +.NotEqual,0,, +.Arrow,0,, +.Dot,0,, +.Ellipsis,0,, +.Identifier,0,, +.Constant,0,, +.StringLiteral,0,, +.ComplexDefine,0,, +.AsmBlock,0,, +.LineAfterPreprocessing,0,, +.LineDirective,0,, +.PragmaDirective,0,, +.Whitespace,0,, +.Newline,0,, +.BlockComment,0,, +.LineComment,0,, diff --git a/mappings/clojure.csv b/mappings/clojure.csv new file mode 100644 index 0000000..362e8f3 --- /dev/null +++ b/mappings/clojure.csv @@ -0,0 +1,59 @@ +RULE,ENABLED,PRETTY_NAME,COLOR +file,0,, +form,0,, +forms,0,, +list,0,, +vector,0,, +map,0,, +set,0,, +reader_macro,0,, +quote,0,, +backtick,0,, +unquote,0,, +unquote_splicing,0,, +tag,0,, +deref,0,, +gensym,0,, +lambda,0,, +meta_data,0,, +var_quote,0,, +host_expr,0,, +discard,0,, +dispatch,0,, +regex,0,, +literal,0,, +string,0,, +hex,0,, +bin,0,, +bign,0,, +number,0,, +character,0,, +named_char,0,, +any_char,0,, +u_hex_quad,0,, +nil,0,, +keyword,0,, +simple_keyword,0,, +macro_keyword,0,, +symbol,0,, +simple_sym,0,, +ns_symbol,0,, +param_name,0,, +._EPSILON,0,, +._EOF,0,, +._INVALID,0,, +.STRING,0,, +.FLOAT,0,, +.HEX,0,, +.BIN,0,, +.LONG,0,, +.BIGN,0,, +.CHAR_U,0,, +.CHAR_NAMED,0,, +.CHAR_ANY,0,, +.NIL,0,, +.BOOLEAN,0,, +.SYMBOL,0,, +.NS_SYMBOL,0,, +.PARAM_NAME,0,, +.TRASH,0,, diff --git a/mappings/java.csv b/mappings/java.csv new file mode 100644 index 0000000..4e1b417 --- /dev/null +++ b/mappings/java.csv @@ -0,0 +1,210 @@ +RULE,ENABLED,PRETTY_NAME,COLOR +compilationUnit,0,, +packageDeclaration,0,, +importDeclaration,0,, +typeDeclaration,0,, +modifier,0,, +classOrInterfaceModifier,0,, +variableModifier,0,, +classDeclaration,0,, +typeParameters,0,, +typeParameter,0,, +typeBound,0,, +enumDeclaration,0,, +enumConstants,0,, +enumConstant,0,, +enumBodyDeclarations,0,, +interfaceDeclaration,0,, +typeList,0,, +classBody,0,, +interfaceBody,0,, +classBodyDeclaration,0,, +memberDeclaration,0,, +methodDeclaration,0,, +genericMethodDeclaration,0,, +constructorDeclaration,0,, +genericConstructorDeclaration,0,, +fieldDeclaration,0,, +interfaceBodyDeclaration,0,, +interfaceMemberDeclaration,0,, +constDeclaration,0,, +constantDeclarator,0,, +interfaceMethodDeclaration,0,, +genericInterfaceMethodDeclaration,0,, +variableDeclarators,0,, +variableDeclarator,0,, +variableDeclaratorId,0,, +variableInitializer,0,, +arrayInitializer,0,, +enumConstantName,0,, +typeType,0,, +classOrInterfaceType,0,, +primitiveType,0,, +typeArguments,0,, +typeArgument,0,, +qualifiedNameList,0,, +formalParameters,0,, +formalParameterList,0,, +formalParameter,0,, +lastFormalParameter,0,, +methodBody,0,, +constructorBody,0,, +qualifiedName,0,, +literal,0,, +annotation,0,, +annotationName,0,, +elementValuePairs,0,, +elementValuePair,0,, +elementValue,0,, +elementValueArrayInitializer,0,, +annotationTypeDeclaration,0,, +annotationTypeBody,0,, +annotationTypeElementDeclaration,0,, +annotationTypeElementRest,0,, +annotationMethodOrConstantRest,0,, +annotationMethodRest,0,, +annotationConstantRest,0,, +defaultValue,0,, +block,0,, +blockStatement,0,, +localVariableDeclarationStatement,0,, +localVariableDeclaration,0,, +statement,0,, +catchClause,0,, +catchType,0,, +finallyBlock,0,, +resourceSpecification,0,, +resources,0,, +resource,0,, +switchBlockStatementGroup,0,, +switchLabel,0,, +forControl,0,, +forInit,0,, +enhancedForControl,0,, +forUpdate,0,, +parExpression,0,, +expressionList,0,, +statementExpression,0,, +constantExpression,0,, +expression,0,, +primary,0,, +creator,0,, +createdName,0,, +innerCreator,0,, +arrayCreatorRest,0,, +classCreatorRest,0,, +explicitGenericInvocation,0,, +nonWildcardTypeArguments,0,, +typeArgumentsOrDiamond,0,, +nonWildcardTypeArgumentsOrDiamond,0,, +superSuffix,0,, +explicitGenericInvocationSuffix,0,, +arguments,0,, +._EPSILON,0,, +._EOF,0,, +._INVALID,0,, +.ABSTRACT,0,, +.ASSERT,0,, +.BOOLEAN,0,, +.BREAK,0,, +.BYTE,0,, +.CASE,0,, +.CATCH,0,, +.CHAR,0,, +.CLASS,0,, +.CONST,0,, +.CONTINUE,0,, +.DEFAULT,0,, +.DO,0,, +.DOUBLE,0,, +.ELSE,0,, +.ENUM,0,, +.EXTENDS,0,, +.FINAL,0,, +.FINALLY,0,, +.FLOAT,0,, +.FOR,0,, +.IF,0,, +.GOTO,0,, +.IMPLEMENTS,0,, +.IMPORT,0,, +.INSTANCEOF,0,, +.INT,0,, +.INTERFACE,0,, +.LONG,0,, +.NATIVE,0,, +.NEW,0,, +.PACKAGE,0,, +.PRIVATE,0,, +.PROTECTED,0,, +.PUBLIC,0,, +.RETURN,0,, +.SHORT,0,, +.STATIC,0,, +.STRICTFP,0,, +.SUPER,0,, +.SWITCH,0,, +.SYNCHRONIZED,0,, +.THIS,0,, +.THROW,0,, +.THROWS,0,, +.TRANSIENT,0,, +.TRY,0,, +.VOID,0,, +.VOLATILE,0,, +.WHILE,0,, +.IntegerLiteral,0,, +.FloatingPointLiteral,0,, +.BooleanLiteral,0,, +.CharacterLiteral,0,, +.StringLiteral,0,, +.NullLiteral,0,, +.LPAREN,0,, +.RPAREN,0,, +.LBRACE,0,, +.RBRACE,0,, +.LBRACK,0,, +.RBRACK,0,, +.SEMI,0,, +.COMMA,0,, +.DOT,0,, +.ASSIGN,0,, +.GT,0,, +.LT,0,, +.BANG,0,, +.TILDE,0,, +.QUESTION,0,, +.COLON,0,, +.EQUAL,0,, +.LE,0,, +.GE,0,, +.NOTEQUAL,0,, +.AND,0,, +.OR,0,, +.INC,0,, +.DEC,0,, +.ADD,0,, +.SUB,0,, +.MUL,0,, +.DIV,0,, +.BITAND,0,, +.BITOR,0,, +.CARET,0,, +.MOD,0,, +.ADD_ASSIGN,0,, +.SUB_ASSIGN,0,, +.MUL_ASSIGN,0,, +.DIV_ASSIGN,0,, +.AND_ASSIGN,0,, +.OR_ASSIGN,0,, +.XOR_ASSIGN,0,, +.MOD_ASSIGN,0,, +.LSHIFT_ASSIGN,0,, +.RSHIFT_ASSIGN,0,, +.URSHIFT_ASSIGN,0,, +.Identifier,0,, +.AT,0,, +.ELLIPSIS,0,, +.WS,0,, +.COMMENT,0,, +.LINE_COMMENT,0,, diff --git a/mappings/lua.csv b/mappings/lua.csv new file mode 100644 index 0000000..71fe6c6 --- /dev/null +++ b/mappings/lua.csv @@ -0,0 +1,51 @@ +RULE,ENABLED,PRETTY_NAME,COLOR +chunk,0,, +block,0,, +stat,0,, +retstat,0,, +label,0,, +funcname,0,, +varlist,0,, +namelist,0,, +explist,0,, +exp,0,, +prefixexp,0,, +functioncall,0,, +varOrExp,0,, +var,0,, +varSuffix,0,, +nameAndArgs,0,, +args,0,, +functiondef,0,, +funcbody,0,, +parlist,0,, +tableconstructor,0,, +fieldlist,0,, +field,0,, +fieldsep,0,, +operatorOr,0,, +operatorAnd,0,, +operatorComparison,0,, +operatorStrcat,0,, +operatorAddSub,0,, +operatorMulDivMod,0,, +operatorBitwise,0,, +operatorUnary,0,, +operatorPower,0,, +number,0,, +string,0,, +._EPSILON,0,, +._EOF,0,, +._INVALID,0,, +.NAME,0,, +.NORMALSTRING,0,, +.CHARSTRING,0,, +.LONGSTRING,0,, +.INT,0,, +.HEX,0,, +.FLOAT,0,, +.HEX_FLOAT,0,, +.COMMENT,0,, +.LINE_COMMENT,0,, +.WS,0,, +.SHEBANG,0,, diff --git a/src/tree_matcher_parser/lang_config.compile.js b/src/tree_matcher_parser/lang_config.compile.js index 16e68a2..8e5eceb 100644 --- a/src/tree_matcher_parser/lang_config.compile.js +++ b/src/tree_matcher_parser/lang_config.compile.js @@ -1,9 +1,8 @@ let path = require('path'); module.exports = { - 'grammar_dir': __dirname, 'grammar_files': { - 'JavaScript': 'TreeMatcher.g4', - 'TypeScript': 'TreeMatcher.g4', + 'JavaScript': 'src/tree_matcher_parser/TreeMatcher.g4', + 'TypeScript': 'src/tree_matcher_parser/TreeMatcher.g4', }, }; diff --git a/src/utils/run_antlr.js b/src/utils/run_antlr.js index ab0eaa9..65f8f61 100644 --- a/src/utils/run_antlr.js +++ b/src/utils/run_antlr.js @@ -7,7 +7,6 @@ const expect_error = require('./expect_error.js'); module.exports = async (lang_compile_config, lang_runtime_config, target_language) => { const { - grammar_dir, grammar_files, } = lang_compile_config; const { @@ -16,16 +15,10 @@ module.exports = async (lang_compile_config, lang_runtime_config, target_languag generate_listener, } = lang_runtime_config; - // Figure out the language key - const language_key = language.toLowerCase(); - - // Figure out the path to the grammar file - const g4_dir = grammar_dir ? grammar_dir : path.resolve(__dirname, '..', '..', 'grammars-v4', language_key); - if (typeof grammar_files[target_language] !== 'string') { throw new Error('No grammar file specified for target ' + target_language); } - const g4_path = path.resolve(g4_dir, grammar_files[target_language]); + const g4_path = path.resolve(__dirname, '..', '..', grammar_files[target_language]); const build_dir = config.resolve_build_dir(lang_runtime_config, target_language); const build_g4_path = path.resolve(build_dir, language + '.g4'); From 1c1d99c7fa19e413e4ff794b76e25f222490be73 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Fri, 19 May 2017 11:22:39 -0500 Subject: [PATCH 5/6] Added tests for new languages --- __tests__/tree/code/snippet.1v88olr0.clj | 18 ++ __tests__/tree/code/snippet.ecemhuve.c | 69 +++++ __tests__/tree/code/snippet.lk0rlo4z.java | 298 ++++++++++++++++++++++ __tests__/tree/code/snippet.seefwj9h.lua | 69 +++++ __tests__/tree/compare.test.js | 19 +- 5 files changed, 470 insertions(+), 3 deletions(-) create mode 100644 __tests__/tree/code/snippet.1v88olr0.clj create mode 100644 __tests__/tree/code/snippet.ecemhuve.c create mode 100644 __tests__/tree/code/snippet.lk0rlo4z.java create mode 100644 __tests__/tree/code/snippet.seefwj9h.lua diff --git a/__tests__/tree/code/snippet.1v88olr0.clj b/__tests__/tree/code/snippet.1v88olr0.clj new file mode 100644 index 0000000..c62d8f2 --- /dev/null +++ b/__tests__/tree/code/snippet.1v88olr0.clj @@ -0,0 +1,18 @@ +(defn run [nvecs nitems nthreads niters] + (let [vec-refs (->> (range (* nvecs nitems)) (partition nitems) (map (comp ref vec)) vec) + swap #(let [v1 (rand-int nvecs) + v2 (rand-int nvecs) + i1 (rand-int nitems) + i2 (rand-int nitems)] + (dosync + (let [tmp (nth @(vec-refs v1) i1)] + (alter (vec-refs v1) assoc i1 (nth @(vec-refs v2) i2)) + (alter (vec-refs v2) assoc i2 tmp)))) + report #(let [derefed (map deref vec-refs)] + (prn derefed) + (println "Distinct:" (->> derefed (apply concat) distinct count)))] + (report) + (dorun (apply pcalls (repeat nthreads #(dotimes [_ niters] (swap))))) + (report))) + +(run 100 10 10 100000) diff --git a/__tests__/tree/code/snippet.ecemhuve.c b/__tests__/tree/code/snippet.ecemhuve.c new file mode 100644 index 0000000..61df4ff --- /dev/null +++ b/__tests__/tree/code/snippet.ecemhuve.c @@ -0,0 +1,69 @@ +// Just a concatenation of some of the snippets from https://en.wikipedia.org/wiki/C_syntax + +enum colors { RED, GREEN, BLUE = 5, YELLOW } paint_color; + +struct thing { + int num; +}; /* thing struct type is now completed */ + +typedef struct Bert Bert; +typedef struct Wilma Wilma; + +struct Bert { + Wilma *wilma; +}; + +struct Wilma { + Bert *bert; +}; + +union u { + int x; + float y; + char *z; +} n; + +struct f { + unsigned int flag : 1; /* a bit flag: can either be on (1) or off (0) */ + signed int num : 4; /* a signed 4-bit field; range -7...7 or -8...7 */ + signed int : 3; /* 3 bits of padding to round out to 8 bits */ +} g; + +void func() { + int *p; + int a, b; + int (*ptr_to_array)[100] = &array; + + a = 10; + p = &a; + b = *p; + a = malloc(n * sizeof(int)); + a[3] = 10; + + for (int i = 0; i < limit; ++i) { + printf("helloworld.c: %d: Hello world\n", i); + } +} + +int (*operation)(int x, int y); + +int add(int x, int y) +{ + return x + y; +} + +int subtract(int x, int y) +{ + return x - y; +} + +int main(int argc, char* args[]) +{ + int foo = 1, bar = 1; + + operation = add; + printf("%d + %d = %d\n", foo, bar, operation(foo, bar)); + operation = subtract; + printf("%d - %d = %d\n", foo, bar, operation(foo, bar)); + return 0; +} diff --git a/__tests__/tree/code/snippet.lk0rlo4z.java b/__tests__/tree/code/snippet.lk0rlo4z.java new file mode 100644 index 0000000..fb22310 --- /dev/null +++ b/__tests__/tree/code/snippet.lk0rlo4z.java @@ -0,0 +1,298 @@ +// Shamelessly stolen from https://github.com/spullara/mustache.java/blob/master/compiler/src/main/java/com/github/mustachejava/MustacheParser.java + +package com.github.mustachejava; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * The parser generates callbacks into the MustacheFactory to build them. Do not use these + * directly as you must manage the Mustache object lifecycle as well. + *

+ * User: sam + * Date: 5/14/11 + * Time: 3:52 PM + */ +public class MustacheParser { + public static final String DEFAULT_SM = "{{"; + public static final String DEFAULT_EM = "}}"; + private MustacheFactory mf; + + protected MustacheParser(MustacheFactory mf) { + this.mf = mf; + } + + public Mustache compile(String file) { + Reader reader = mf.getReader(file); + if (reader == null) { + throw new MustacheNotFoundException(file); + } + return compile(reader, file); + } + + public Mustache compile(Reader reader, String file) { + return compile(reader, file, DEFAULT_SM, DEFAULT_EM); + } + + public Mustache compile(Reader reader, String file, String sm, String em) { + return compile(reader, null, new AtomicInteger(0), file, sm, em, true); + } + + public Mustache compile(Reader reader, String file, String sm, String em, boolean startOfLine) { + return compile(reader, null, new AtomicInteger(0), file, sm, em, startOfLine); + } + + @SuppressWarnings("ConstantConditions") // this method is too complex + protected Mustache compile(final Reader reader, String tag, final AtomicInteger currentLine, String file, String sm, String em, boolean startOfLine) throws MustacheException { + if (reader == null) { + throw new MustacheException("Reader is null"); + } + Reader br; + if (reader.markSupported()) { + br = reader; + } else { + br = new BufferedReader(reader); + } + try { + boolean sawCR = false; + int startLine = currentLine.get(); + MustacheVisitor mv = mf.createMustacheVisitor(); + // Now we grab the mustache template + boolean onlywhitespace = true; + // Starting a new line + boolean iterable = currentLine.get() != 0; + currentLine.compareAndSet(0, 1); + StringBuilder out = new StringBuilder(); + try { + int c; + while ((c = br.read()) != -1) { + // We remember that we saw a carriage return so we can put it back in later + if (c == '\r') { + sawCR = true; + continue; + } + // Increment the line + if (c == '\n') { + currentLine.incrementAndGet(); + if (!iterable || (iterable && !onlywhitespace)) { + if (sawCR) out.append("\r"); + out.append("\n"); + } + out = write(mv, out, file, currentLine.intValue(), startOfLine); + + iterable = false; + onlywhitespace = true; + startOfLine = true; + continue; + } + sawCR = false; + // Check for a mustache start + if (c == sm.charAt(0)) { + br.mark(1); + if (sm.length() == 1 || br.read() == sm.charAt(1)) { + // If it is a delimiter change we need to specially handle it + // Two mustaches, now capture command + StringBuilder sb = new StringBuilder(); + br.mark(1); + c = br.read(); + boolean delimiter = c == '='; + if (delimiter) { + sb.append((char) c); + } else { + br.reset(); + } + while ((c = br.read()) != -1) { + br.mark(1); + if (delimiter) { + if (c == '=') { + // Reached the end of the definition + delimiter = false; + } else { + sb.append((char) c); + } + continue; + } + if (c == em.charAt(0)) { + if (em.length() > 1) { + if (br.read() == em.charAt(1)) { + // Matched end + break; + } else { + // Only one + br.reset(); + } + } else break; + } + sb.append((char) c); + } + final String command = mf.translate(sb.toString()); + if (command.length() == 0) { + TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); + throw new MustacheException("Empty mustache", tc); + } + final char ch = command.charAt(0); + final String variable = command.substring(1).trim(); + switch (ch) { + case '#': + case '^': + case '<': + case '$': { + boolean oldStartOfLine = startOfLine; + startOfLine = startOfLine & onlywhitespace; + int line = currentLine.get(); + final Mustache mustache = compile(br, variable, currentLine, file, sm, em, startOfLine); + int lines = currentLine.get() - line; + if (!onlywhitespace || lines == 0) { + write(mv, out, file, currentLine.intValue(), oldStartOfLine); + } + out = new StringBuilder(); + switch (ch) { + case '#': + mv.iterable(new TemplateContext(sm, em, file, line, startOfLine), variable, mustache); + break; + case '^': + mv.notIterable(new TemplateContext(sm, em, file, line, startOfLine), variable, mustache); + break; + case '<': + mv.extend(new TemplateContext(sm, em, file, line, startOfLine), variable, mustache); + break; + case '$': + mv.name(new TemplateContext(sm, em, file, line, startOfLine), variable, mustache); + break; + } + iterable = lines != 0; + break; + } + case '/': { + // Tag end + if (!startOfLine || !onlywhitespace) { + write(mv, out, file, currentLine.intValue(), startOfLine); + } + if (!variable.equals(tag)) { + TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); + throw new MustacheException( + "Mismatched start/end tags: " + tag + " != " + variable + " in " + file + ":" + currentLine, tc); + } + + return mv.mustache(new TemplateContext(sm, em, file, 0, startOfLine)); + } + case '>': { + out = write(mv, out, file, currentLine.intValue(), startOfLine); + startOfLine = startOfLine & onlywhitespace; + mv.partial(new TemplateContext(sm, em, file, currentLine.get(), startOfLine), variable); + break; + } + case '{': { + out = write(mv, out, file, currentLine.intValue(), startOfLine); + // Not escaped + String name = variable; + if (em.charAt(1) != '}') { + name = variable.substring(0, variable.length() - 1); + } else { + if (br.read() != '}') { + TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); + throw new MustacheException( + "Improperly closed variable in " + file + ":" + currentLine, tc); + } + } + mv.value(new TemplateContext(sm, em, file, currentLine.get(), false), name, false); + break; + } + case '&': { + // Not escaped + out = write(mv, out, file, currentLine.intValue(), startOfLine); + mv.value(new TemplateContext(sm, em, file, currentLine.get(), false), variable, false); + break; + } + case '%': + // Pragmas + out = write(mv, out, file, currentLine.intValue(), startOfLine); + int index = variable.indexOf(" "); + String pragma; + String args; + if (index == -1) { + pragma = variable; + args = null; + } else { + pragma = variable.substring(0, index); + args = variable.substring(index + 1); + } + mv.pragma(new TemplateContext(sm, em, file, currentLine.get(), startOfLine), pragma, args); + break; + case '!': + // Comment + mv.comment(new TemplateContext(sm, em, file, currentLine.get(), startOfLine), variable); + out = write(mv, out, file, currentLine.intValue(), startOfLine); + break; + case '=': + // Change delimiters + out = write(mv, out, file, currentLine.intValue(), startOfLine); + String trimmed = command.substring(1).trim(); + String[] split = trimmed.split("\\s+"); + if (split.length != 2) { + TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); + throw new MustacheException("Invalid delimiter string: " + trimmed, tc); + } + sm = split[0]; + em = split[1]; + break; + default: { + if (c == -1) { + TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); + throw new MustacheException("Improperly closed variable", tc); + } + // Reference + out = write(mv, out, file, currentLine.intValue(), startOfLine); + mv.value(new TemplateContext(sm, em, file, currentLine.get(), false), command.trim(), true); + break; + } + } + // Additional text is no longer at the start of the line + startOfLine = false; + continue; + } else { + // Only one + br.reset(); + } + } + onlywhitespace = onlywhitespace && (c == ' ' || c == '\t' || c == '\r'); + out.append((char) c); + } + write(mv, out, file, currentLine.intValue(), startOfLine); + if (tag == null) { + br.close(); + } else { + TemplateContext tc = new TemplateContext(sm, em, file, startLine, startOfLine); + throw new MustacheException("Failed to close '" + tag + "' tag", tc); + } + } catch (IOException e) { + TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); + throw new MustacheException("Failed to read", tc); + } + mv.eof(new TemplateContext(sm, em, file, currentLine.get(), startOfLine)); + return mv.mustache(new TemplateContext(sm, em, file, 0, startOfLine)); + } catch (MustacheException me) { + try { + // We're going to blow the whole stack of compilation and + // close the readers on the way up. + br.close(); + } catch (IOException e) { + // Ignore IOExceptions on close + } + throw me; + } + + } + + /** + * Ignore empty strings and append to the previous code if it was also a write. + */ + private StringBuilder write(MustacheVisitor mv, StringBuilder out, String file, int line, boolean startOfLine) { + String text = out.toString(); + mv.write(new TemplateContext(null, null, file, line, startOfLine), text); + return new StringBuilder(); + } + +} diff --git a/__tests__/tree/code/snippet.seefwj9h.lua b/__tests__/tree/code/snippet.seefwj9h.lua new file mode 100644 index 0000000..d5444ef --- /dev/null +++ b/__tests__/tree/code/snippet.seefwj9h.lua @@ -0,0 +1,69 @@ +-- Just a concatenation of some of the snippets from https://en.wikipedia.org/wiki/Lua_(programming_language) + +function factorial(n) + local x = 1 + for i = 2, n do + x = x * i + end + return x +end + +function addto(x) + -- Return a new function that adds x to the argument + return function(y) + --[=[ When we refer to the variable x, which is outside of the current + scope and whose lifetime would be shorter than that of this anonymous + function, Lua creates a closure.]=] + return x + y + end +end +fourplus = addto(4) +print(fourplus(3)) -- Prints 7 + +--This can also be achieved by calling the function in the following way: +print(addto(4)(3)) +--[[ This is because we are calling the returned function from `addto(4)' with the argument `3' directly. + This also helps to reduce data cost and up performance if being called iteratively. +]] + +a_table = {x = 10} -- Creates a new table, with one entry mapping "x" to the number 10. +print(a_table["x"]) -- Prints the value associated with the string key, in this case 10. +b_table = a_table +b_table["x"] = 20 -- The value in the table has been changed to 20. +print(b_table["x"]) -- Prints 20. +print(a_table["x"]) -- Also prints 20, because a_table and b_table both refer to the same table. + +Point = {} + +Point.new = function(x, y) + return {x = x, y = y} -- return {["x"] = x, ["y"] = y} +end + +Point.set_x = function(point, x) + point.x = x -- point["x"] = x; +end + +fibs = { 1, 1 } -- Initial values for fibs[1] and fibs[2]. +setmetatable(fibs, { + __index = function(values, n) --[[ __index is a function predefined by Lua, + it is called if key "n" does not exist. ]] + values[n] = values[n - 1] + values[n - 2] -- Calculate and memoize fibs[n]. + return values[n] + end +}) + +local Vector = {} +Vector.__index = Vector + +function Vector:new(x, y, z) -- The constructor + return setmetatable({x = x, y = y, z = z}, Vector) +end + +function Vector:magnitude() -- Another method + -- Reference the implicit object using self + return math.sqrt(self.x^2 + self.y^2 + self.z^2) +end + +local vec = Vector:new(0, 1, 0) -- Create a vector +print(vec:magnitude()) -- Call a method (output: 1) +print(vec.x) -- Access a member variable (output: 0) diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index ee49e86..9959f85 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -151,12 +151,25 @@ describe('grammars-v4/', () => { }); }; - describe('grammars-v4/python3/', () => { - test_snippet('python3', 'handles basic hello worlds', 'snippet.voc84cjo.py'); - test_snippet('python3', 'handles a script adding the input arguments', 'snippet.kt29xnfw.py'); + describe('grammars-v4/c/', () => { + test_snippet('c', 'handles lots of stuff', 'snippet.ecemhuve.c'); }); + describe('grammars-v4/clojure/', () => { + test_snippet('clojure', 'handles lots of stuff', 'snippet.1v88olr0.clj'); + }) + describe('grammars-v4/java8/', () => { test_snippet('java8', 'handles basic hello worlds', 'snippet.k17eu4f2.java'); + test_snippet('java8', 'handles lots of stuff', 'snippet.lk0rlo4z.java'); + }); + + describe('grammars-v4/lua/', () => { + test_snippet('lua', 'handles lots of stuff', 'snippet.seefwj9h.lua'); + }); + + describe('grammars-v4/python3/', () => { + test_snippet('python3', 'handles basic hello worlds', 'snippet.voc84cjo.py'); + test_snippet('python3', 'handles a script adding the input arguments', 'snippet.kt29xnfw.py'); }); }); From a20719be151f2d4d96e29de46cf126ed0f666f82 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Fri, 19 May 2017 14:52:43 -0500 Subject: [PATCH 6/6] Simplified java test --- __tests__/tree/code/snippet.lk0rlo4z.java | 320 +++------------------- 1 file changed, 33 insertions(+), 287 deletions(-) diff --git a/__tests__/tree/code/snippet.lk0rlo4z.java b/__tests__/tree/code/snippet.lk0rlo4z.java index fb22310..930dbd6 100644 --- a/__tests__/tree/code/snippet.lk0rlo4z.java +++ b/__tests__/tree/code/snippet.lk0rlo4z.java @@ -1,298 +1,44 @@ -// Shamelessly stolen from https://github.com/spullara/mustache.java/blob/master/compiler/src/main/java/com/github/mustachejava/MustacheParser.java +// Just a concatenation of some of the snippets from https://en.wikipedia.org/wiki/Java_(programming_language) -package com.github.mustachejava; +// This is an example of a single line comment using two slashes -import java.io.BufferedReader; -import java.io.IOException; -import java.io.Reader; -import java.util.concurrent.atomic.AtomicInteger; +/* This is an example of a multiple line comment using the slash and asterisk. + This type of comment can be used to hold a lot of information or deactivate + code, but it is very important to remember to close the comment. */ + +package fibsandlies; +import java.util.HashMap; /** - * The parser generates callbacks into the MustacheFactory to build them. Do not use these - * directly as you must manage the Mustache object lifecycle as well. - *

- * User: sam - * Date: 5/14/11 - * Time: 3:52 PM + * This is an example of a Javadoc comment; Javadoc can compile documentation + * from this text. Javadoc comments must immediately precede the class, method, or field being documented. */ -public class MustacheParser { - public static final String DEFAULT_SM = "{{"; - public static final String DEFAULT_EM = "}}"; - private MustacheFactory mf; - - protected MustacheParser(MustacheFactory mf) { - this.mf = mf; - } - - public Mustache compile(String file) { - Reader reader = mf.getReader(file); - if (reader == null) { - throw new MustacheNotFoundException(file); - } - return compile(reader, file); - } - - public Mustache compile(Reader reader, String file) { - return compile(reader, file, DEFAULT_SM, DEFAULT_EM); - } - - public Mustache compile(Reader reader, String file, String sm, String em) { - return compile(reader, null, new AtomicInteger(0), file, sm, em, true); - } - - public Mustache compile(Reader reader, String file, String sm, String em, boolean startOfLine) { - return compile(reader, null, new AtomicInteger(0), file, sm, em, startOfLine); - } - - @SuppressWarnings("ConstantConditions") // this method is too complex - protected Mustache compile(final Reader reader, String tag, final AtomicInteger currentLine, String file, String sm, String em, boolean startOfLine) throws MustacheException { - if (reader == null) { - throw new MustacheException("Reader is null"); - } - Reader br; - if (reader.markSupported()) { - br = reader; - } else { - br = new BufferedReader(reader); +public class FibCalculator extends Fibonacci implements Calculator { + private static Map memoized = new HashMap(); + + /* + * The main method written as follows is used by the JVM as a starting point for the program. + */ + public static void main(String[] args) { + memoized.put(1, 1); + memoized.put(2, 1); + System.out.println(fibonacci(12)); //Get the 12th Fibonacci number and print to console } - try { - boolean sawCR = false; - int startLine = currentLine.get(); - MustacheVisitor mv = mf.createMustacheVisitor(); - // Now we grab the mustache template - boolean onlywhitespace = true; - // Starting a new line - boolean iterable = currentLine.get() != 0; - currentLine.compareAndSet(0, 1); - StringBuilder out = new StringBuilder(); - try { - int c; - while ((c = br.read()) != -1) { - // We remember that we saw a carriage return so we can put it back in later - if (c == '\r') { - sawCR = true; - continue; - } - // Increment the line - if (c == '\n') { - currentLine.incrementAndGet(); - if (!iterable || (iterable && !onlywhitespace)) { - if (sawCR) out.append("\r"); - out.append("\n"); - } - out = write(mv, out, file, currentLine.intValue(), startOfLine); - iterable = false; - onlywhitespace = true; - startOfLine = true; - continue; - } - sawCR = false; - // Check for a mustache start - if (c == sm.charAt(0)) { - br.mark(1); - if (sm.length() == 1 || br.read() == sm.charAt(1)) { - // If it is a delimiter change we need to specially handle it - // Two mustaches, now capture command - StringBuilder sb = new StringBuilder(); - br.mark(1); - c = br.read(); - boolean delimiter = c == '='; - if (delimiter) { - sb.append((char) c); - } else { - br.reset(); - } - while ((c = br.read()) != -1) { - br.mark(1); - if (delimiter) { - if (c == '=') { - // Reached the end of the definition - delimiter = false; - } else { - sb.append((char) c); - } - continue; - } - if (c == em.charAt(0)) { - if (em.length() > 1) { - if (br.read() == em.charAt(1)) { - // Matched end - break; - } else { - // Only one - br.reset(); - } - } else break; - } - sb.append((char) c); - } - final String command = mf.translate(sb.toString()); - if (command.length() == 0) { - TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); - throw new MustacheException("Empty mustache", tc); - } - final char ch = command.charAt(0); - final String variable = command.substring(1).trim(); - switch (ch) { - case '#': - case '^': - case '<': - case '$': { - boolean oldStartOfLine = startOfLine; - startOfLine = startOfLine & onlywhitespace; - int line = currentLine.get(); - final Mustache mustache = compile(br, variable, currentLine, file, sm, em, startOfLine); - int lines = currentLine.get() - line; - if (!onlywhitespace || lines == 0) { - write(mv, out, file, currentLine.intValue(), oldStartOfLine); - } - out = new StringBuilder(); - switch (ch) { - case '#': - mv.iterable(new TemplateContext(sm, em, file, line, startOfLine), variable, mustache); - break; - case '^': - mv.notIterable(new TemplateContext(sm, em, file, line, startOfLine), variable, mustache); - break; - case '<': - mv.extend(new TemplateContext(sm, em, file, line, startOfLine), variable, mustache); - break; - case '$': - mv.name(new TemplateContext(sm, em, file, line, startOfLine), variable, mustache); - break; - } - iterable = lines != 0; - break; - } - case '/': { - // Tag end - if (!startOfLine || !onlywhitespace) { - write(mv, out, file, currentLine.intValue(), startOfLine); - } - if (!variable.equals(tag)) { - TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); - throw new MustacheException( - "Mismatched start/end tags: " + tag + " != " + variable + " in " + file + ":" + currentLine, tc); - } - - return mv.mustache(new TemplateContext(sm, em, file, 0, startOfLine)); - } - case '>': { - out = write(mv, out, file, currentLine.intValue(), startOfLine); - startOfLine = startOfLine & onlywhitespace; - mv.partial(new TemplateContext(sm, em, file, currentLine.get(), startOfLine), variable); - break; - } - case '{': { - out = write(mv, out, file, currentLine.intValue(), startOfLine); - // Not escaped - String name = variable; - if (em.charAt(1) != '}') { - name = variable.substring(0, variable.length() - 1); - } else { - if (br.read() != '}') { - TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); - throw new MustacheException( - "Improperly closed variable in " + file + ":" + currentLine, tc); - } - } - mv.value(new TemplateContext(sm, em, file, currentLine.get(), false), name, false); - break; - } - case '&': { - // Not escaped - out = write(mv, out, file, currentLine.intValue(), startOfLine); - mv.value(new TemplateContext(sm, em, file, currentLine.get(), false), variable, false); - break; - } - case '%': - // Pragmas - out = write(mv, out, file, currentLine.intValue(), startOfLine); - int index = variable.indexOf(" "); - String pragma; - String args; - if (index == -1) { - pragma = variable; - args = null; - } else { - pragma = variable.substring(0, index); - args = variable.substring(index + 1); - } - mv.pragma(new TemplateContext(sm, em, file, currentLine.get(), startOfLine), pragma, args); - break; - case '!': - // Comment - mv.comment(new TemplateContext(sm, em, file, currentLine.get(), startOfLine), variable); - out = write(mv, out, file, currentLine.intValue(), startOfLine); - break; - case '=': - // Change delimiters - out = write(mv, out, file, currentLine.intValue(), startOfLine); - String trimmed = command.substring(1).trim(); - String[] split = trimmed.split("\\s+"); - if (split.length != 2) { - TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); - throw new MustacheException("Invalid delimiter string: " + trimmed, tc); - } - sm = split[0]; - em = split[1]; - break; - default: { - if (c == -1) { - TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); - throw new MustacheException("Improperly closed variable", tc); - } - // Reference - out = write(mv, out, file, currentLine.intValue(), startOfLine); - mv.value(new TemplateContext(sm, em, file, currentLine.get(), false), command.trim(), true); - break; - } - } - // Additional text is no longer at the start of the line - startOfLine = false; - continue; - } else { - // Only one - br.reset(); - } - } - onlywhitespace = onlywhitespace && (c == ' ' || c == '\t' || c == '\r'); - out.append((char) c); - } - write(mv, out, file, currentLine.intValue(), startOfLine); - if (tag == null) { - br.close(); + /** + * An example of a method written in Java, wrapped in a class. + * Given a non-negative number FIBINDEX, returns + * the Nth Fibonacci number, where N equals FIBINDEX. + * @param fibIndex The index of the Fibonacci number + * @return The Fibonacci number + */ + public static int fibonacci(int fibIndex) { + if (memoized.containsKey(fibIndex)) { + return memoized.get(fibIndex); } else { - TemplateContext tc = new TemplateContext(sm, em, file, startLine, startOfLine); - throw new MustacheException("Failed to close '" + tag + "' tag", tc); + int answer = fibonacci(fibIndex - 1) + fibonacci(fibIndex - 2); + memoized.put(fibIndex, answer); + return answer; } - } catch (IOException e) { - TemplateContext tc = new TemplateContext(sm, em, file, currentLine.get(), startOfLine); - throw new MustacheException("Failed to read", tc); - } - mv.eof(new TemplateContext(sm, em, file, currentLine.get(), startOfLine)); - return mv.mustache(new TemplateContext(sm, em, file, 0, startOfLine)); - } catch (MustacheException me) { - try { - // We're going to blow the whole stack of compilation and - // close the readers on the way up. - br.close(); - } catch (IOException e) { - // Ignore IOExceptions on close - } - throw me; } - - } - - /** - * Ignore empty strings and append to the previous code if it was also a write. - */ - private StringBuilder write(MustacheVisitor mv, StringBuilder out, String file, int line, boolean startOfLine) { - String text = out.toString(); - mv.write(new TemplateContext(null, null, file, line, startOfLine), text); - return new StringBuilder(); - } - }