diff --git a/.vscode/launch.json b/.vscode/launch.json index b537d3e..8de6d94 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,6 +17,19 @@ "environment": [], "console": "internalConsole", }, + { + "name": "Debug Parser Perf (Windows)", + "type": "cppvsdbg", + "request": "launch", + "program": "${workspaceFolder}/build/debug/parser-perf.exe", + "args": [ + "${workspaceFolder}/eohippus-vscode/test_folder/main.pony" + ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "console": "internalConsole", + }, { "name": "Debug CLI (Windows)", "type": "cppvsdbg", diff --git a/corral.json b/corral.json index d26ffa9..0939864 100644 --- a/corral.json +++ b/corral.json @@ -4,12 +4,12 @@ ], "deps": [ { - "locator": "github.com/ponylang/appdirs.git", - "version": "0.1.5" + "locator": "github.com/chalcolith/kiuatan.git", + "version": "1.5.5" }, { - "locator": "github.com/chalcolith/kiuatan.git", - "version": "1.5.2" + "locator": "github.com/ponylang/appdirs.git", + "version": "0.1.5" }, { "locator": "github.com/ponylang/logger.git", diff --git a/eohippus-vscode/test_folder/builtin_test/ini.pony b/eohippus-vscode/test_folder/builtin_test/ini.pony deleted file mode 100644 index 38144f0..0000000 --- a/eohippus-vscode/test_folder/builtin_test/ini.pony +++ /dev/null @@ -1,147 +0,0 @@ -""" -# Ini package - -The Ini package provides support for parsing -[INI file](https://en.wikipedia.org/wiki/INI_file) formatted text. - -* Currently _does not_ support multi-line entries. -* Any keys not in a section will be placed in the section "" - -# Example code -```pony -// Parses the file 'example.ini' in the current working directory -// Output all the content -use "ini" -use "files" - -actor Main - new create(env:Env) => - try - let ini_file = File(FilePath(FileAuth(env.root), "example.ini")) - let sections = IniParse(ini_file.lines())? - for section in sections.keys() do - env.out.print("Section name is: " + section) - for key in sections(section)?.keys() do - env.out.print(key + " = " + sections(section)?(key)?) - end - end - end -``` -""" -primitive IniIncompleteSection -primitive IniNoDelimiter - -type IniError is - ( IniIncompleteSection - | IniNoDelimiter - ) - -interface IniNotify - """ - Notifications for INI parsing. - """ - fun ref apply(section: String, key: String, value: String): Bool - """ - This is called for every valid entry in the INI file. If key/value pairs - occur before a section name, the section can be an empty string. Return - false to halt processing. - """ - - fun ref add_section(section: String): Bool => - """ - This is called for every valid section in the INI file. Return false - to halt processing. - """ - true - - fun ref errors(line: USize, err: IniError): Bool => - """ - This is called for each error encountered. Return false to halt processing. - """ - true - -primitive Ini - """ - A streaming parser for INI formatted lines of test. - """ - fun apply(lines: Iterator[String box], f: IniNotify): Bool => - """ - This accepts a string iterator and calls the IniNotify for each new entry. - If any errors are encountered, this will return false. Otherwise, it - returns true. - """ - var section = "" - var lineno = USize(0) - var ok = true - - for line in lines do - lineno = lineno + 1 - var current = line.clone() - current.strip() - - if current.size() == 0 then - continue - end - - try - match current(0)? - | ';' | '#' => - // Skip comments. - continue - | '[' => - try - current.delete(current.find("]", 1)?, -1) - current.delete(0) - section = consume current - if not f.add_section(section) then - return ok - end - else - ok = false - - if not f.errors(lineno, IniIncompleteSection) then - return false - end - end - else - try - let delim = try - current.find("=")? - else - current.find(":")? - end - - let value = current.substring(delim + 1) - value.strip() - - current.delete(delim, -1) - current.strip() - - try - let comment = try - value.find(";")? - else - value.find("#")? - end - - match value(comment.usize() - 1)? - | ' ' | '\t' => - value.delete(comment, -1) - value.rstrip() - end - end - - if not f(section, consume current, consume value) then - return ok - end - else - ok = false - - if not f.errors(lineno, IniNoDelimiter) then - return false - end - end - end - end - end - ok diff --git a/eohippus/ast/exp_match.pony b/eohippus/ast/exp_match.pony index 384ebba..8f66a97 100644 --- a/eohippus/ast/exp_match.pony +++ b/eohippus/ast/exp_match.pony @@ -73,28 +73,23 @@ primitive ParseExpMatch end ExpMatch(expression, cases, else_block) -class val MatchCase is NodeData - """A case in a `match` expression.""" +class val MatchPattern is NodeData let pattern: NodeWith[Expression] let condition: (NodeWith[Expression] | None) - let body: NodeWith[Expression] new val create( pattern': NodeWith[Expression], - condition': (NodeWith[Expression] | None), - body': NodeWith[Expression]) + condition': (NodeWith[Expression] | None)) => pattern = pattern' condition = condition' - body = body' - fun name(): String => "MatchCase" + fun name(): String => "MatchPattern" fun val clone(updates: ChildUpdateMap): NodeData => - MatchCase( + MatchPattern( _map_with[Expression](pattern, updates), - _map_or_none[Expression](condition, updates), - _map_with[Expression](body, updates)) + _map_or_none[Expression](condition, updates)) fun add_json_props(node: Node box, props: Array[(String, json.Item)]) => props.push(("pattern", node.child_ref(pattern))) @@ -102,35 +97,74 @@ class val MatchCase is NodeData | let condition': NodeWith[Expression] => props.push(("condition", node.child_ref(condition'))) end - props.push(("body", node.child_ref(body))) -primitive ParseMatchCase - fun apply(obj: json.Object, children: NodeSeq): (MatchCase | String) => +primitive ParseMatchPattern + fun apply(obj: json.Object, children: NodeSeq): (MatchPattern | String) => let pattern = match ParseNode._get_child_with[Expression]( obj, children, "pattern", - "MatchCase.pattern must be an Expression") + "MatchPattern.pattern must be an Expression") | let node: NodeWith[Expression] => node | let err: String => return err else - return "MatchCase.pattern must be an Expression" + return "MatchPattern.pattern must be an Expression" end let condition = match ParseNode._get_child_with[Expression]( obj, children, "condition", - "MatchCase.condition must be an Expression", - false) + "MatchPattern.condition must be an Expression") | let node: NodeWith[Expression] => node | let err: String => return err end + MatchPattern(pattern, condition) + +class val MatchCase is NodeData + """A case in a `match` expression.""" + let patterns: NodeSeqWith[MatchPattern] + let body: NodeWith[Expression] + + new val create( + patterns': NodeSeqWith[MatchPattern], + body': NodeWith[Expression]) + => + patterns = patterns' + body = body' + + fun name(): String => "MatchCase" + + fun val clone(updates: ChildUpdateMap): NodeData => + MatchCase( + _map[MatchPattern](patterns, updates), + _map_with[Expression](body, updates)) + + fun add_json_props(node: Node box, props: Array[(String, json.Item)]) => + if patterns.size() > 0 then + props.push(("patterns", node.child_refs(patterns))) + end + props.push(("body", node.child_ref(body))) + +primitive ParseMatchCase + fun apply(obj: json.Object, children: NodeSeq): (MatchCase | String) => + let patterns = + match ParseNode._get_seq_with[MatchPattern]( + obj, + children, + "patterns", + "MatchCase.patterns must refer to MatchCasePatterns", + false) + | let seq: NodeSeqWith[MatchPattern] => + seq + | let err: String => + return err + end let body = match ParseNode._get_child_with[Expression]( obj, @@ -144,4 +178,4 @@ primitive ParseMatchCase else return "MatchCase.body must be an Expression" end - MatchCase(pattern, condition, body) + MatchCase(patterns, body) diff --git a/eohippus/ast/parse_node.pony b/eohippus/ast/parse_node.pony index b6b67c4..3cf7886 100644 --- a/eohippus/ast/parse_node.pony +++ b/eohippus/ast/parse_node.pony @@ -164,6 +164,8 @@ primitive ParseNode ParseNode~_ctor[Expression](ParseExpLambda~apply(obj, children)) | "ExpMatch" => ParseNode~_ctor[Expression](ParseExpMatch~apply(obj, children)) + | "MatchPattern" => + ParseNode~_ctor[MatchPattern](ParseMatchPattern~apply(obj, children)) | "MatchCase" => ParseNode~_ctor[MatchCase](ParseMatchCase~apply(obj, children)) | "ExpObject" => diff --git a/eohippus/parser/_exp_actions.pony b/eohippus/parser/_exp_actions.pony index de821e1..c65e5e5 100644 --- a/eohippus/parser/_exp_actions.pony +++ b/eohippus/parser/_exp_actions.pony @@ -343,10 +343,9 @@ primitive _ExpActions _Build.info(d, r), c, ast.ExpMatch(exp', cases', else_block')) (value, b) - fun tag _match_case( + fun tag _match_pattern( pattern: Variable, condition: Variable, - body: Variable, d: Data, r: Success, c: ast.NodeSeq, @@ -357,9 +356,24 @@ primitive _ExpActions try _Build.value_with[ast.Expression](b, pattern, r)? else - return _Build.bind_error(d, r, c, b, "Expression/MatchCase/Pattern") + return _Build.bind_error(d, r, c, b, "Expression/MatchPattern/Pattern") end let condition' = _Build.value_with_or_none[ast.Expression](b, condition, r) + + let value = ast.NodeWith[ast.MatchPattern]( + _Build.info(d, r), c, ast.MatchPattern(pattern', condition')) + (value, b) + + fun tag _match_case( + patterns: Variable, + body: Variable, + d: Data, + r: Success, + c: ast.NodeSeq, + b: Bindings) + : ((ast.Node | None), Bindings) + => + let patterns' = _Build.values_with[ast.MatchPattern](b, patterns, r) let body' = try _Build.value_with[ast.Expression](b, body, r)? @@ -368,7 +382,7 @@ primitive _ExpActions end let value = ast.NodeWith[ast.MatchCase]( - _Build.info(d, r), c, ast.MatchCase(pattern', condition', body')) + _Build.info(d, r), c, ast.MatchCase(patterns', body')) (value, b) fun tag _while( diff --git a/eohippus/parser/expression_builder.pony b/eohippus/parser/expression_builder.pony index 5c50ba8..adde465 100644 --- a/eohippus/parser/expression_builder.pony +++ b/eohippus/parser/expression_builder.pony @@ -16,7 +16,7 @@ class ExpressionBuilder let annotation: NamedRule = NamedRule("an annotation" where memoize' = true) let item: NamedRule = NamedRule("an expression" where memoize' = true) let infix: NamedRule = NamedRule("an infix expression" where memoize' = true) - let seq: NamedRule = NamedRule("an expression sequence") + let seq: NamedRule = NamedRule("an expression sequence" where memoize' = true) let tuple_pattern: NamedRule = NamedRule("a tuple destructuring pattern") let _method_params: NamedRule let _typedef_members: NamedRule @@ -91,6 +91,7 @@ class ExpressionBuilder let exp_while: NamedRule = NamedRule("a while loop") let exp_with: NamedRule = NamedRule("a with expression") let match_case: NamedRule = NamedRule("a match case") + let match_pattern: NamedRule = NamedRule("a match pattern") let with_elem: NamedRule = NamedRule("a with element") let amp = _token(ast.Tokens.amp()) @@ -372,19 +373,30 @@ class ExpressionBuilder kwd_end ]), _ExpActions~_match(match_exp, match_cases, match_else_block)) - // match_case <= '|' item ('if' seq)? '=>' seq - let match_case_pattern = Variable("match_case_pattern") - let match_case_condition = Variable("match_case_condition") + // match_pattern <= '|' item ('if' seq)? + let match_pattern_pattern = Variable("match_pattern_pattern") + let match_pattern_condition = Variable("match_pattern_condition") + match_pattern.set_body( + Conj( + [ bar + Bind( + match_pattern_pattern, + Disj([ exp_decl; exp_prefix; exp_hash ])) + Ques(Conj([ kwd_if; Bind(match_pattern_condition, seq) ])) + ]), + _ExpActions~_match_pattern( + match_pattern_pattern, match_pattern_condition)) + + // match_case <= match_case_pattern+ '=>' seq + let match_case_patterns = Variable("match_case_patterns") let match_case_body = Variable("match_case_body") match_case.set_body( Conj( - [ bar - Bind(match_case_pattern, Disj([ exp_decl; exp_prefix; exp_hash ])) - Ques(Conj([ kwd_if; Bind(match_case_condition, seq) ])) + [ Bind(match_case_patterns, Star(match_pattern, 1)) equal_arrow - Bind(match_case_body, seq) ]), - _ExpActions~_match_case( - match_case_pattern, match_case_condition, match_case_body)) + Bind(match_case_body, seq) + ]), + _ExpActions~_match_case(match_case_patterns, match_case_body)) // while <= 'while' seq 'do' seq ('else' seq)? 'end' let while_cond = Variable("while_cond") diff --git a/eohippus/parser/operator_builder.pony b/eohippus/parser/operator_builder.pony index 56670d0..3df536d 100644 --- a/eohippus/parser/operator_builder.pony +++ b/eohippus/parser/operator_builder.pony @@ -5,9 +5,12 @@ class OperatorBuilder let _token: TokenBuilder let _keyword: KeywordBuilder - let prefix_op: NamedRule = NamedRule("a prefix operator") - let binary_op: NamedRule = NamedRule("a binary operator") - let postfix_op: NamedRule = NamedRule("a postfix operator") + let prefix_op: NamedRule = + NamedRule("a prefix operator" where memoize' = true) + let binary_op: NamedRule = + NamedRule("a binary operator" where memoize' = true) + let postfix_op: NamedRule = + NamedRule("a postfix operator" where memoize' = true) new create( trivia: TriviaBuilder, diff --git a/eohippus/parser/token_builder.pony b/eohippus/parser/token_builder.pony index 392e061..89c3540 100644 --- a/eohippus/parser/token_builder.pony +++ b/eohippus/parser/token_builder.pony @@ -40,7 +40,7 @@ class TokenBuilder let _trivia: TriviaBuilder let _tokens: Map[String, NamedRule] - let identifier: NamedRule = NamedRule("an identifier") + let identifier: NamedRule = NamedRule("an identifier" where memoize' = true) new create(context: Context, trivia: TriviaBuilder) => _context = context diff --git a/eohippus/parser/typedef_builder.pony b/eohippus/parser/typedef_builder.pony index e62b5ee..d5c3333 100644 --- a/eohippus/parser/typedef_builder.pony +++ b/eohippus/parser/typedef_builder.pony @@ -9,10 +9,10 @@ class TypedefBuilder let _type_type: TypeBuilder let _expression: ExpressionBuilder - let doc_string: NamedRule = NamedRule("a doc string") + let doc_string: NamedRule = NamedRule("a doc string" where memoize' = true) let method_params: NamedRule let members: NamedRule - let field: NamedRule = NamedRule("a field") + let field: NamedRule = NamedRule("a field" where memoize' = true) let method: NamedRule = NamedRule("a method" where memoize' = true) let typedef: NamedRule = NamedRule("a type definition" where memoize' = true) let typedef_primitive: NamedRule = NamedRule("a primitive type definition") diff --git a/eohippus/test/_test_linter.pony b/eohippus/test/_test_linter.pony index 315b681..557eb9c 100644 --- a/eohippus/test/_test_linter.pony +++ b/eohippus/test/_test_linter.pony @@ -76,7 +76,7 @@ class iso _TestLinterAnalyzeTrimTrailingWhitespace is UnitTest h.complete(false) end }) - h.long_test(2_000_000_000) + h.long_test(10_000_000_000) class iso _TestLinterFixTrimTrailingWhitespace is UnitTest fun name(): String => "linter/fix/trim_trailing_whitespace" @@ -470,4 +470,4 @@ class iso _TestLinterFixTrimTrailingWhitespace is UnitTest h.complete(false) end }) - h.long_test(2_000_000_000) + h.long_test(10_000_000_000) diff --git a/eohippus/test/_test_parser_expression.pony b/eohippus/test/_test_parser_expression.pony index 6be46af..32d89bd 100644 --- a/eohippus/test/_test_parser_expression.pony +++ b/eohippus/test/_test_parser_expression.pony @@ -31,6 +31,7 @@ primitive _TestParserExpression test(_TestParserExpressionTuplePattern) test(_TestParserExpressionFor) test(_TestParserExpressionMatch) + test(_TestParserExpressionMatchFallThrough) test(_TestParserExpressionMatchNegative) test(_TestParserExpressionDecl) test(_TestParserExpressionWith) @@ -1698,10 +1699,7 @@ class iso _TestParserExpressionMatch is UnitTest { "name": "ExpMatch", "expression": 1, - "cases": [ - 2, - 3 - ], + "cases": [ 2, 3 ], "else_block": 5, "children": [ { @@ -1720,96 +1718,60 @@ class iso _TestParserExpressionMatch is UnitTest }, { "name": "MatchCase", - "pattern": 1, - "condition": 3, - "body": 5, + "patterns": [ 0 ], + "body": 2, "children": [ { - "name": "Token", - "string": "|" - }, - { - "name": "ExpAtom", - "body": 0, + "name": "MatchPattern", + "pattern": 1, + "condition": 3, "children": [ + { "name": "Token", "string": "|" }, { - "name": "Identifier", - "string": "b" - } - ] - }, - { - "name": "Keyword", - "string": "if" - }, - { - "name": "ExpAtom", - "body": 0, - "children": [ + "name": "ExpAtom", + "body": 0, + "children": [ { "name": "Identifier", "string": "b" } ] + }, + { "name": "Keyword", "string": "if" }, { - "name": "Identifier", - "string": "c" + "name": "ExpAtom", + "body": 0, + "children": [ { "name": "Identifier", "string": "c" } ] } ] }, { - "name": "Token", - "string": "=>" + "name": "Token", "string": "=>" }, { "name": "ExpAtom", "body": 0, - "children": [ - { - "name": "Identifier", - "string": "d" - } - ] + "children": [ { "name": "Identifier", "string": "d" } ] } ] }, { "name": "MatchCase", - "pattern": 1, - "body": 3, + "patterns": [ 0 ], + "body": 2, "children": [ { - "name": "Token", - "string": "|" - }, - { - "name": "ExpAtom", - "body": 0, + "name": "MatchPattern", + "pattern": 1, "children": [ + { "name": "Token", "string": "|" }, { - "name": "LiteralInteger", - "kind": "DecimalInteger", - "value": 2 + "name": "ExpAtom", + "body": 0, + "children": [ { "name": "LiteralInteger", "value": 2 } ] } ] }, - { - "name": "Token", - "string": "=>" - }, + { "name": "Token", "string": "=>" }, { "name": "ExpAtom", "body": 0, - "children": [ - { - "name": "LiteralBool", - "value": true, - "children": [ - { - "name": "Span" - }, - { - "name": "Keyword", - "string": "true" - } - ] - } - ] + "children": [ { "name": "LiteralBool", "value": true } ] } ] }, @@ -1837,6 +1799,34 @@ class iso _TestParserExpressionMatch is UnitTest _Assert.test_all(h, [ _Assert.test_match(h, rule, setup.data, src, exp) ]) +class iso _TestParserExpressionMatchFallThrough is UnitTest + fun name(): String => "parser/expression/Match/fallthrough" + fun exclusion_group(): String => "parser/expression" + + fun apply(h: TestHelper) => + let setup = _TestSetup(name()) + let rule = setup.builder.expression.item + + let src = + """ + match foo + | ';' | '#' => + continue + end + """ + let src_len = src.size() + + _Assert.test_all( + h, + [ _Assert.test_with( + h, rule, setup.data, src, + {(success, values) => + let len = success.next.index() - success.start.index() + ( len == src_len + , "expected length " + src_len.string() + ", got " + len.string() ) + }) + ]) + class iso _TestParserExpressionMatchNegative is UnitTest fun name(): String => "parser/expression/Match/negative" fun exclusion_group(): String => "parser/expression" diff --git a/eohippus/test/parser-perf/main.pony b/eohippus/test/parser-perf/main.pony index c302a9b..4bdf2c4 100644 --- a/eohippus/test/parser-perf/main.pony +++ b/eohippus/test/parser-perf/main.pony @@ -42,9 +42,9 @@ actor Main {(result: parser.Result, values: ast.NodeSeq) => match result | let success: parser.Success => - env.err.print(arg + " succeeded") - else - env.err.print(arg + " failed") + _env.out.print(arg + " succeeded") + | let failure: parser.Failure => + _env.out.print(arg + " failed: " + failure.get_message()) end self.print_time() }) diff --git a/make.ps1 b/make.ps1 index 46e3702..150d1e8 100644 --- a/make.ps1 +++ b/make.ps1 @@ -5,7 +5,7 @@ param( [Parameter(HelpMessage="The target(s) to build or test (test, fmt, lsp, cli)")] [string] - $Target = 'test,parser-perf,fmt,lsp,cli', + $Target = 'test,perf,fmt,lsp,cli', [Parameter(HelpMessage="The build configuration (Release, Debug).")] [string] @@ -94,7 +94,7 @@ function Build if ($targets -like 'test') { Run("corral.exe run -- ponyc $configFlag $ponyArgs --cpu `"$Arch`" --output `"$buildDir`" `"$rootDir\eohippus\test`"") } - if ($targets -like 'parser-perf') { + if ($targets -like 'perf') { Run("corral.exe run -- ponyc $configFlag $ponyArgs --cpu `"$Arch`" --output `"$buildDir`" `"$rootDir\eohippus\test\parser-perf`"") } if ($targets -like 'lsp') {