From 6cf79d9a8714a0c93d3e4cc6fe96cd71f76e0922 Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Sat, 22 Jun 2024 21:45:57 -0400 Subject: [PATCH 1/7] nitcc: lexer prepare chars only once Signed-off-by: Jean Privat --- lib/nitcc_runtime/nitcc_runtime.nit | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/nitcc_runtime/nitcc_runtime.nit b/lib/nitcc_runtime/nitcc_runtime.nit index cfddec3f65..95d3fbf5d3 100644 --- a/lib/nitcc_runtime/nitcc_runtime.nit +++ b/lib/nitcc_runtime/nitcc_runtime.nit @@ -167,6 +167,7 @@ abstract class Lexer fun lex: CircularArray[NToken] do var res = new CircularArray[NToken] + chars = stream.chars loop var t = next_token if t != null then res.add t @@ -184,6 +185,9 @@ abstract class Lexer # Cursor current column (in chars, starting from 1) var col_start = 1 + # Current + var chars: SequenceRead[Char] is noinit + # Move the cursor and return the next token. # # Returns a `NEof` and the end. @@ -199,6 +203,7 @@ abstract class Lexer var col_end = col_start - 1 var last_state: nullable DFAState = null var text = stream + var chars = self.chars var length = text.length loop if state.is_accept then @@ -213,7 +218,7 @@ abstract class Lexer c = '\0' next = null else - c = text.chars[pos] + c = chars[pos] next = state.trans(c) end if next == null then From 033b0bc63596bf674209fab7b46323e054e53f07 Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Sun, 23 Jun 2024 13:02:58 -0400 Subject: [PATCH 2/7] nitcc: make code for newnodes and add Signed-off-by: Jean Privat --- contrib/nitcc/src/grammar.nit | 95 ++++++++++++++++------------ contrib/nitcc/src/nitcc_semantic.nit | 6 +- 2 files changed, 60 insertions(+), 41 deletions(-) diff --git a/contrib/nitcc/src/grammar.nit b/contrib/nitcc/src/grammar.nit index feb4498790..3484d0a505 100644 --- a/contrib/nitcc/src/grammar.nit +++ b/contrib/nitcc/src/grammar.nit @@ -468,6 +468,23 @@ class CodeNew redef fun to_s do return "New {alt.name}/{alt.elems.length}" end +# Allocate a new empty AST node array for an alternative +class CodeNewNodes + super Code + + # The associated alternative + var alt: Alternative + + redef fun to_s do return "NewArray {alt.name}" +end + +# Add an element to a Nodes array +class CodeAdd + super Code + + redef fun to_s do return "Add" +end + # Get null class CodeNull super Code @@ -835,49 +852,49 @@ private class Generator i -= 1 end - if alt.name.has_suffix("+_more") then - add "\t\tvar prod = n0" - add "\t\tn0.children.add(n1)" - else if alt.name.has_suffix("+_one") then - add "\t\tvar prod = new {alt.prod.acname}" - add "\t\tprod.children.add(n0)" - else - alt.make_codes - var cpt = 0 - i = 0 - var st = new Array[String] - for c in alt.codes.as(not null) do - if c isa CodePop then - st.add "n{i}" - i += 1 - else if c isa CodeNull then - st.add "null" - else if c isa CodeNew then - var calt = c.alt - cpt += 1 - var from = st.length - calt.elems.length - var args = new List[String] - for j in [from..st.length[ do - args.unshift(st.pop) - end + alt.make_codes + var cpt = 0 + i = 0 + var st = new Array[String] + for c in alt.codes.as(not null) do + if c isa CodePop then + st.add "n{i}" + i += 1 + else if c isa CodeNull then + st.add "null" + else if c isa CodeNew then + var calt = c.alt + cpt += 1 + var from = st.length - calt.elems.length + var args = new List[String] + for j in [from..st.length[ do + args.unshift(st.pop) + end - if args.is_empty then - add "\t\tvar p{cpt} = new {calt.cname}" - else - add "\t\tvar p{cpt} = new {calt.cname}({args.join(", ")})" - end - #var x = 0 - #for j in [from..st.length[ do - #if st[j] == "null" then continue - #add "\t\tp{cpt}.n_{calt.elemname(x)} = {st[j]}" - #x += 1 - #end - st.add("p{cpt}") + if args.is_empty then + add "\t\tvar p{cpt} = new {calt.cname}" + else + add "\t\tvar p{cpt} = new {calt.cname}({args.join(", ")})" end + #var x = 0 + #for j in [from..st.length[ do + #if st[j] == "null" then continue + #add "\t\tp{cpt}.n_{calt.elemname(x)} = {st[j]}" + #x += 1 + #end + st.add("p{cpt}") + else if c isa CodeNewNodes then + cpt += 1 + add "\t\tvar p{cpt} = new {c.alt.prod.acname}" + st.add("p{cpt}") + else if c isa CodeAdd then + var a1 = st.pop + var a0 = st.last + add "\t\t{a0}.children.add({a1})" end - assert st.length == 1 - add "\t\tvar prod = {st.first}" end + assert st.length == 1 + add "\t\tvar prod = {st.first}" add "\t\tparser.node_stack.push prod" if alt.prod.accept then diff --git a/contrib/nitcc/src/nitcc_semantic.nit b/contrib/nitcc/src/nitcc_semantic.nit index 1fafab893a..17b7850e3a 100644 --- a/contrib/nitcc/src/nitcc_semantic.nit +++ b/contrib/nitcc/src/nitcc_semantic.nit @@ -130,8 +130,10 @@ private class CheckNameVisitor var prod = new Production(name) prod.acname = "Nodes[{e.acname}]" v1.gram.prods.add(prod) - prod.new_alt("{name}_one", e) - prod.new_alt("{name}_more", prod, e) + var alt1 = prod.new_alt("{name}_one", e) + alt1.codes = [new CodeNewNodes(alt1), new CodePop, new CodeAdd: Code] + var alt2 = prod.new_alt("{name}_more", prod, e) + alt2.codes = [new CodePop, new CodePop, new CodeAdd: Code] plusizes[e] = prod return prod end From 3aae1d08da299e9064fa6cc6bc4e6270fd0c5362 Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Sun, 23 Jun 2024 13:03:18 -0400 Subject: [PATCH 3/7] nitcc: add tests for plusize and cie. Signed-off-by: Jean Privat --- contrib/nitcc/tests/parser-plusize.input | 3 ++ contrib/nitcc/tests/parser-plusize.sablecc | 8 +++++ contrib/nitcc/tests/parser-questize.input | 3 ++ contrib/nitcc/tests/parser-questize.sablecc | 7 +++++ contrib/nitcc/tests/parser-starize.input | 4 +++ contrib/nitcc/tests/parser-starize.sablecc | 8 +++++ .../nitcc/tests/sav/parser-plusize.input.res | 28 +++++++++++++++++ .../nitcc/tests/sav/parser-questize.input.res | 18 +++++++++++ .../nitcc/tests/sav/parser-starize.input.res | 30 +++++++++++++++++++ 9 files changed, 109 insertions(+) create mode 100644 contrib/nitcc/tests/parser-plusize.input create mode 100644 contrib/nitcc/tests/parser-plusize.sablecc create mode 100644 contrib/nitcc/tests/parser-questize.input create mode 100644 contrib/nitcc/tests/parser-questize.sablecc create mode 100644 contrib/nitcc/tests/parser-starize.input create mode 100644 contrib/nitcc/tests/parser-starize.sablecc create mode 100644 contrib/nitcc/tests/sav/parser-plusize.input.res create mode 100644 contrib/nitcc/tests/sav/parser-questize.input.res create mode 100644 contrib/nitcc/tests/sav/parser-starize.input.res diff --git a/contrib/nitcc/tests/parser-plusize.input b/contrib/nitcc/tests/parser-plusize.input new file mode 100644 index 0000000000..46b332786b --- /dev/null +++ b/contrib/nitcc/tests/parser-plusize.input @@ -0,0 +1,3 @@ +s ab +s abab +s abb diff --git a/contrib/nitcc/tests/parser-plusize.sablecc b/contrib/nitcc/tests/parser-plusize.sablecc new file mode 100644 index 0000000000..7bbec89526 --- /dev/null +++ b/contrib/nitcc/tests/parser-plusize.sablecc @@ -0,0 +1,8 @@ +Grammar gram; +Lexer +blank = #10 | #13 | #32; +Parser +Ignored blank; +s = s 's' t | Empty; +t = ab+ ; +ab = 'a' 'b' ; diff --git a/contrib/nitcc/tests/parser-questize.input b/contrib/nitcc/tests/parser-questize.input new file mode 100644 index 0000000000..9b3dd27ae0 --- /dev/null +++ b/contrib/nitcc/tests/parser-questize.input @@ -0,0 +1,3 @@ +sabc +sac +sabbc diff --git a/contrib/nitcc/tests/parser-questize.sablecc b/contrib/nitcc/tests/parser-questize.sablecc new file mode 100644 index 0000000000..5ca2539078 --- /dev/null +++ b/contrib/nitcc/tests/parser-questize.sablecc @@ -0,0 +1,7 @@ +Grammar gram; +Lexer +blank = #10 | #13 | #32; +Parser +Ignored blank; +s = s 's' t | Empty; +t = 'a' 'b'? 'c'; diff --git a/contrib/nitcc/tests/parser-starize.input b/contrib/nitcc/tests/parser-starize.input new file mode 100644 index 0000000000..16f7088df1 --- /dev/null +++ b/contrib/nitcc/tests/parser-starize.input @@ -0,0 +1,4 @@ +s +sab +sabab +saba diff --git a/contrib/nitcc/tests/parser-starize.sablecc b/contrib/nitcc/tests/parser-starize.sablecc new file mode 100644 index 0000000000..93e3c5fe76 --- /dev/null +++ b/contrib/nitcc/tests/parser-starize.sablecc @@ -0,0 +1,8 @@ +Grammar gram; +Lexer +blank = #10 | #13 | #32; +Parser +Ignored blank; +s = s 's' t | Empty; +t = ab* ; +ab = 'a' 'b'; diff --git a/contrib/nitcc/tests/sav/parser-plusize.input.res b/contrib/nitcc/tests/sav/parser-plusize.input.res new file mode 100644 index 0000000000..2e4eab4721 --- /dev/null +++ b/contrib/nitcc/tests/sav/parser-plusize.input.res @@ -0,0 +1,28 @@ +NParserError@(3:5-3:6)='b' +Nodes[Node] + s_0 + s_0 + s_0 + s_1 + 's'@(1:1-1:2) + t + Nodes[Nab] + ab + 'a'@(1:3-1:4) + 'b'@(1:4-1:5) + 's'@(2:1-2:2) + t + Nodes[Nab] + ab + 'a'@(2:3-2:4) + 'b'@(2:4-2:5) + ab + 'a'@(2:5-2:6) + 'b'@(2:6-2:7) + 's'@(3:1-3:2) + t + Nodes[Nab] + ab + 'a'@(3:3-3:4) + 'b'@(3:4-3:5) + 'b'@(3:5-3:6) diff --git a/contrib/nitcc/tests/sav/parser-questize.input.res b/contrib/nitcc/tests/sav/parser-questize.input.res new file mode 100644 index 0000000000..6fced6d538 --- /dev/null +++ b/contrib/nitcc/tests/sav/parser-questize.input.res @@ -0,0 +1,18 @@ +NParserError@(3:4-3:5)='b' +Nodes[Node] + s_0 + s_0 + s_1 + 's'@(1:1-1:2) + t + 'a'@(1:2-1:3) + 'b'@(1:3-1:4) + 'c'@(1:4-1:5) + 's'@(2:1-2:2) + t + 'a'@(2:2-2:3) + 'c'@(2:3-2:4) + 's'@(3:1-3:2) + 'a'@(3:2-3:3) + 'b'@(3:3-3:4) + 'b'@(3:4-3:5) diff --git a/contrib/nitcc/tests/sav/parser-starize.input.res b/contrib/nitcc/tests/sav/parser-starize.input.res new file mode 100644 index 0000000000..b19bdb9a3f --- /dev/null +++ b/contrib/nitcc/tests/sav/parser-starize.input.res @@ -0,0 +1,30 @@ +NParserError@(5:1-5:1)='' +Nodes[Node] + s_0 + s_0 + s_0 + s_1 + 's'@(1:1-1:2) + t + 's'@(2:1-2:2) + t + Nodes[Nab] + ab + 'a'@(2:2-2:3) + 'b'@(2:3-2:4) + 's'@(3:1-3:2) + t + Nodes[Nab] + ab + 'a'@(3:2-3:3) + 'b'@(3:3-3:4) + ab + 'a'@(3:4-3:5) + 'b'@(3:5-3:6) + 's'@(4:1-4:2) + Nodes[Nab] + ab + 'a'@(4:2-4:3) + 'b'@(4:3-4:4) + 'a'@(4:4-4:5) + Eof@(5:1-5:1)='' From abe4d26f279b251df5bf1e8aaa182f8512689039 Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Sun, 23 Jun 2024 13:29:09 -0400 Subject: [PATCH 4/7] nitcc: add groupize to create anonymous productions Signed-off-by: Jean Privat --- contrib/nitcc/src/nitcc.sablecc | 1 + contrib/nitcc/src/nitcc_semantic.nit | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/contrib/nitcc/src/nitcc.sablecc b/contrib/nitcc/src/nitcc.sablecc index d435c776cf..34c6e6aae4 100644 --- a/contrib/nitcc/src/nitcc.sablecc +++ b/contrib/nitcc/src/nitcc.sablecc @@ -111,6 +111,7 @@ elem = {star:} elem '*' | {ques:} elem '?' | {plus:} elem '+' | + {par:} '(' elem+ ')' | {empty:} 'Empty' ; priority = diff --git a/contrib/nitcc/src/nitcc_semantic.nit b/contrib/nitcc/src/nitcc_semantic.nit index 17b7850e3a..50b0ec7bbd 100644 --- a/contrib/nitcc/src/nitcc_semantic.nit +++ b/contrib/nitcc/src/nitcc_semantic.nit @@ -157,6 +157,22 @@ private class CheckNameVisitor return prod end + # Pool for anoymous grouped production (reuse them!) + var groupizes = new HashMap[Array[Element], Production] + + # Create an anonymous production that groups some elements. + # Note: an anonymous production is always returned, even if `es` is empty or single. + fun groupize(es: Array[Element]): Production + do + if groupizes.has_key(es) then return groupizes[es] + var name = "_group{groupizes.length}" + var prod = new Production(name) + v1.gram.prods.add(prod) + var a1 = prod.new_alt2("{name}_single", es) + groupizes[es] = prod + return prod + end + # The current nexpr, used to track dependency on named expressions (see `Nexpr::precs`) var nexpr: nullable Nexpr = null @@ -703,6 +719,17 @@ redef class Nelem_plus end end +redef class Nelem_par + redef fun accept_check_name_visitor(v) do + var old = v.elems + v.elems = new Array[Element] + super + var elem = v.groupize(v.elems) + v.elems = old + set_elem(v, null, elem) + end +end + redef class Ntree_part redef fun accept_collect_prod(v) do print "NOT YET IMPLEMENTED: `Tree` part; it is ignored" From 255d64d3915d19514689ac015ba11e11ba1c7fe9 Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Sun, 23 Jun 2024 15:56:09 -0400 Subject: [PATCH 5/7] nitcc: add test for groupize Signed-off-by: Jean Privat --- contrib/nitcc/tests/parser-groupize.input | 9 ++++ contrib/nitcc/tests/parser-groupize.sablecc | 12 +++++ .../nitcc/tests/sav/parser-groupize.input.res | 51 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 contrib/nitcc/tests/parser-groupize.input create mode 100644 contrib/nitcc/tests/parser-groupize.sablecc create mode 100644 contrib/nitcc/tests/sav/parser-groupize.input.res diff --git a/contrib/nitcc/tests/parser-groupize.input b/contrib/nitcc/tests/parser-groupize.input new file mode 100644 index 0000000000..0498245f52 --- /dev/null +++ b/contrib/nitcc/tests/parser-groupize.input @@ -0,0 +1,9 @@ +sa +sab +sac +s +sad +sadad +t +tae +taeae diff --git a/contrib/nitcc/tests/parser-groupize.sablecc b/contrib/nitcc/tests/parser-groupize.sablecc new file mode 100644 index 0000000000..819d44460b --- /dev/null +++ b/contrib/nitcc/tests/parser-groupize.sablecc @@ -0,0 +1,12 @@ +Grammar gram; +Lexer +blank = #10 | #13 | #32; +Parser +Ignored blank; +s = + s 's' ('a' 'b') | + s 's' ('a') | + s 's' ('a' 'c')? | + s 's' ('a' 'd')+ | + s 't' ('a' 'e')* | + Empty; diff --git a/contrib/nitcc/tests/sav/parser-groupize.input.res b/contrib/nitcc/tests/sav/parser-groupize.input.res new file mode 100644 index 0000000000..6e21baff68 --- /dev/null +++ b/contrib/nitcc/tests/sav/parser-groupize.input.res @@ -0,0 +1,51 @@ +Start + s_4 + s_4 + s_4 + s_3 + s_3 + s_2 + s_2 + s_0 + s_1 + s_5 + 's'@(1:1-1:2) + _group1_single + 'a'@(1:2-1:3) + 's'@(2:1-2:2) + _group0_single + 'a'@(2:2-2:3) + 'b'@(2:3-2:4) + 's'@(3:1-3:2) + _group2_single + 'a'@(3:2-3:3) + 'c'@(3:3-3:4) + 's'@(4:1-4:2) + 's'@(5:1-5:2) + Nodes[N_group3] + _group3_single + 'a'@(5:2-5:3) + 'd'@(5:3-5:4) + 's'@(6:1-6:2) + Nodes[N_group3] + _group3_single + 'a'@(6:2-6:3) + 'd'@(6:3-6:4) + _group3_single + 'a'@(6:4-6:5) + 'd'@(6:5-6:6) + 't'@(7:1-7:2) + 't'@(8:1-8:2) + Nodes[N_group4] + _group4_single + 'a'@(8:2-8:3) + 'e'@(8:3-8:4) + 't'@(9:1-9:2) + Nodes[N_group4] + _group4_single + 'a'@(9:2-9:3) + 'e'@(9:3-9:4) + _group4_single + 'a'@(9:4-9:5) + 'e'@(9:5-9:6) + Eof@(10:1-10:1)='' From 5f361b006cb795ad3a01bc282419ffd636534614 Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Sun, 23 Jun 2024 15:59:39 -0400 Subject: [PATCH 6/7] nitcc: document plusize and quesize Signed-off-by: Jean Privat --- contrib/nitcc/src/nitcc_semantic.nit | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/contrib/nitcc/src/nitcc_semantic.nit b/contrib/nitcc/src/nitcc_semantic.nit index 50b0ec7bbd..5905d9f940 100644 --- a/contrib/nitcc/src/nitcc_semantic.nit +++ b/contrib/nitcc/src/nitcc_semantic.nit @@ -123,6 +123,17 @@ private class CheckNameVisitor var plusizes = new HashMap[Element, Production] # Create a + version of an element + # + # ``` + # foo = x bar+ y | z ; + # ``` + # + # becomes + # + # ``` + # foo = x abs0 y | z ; + # abs0 = {one:} bar | {more:} abs0 bar ; + # ``` fun plusize(e: Element): Production do if plusizes.has_key(e) then return plusizes[e] @@ -142,6 +153,17 @@ private class CheckNameVisitor var quesizes = new HashMap[Element, Production] # Create a ? version of an element + # + # ``` + # foo = x bar? y | z ; + # ``` + # + # becomes + # + # ``` + # foo = x abs0 y | z ; + # abs0 = {one:} bar | {none} Empty ; + # ``` fun quesize(e: Element): Production do if quesizes.has_key(e) then return quesizes[e] From a82ff721bf150b7c3f8d5aa8c3480723b2fc4a62 Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Fri, 28 Jun 2024 06:00:08 -0400 Subject: [PATCH 7/7] nitcc: update tests that use Separator Signed-off-by: Jean Privat --- contrib/nitcc/tests/sav/inf5000-06-grammaire2-grammaire2.res | 2 +- .../nitcc/tests/sav/inf5000-06-grammaire2-instructions.alt3.res | 2 +- contrib/nitcc/tests/sav/inf5000-06-grammaire2-polygone.res | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/nitcc/tests/sav/inf5000-06-grammaire2-grammaire2.res b/contrib/nitcc/tests/sav/inf5000-06-grammaire2-grammaire2.res index 8b96f3e259..c344a5103b 100644 --- a/contrib/nitcc/tests/sav/inf5000-06-grammaire2-grammaire2.res +++ b/contrib/nitcc/tests/sav/inf5000-06-grammaire2-grammaire2.res @@ -1 +1 @@ -11:15-11:16 Syntax Error: Unexpected '('; is acceptable instead: priority+, priority +11:20-11:29 Syntax Error: Unexpected unknown_keyword 'Separator'; is acceptable instead: elem, text diff --git a/contrib/nitcc/tests/sav/inf5000-06-grammaire2-instructions.alt3.res b/contrib/nitcc/tests/sav/inf5000-06-grammaire2-instructions.alt3.res index db1e0bb76d..46710676f3 100644 --- a/contrib/nitcc/tests/sav/inf5000-06-grammaire2-instructions.alt3.res +++ b/contrib/nitcc/tests/sav/inf5000-06-grammaire2-instructions.alt3.res @@ -1 +1 @@ -20:8-20:9 Syntax Error: Unexpected '('; is acceptable instead: priority+, priority +20:14-20:23 Syntax Error: Unexpected unknown_keyword 'Separator'; is acceptable instead: elem, text diff --git a/contrib/nitcc/tests/sav/inf5000-06-grammaire2-polygone.res b/contrib/nitcc/tests/sav/inf5000-06-grammaire2-polygone.res index 69ed902340..0f93b1b841 100644 --- a/contrib/nitcc/tests/sav/inf5000-06-grammaire2-polygone.res +++ b/contrib/nitcc/tests/sav/inf5000-06-grammaire2-polygone.res @@ -1 +1 @@ -8:18-8:19 Syntax Error: Unexpected '('; is acceptable instead: priority+, priority +8:25-8:34 Syntax Error: Unexpected unknown_keyword 'Separator'; is acceptable instead: elem, text