NitCC implements Separator and Tail

contrib/nitcc/src/grammar.nit

@@ -468,6 +468,23 @@ class CodeNew
 	redef fun to_s do return "New {alt.name}/{alt.elems.length}"
 end
 
+# Allocate a new empty AST node array for an alternative
+class CodeNewNodes
+	super Code
+
+	# The associated alternative
+	var alt: Alternative
+
+	redef fun to_s do return "NewArray {alt.name}"
+end
+
+# Add an element to a Nodes array
+class CodeAdd
+	super Code
+
+	redef fun to_s do return "Add"
+end
+
 # Get null
 class CodeNull
 	super Code
@@ -835,49 +852,49 @@ private class Generator
 			i -= 1
 		end
 
-		if alt.name.has_suffix("+_more") then
-			add "\t\tvar prod = n0"
-			add "\t\tn0.children.add(n1)"
-		else if alt.name.has_suffix("+_one") then
-			add "\t\tvar prod = new {alt.prod.acname}"
-			add "\t\tprod.children.add(n0)"
-		else
-			alt.make_codes
-			var cpt = 0
-			i = 0
-			var st = new Array[String]
-			for c in alt.codes.as(not null) do
-				if c isa CodePop then
-					st.add "n{i}"
-					i += 1
-				else if c isa CodeNull then
-					st.add "null"
-				else if c isa CodeNew then
-					var calt = c.alt
-					cpt += 1
-					var from = st.length - calt.elems.length
-					var args = new List[String]
-					for j in [from..st.length[ do
-						args.unshift(st.pop)
-					end
+		alt.make_codes
+		var cpt = 0
+		i = 0
+		var st = new Array[String]
+		for c in alt.codes.as(not null) do
+			if c isa CodePop then
+				st.add "n{i}"
+				i += 1
+			else if c isa CodeNull then
+				st.add "null"
+			else if c isa CodeNew then
+				var calt = c.alt
+				cpt += 1
+				var from = st.length - calt.elems.length
+				var args = new List[String]
+				for j in [from..st.length[ do
+					args.unshift(st.pop)
+				end
 
-					if args.is_empty then
-						add "\t\tvar p{cpt} = new {calt.cname}"
-					else
-						add "\t\tvar p{cpt} = new {calt.cname}({args.join(", ")})"
-					end
-					#var x = 0
-					#for j in [from..st.length[ do
-						#if st[j] == "null" then continue
-						#add "\t\tp{cpt}.n_{calt.elemname(x)} = {st[j]}"
-						#x += 1
-					#end
-					st.add("p{cpt}")
+				if args.is_empty then
+					add "\t\tvar p{cpt} = new {calt.cname}"
+				else
+					add "\t\tvar p{cpt} = new {calt.cname}({args.join(", ")})"
 				end
+				#var x = 0
+				#for j in [from..st.length[ do
+					#if st[j] == "null" then continue
+					#add "\t\tp{cpt}.n_{calt.elemname(x)} = {st[j]}"
+					#x += 1
+				#end
+				st.add("p{cpt}")
+			else if c isa CodeNewNodes then
+				cpt += 1
+				add "\t\tvar p{cpt} = new {c.alt.prod.acname}"
+				st.add("p{cpt}")
+			else if c isa CodeAdd then
+				var a1 = st.pop
+				var a0 = st.last
+				add "\t\t{a0}.children.add({a1})"
 			end
-			assert st.length == 1
-			add "\t\tvar prod = {st.first}"
 		end
+		assert st.length == 1
+		add "\t\tvar prod = {st.first}"
 
 		add "\t\tparser.node_stack.push prod"
 		if alt.prod.accept then

contrib/nitcc/src/nitcc.sablecc

@@ -111,8 +111,18 @@ elem =
 	{star:} elem '*' |
 	{ques:} elem '?' |
 	{plus:} elem '+' |
+	{group:} '(' elem+ ')' |
+	{groupsep:} '(' elem+ separator [sep:]elem+ ')' mult |
 	{empty:} 'Empty' ;
 
+separator =
+	{sep:} 'Separator' |
+	{tail:} 'Tail' ;
+
+mult =
+	{star:} '*' |
+	{plus:} '+' ;
+
 priority =
 	{left:} 'Left' alts |
 	{right:} 'Right' alts |

contrib/nitcc/src/nitcc_semantic.nit

@@ -123,23 +123,122 @@ private class CheckNameVisitor
 	var plusizes = new HashMap[Element, Production]
 
 	# Create a + version of an element
+	#
+	# ```
+	# foo = x bar+ y | z ;
+	# ```
+	#
+	# becomes
+	#
+	# ```
+	# foo = x abs0 y | z ;
+	# abs0 = {one:} bar | {more:} abs0 bar ;
+	# ```
 	fun plusize(e: Element): Production
 	do
 		if plusizes.has_key(e) then return plusizes[e]
 		var name = "{e}+"
 		var prod = new Production(name)
 		prod.acname = "Nodes[{e.acname}]"
 		v1.gram.prods.add(prod)
-		prod.new_alt("{name}_one", e)
-		prod.new_alt("{name}_more", prod, e)
+		var alt1 = prod.new_alt("{name}_one", e)
+		alt1.codes = [new CodeNewNodes(alt1), new CodePop, new CodeAdd: Code]
+		var alt2 = prod.new_alt("{name}_more", prod, e)
+		alt2.codes = [new CodePop, new CodePop, new CodeAdd: Code]
 		plusizes[e] = prod
 		return prod
 	end
 
+	# Pool of elements that are modified with Separator
+	var separatizes = new HashMap[Array[Element], Production]
+
+	# Create a separated version of an element `e1` where the last element is `e2`.
+	# This is used to implement `Separator` and `Tail` keywords.
+	#
+	# ```
+	# foo = x (bar Separator baz)+ y | z ;
+	# ```
+	#
+	# becomes
+	#
+	# ```
+	# foo = x abs1 y | z ;
+	# abs0 = bar baz ;
+	# abs1 = (one:) bar | {more:} abs0+ bar ;
+	# ```
+	fun separatize(e1, e2: Element): Production
+	do
+		var es = [e1, e2]
+		if separatizes.has_key(es) then return separatizes[es]
+		var name = "_sep{separatizes.length}"
+		var prod = new Production(name)
+		v1.gram.prods.add(prod)
+		var p = plusize(e1)
+		prod.spe = p
+		var alt1 = prod.new_alt("{name}_more", p, e2)
+		alt1.codes = [new CodePop, new CodePop, new CodeAdd: Code]
+		alt1.trans = true
+		var alt2 = prod.new_alt("{name}_one", e2)
+		alt2.codes = [new CodeNewNodes(alt1), new CodePop, new CodeAdd: Code]
+		alt2.trans = true
+		separatizes[es] = prod
+		return prod
+	end
+
+	# Pool of elements that are modified with Tail
+	var tailizes = new HashMap[Array[Element], Production]
+
+	# Create a separated version of an element `e1` where the last `e2` can be present.
+	# This is used to implement `Separator` and `Tail` keywords.
+	#
+	# ```
+	# foo = x (bar Separator baz)+ y | z ;
+	# ```
+	#
+	# becomes
+	#
+	# ```
+	# foo = x abs1 y | z ;
+	# abs0 = bar baz ;
+	# abs1 = (one:) bar | {more:} abs0+ bar | {tail:} abs0+;
+	# ```
+	fun tailize(e1, e2: Element): Production
+	do
+		var es = [e1, e2]
+		if tailizes.has_key(es) then return tailizes[es]
+		var name = "_tail{tailizes.length}"
+		var prod = new Production(name)
+		v1.gram.prods.add(prod)
+		var p = plusize(e1)
+		prod.spe = p
+		var alt1 = prod.new_alt("{name}_more", p, e2)
+		alt1.codes = [new CodePop, new CodePop, new CodeAdd: Code]
+		alt1.trans = true
+		var alt2 = prod.new_alt("{name}_one", e2)
+		alt2.codes = [new CodeNewNodes(alt1), new CodePop, new CodeAdd: Code]
+		alt2.trans = true
+		var alt3 = prod.new_alt("{name}_tail", p)
+		alt3.codes = [new CodePop]
+		alt3.trans = true
+		tailizes[es] = prod
+		return prod
+	end
+
 	# Pool of elements that are modified with ? (reuse them!)
 	var quesizes = new HashMap[Element, Production]
 
 	# Create a ? version of an element
+	#
+	# ```
+	# foo = x bar? y | z ;
+	# ```
+	#
+	# becomes
+	#
+	# ```
+	# foo = x abs0 y | z ;
+	# abs0 = {one:} bar | {none} Empty ;
+	# ```
 	fun quesize(e: Element): Production
 	do
 		if quesizes.has_key(e) then return quesizes[e]
@@ -155,6 +254,22 @@ private class CheckNameVisitor
 		return prod
 	end
 
+	# Pool for anoymous grouped production (reuse them!)
+	var groupizes = new HashMap[Array[Element], Production]
+
+	# Create an anonymous production that groups some elements.
+	# Note: an anonymous production is always returned, even if `es` is empty or single.
+	fun groupize(es: Array[Element]): Production
+	do
+		if groupizes.has_key(es) then return groupizes[es]
+		var name = "_group{groupizes.length}"
+		var prod = new Production(name)
+		v1.gram.prods.add(prod)
+		var a1 = prod.new_alt2("{name}_single", es)
+		groupizes[es] = prod
+		return prod
+	end
+
 	# The current nexpr, used to track dependency on named expressions (see `Nexpr::precs`)
 	var nexpr: nullable Nexpr = null
 
@@ -701,6 +816,40 @@ redef class Nelem_plus
 	end
 end
 
+redef class Nelem_group
+	redef fun accept_check_name_visitor(v) do
+		var old = v.elems
+		v.elems = new Array[Element]
+		super
+		var elem = v.groupize(v.elems)
+		v.elems = old
+		set_elem(v, null, elem)
+	end
+end
+
+redef class Nelem_groupsep
+	redef fun accept_check_name_visitor(v) do
+		var old = v.elems
+		v.elems = new Array[Element]
+		n_elem.accept_check_name_visitor(v)
+		var prod1 = v.groupize(v.elems.clone)
+		n_sep.accept_check_name_visitor(v)
+		var prod2 = v.groupize(v.elems)
+		prod1.spe = prod2
+		v.elems = old
+		var prod
+		if n_separator isa Nseparator_sep then
+			prod = v.separatize(prod2, prod1)
+		else
+			prod = v.tailize(prod2, prod1)
+		end
+		if n_mult isa Nmult_star then
+			prod = v.quesize(prod)
+		end
+		set_elem(v, null, prod)
+	end
+end
+
 redef class Ntree_part
 	redef fun accept_collect_prod(v) do
 		print "NOT YET IMPLEMENTED: `Tree` part; it is ignored"

contrib/nitcc/tests/parser-groupize.input

@@ -0,0 +1,9 @@
+sa
+sab
+sac
+s
+sad
+sadad
+t
+tae
+taeae

contrib/nitcc/tests/parser-groupize.sablecc

@@ -0,0 +1,12 @@
+Grammar gram;
+Lexer
+blank = #10 | #13 | #32;
+Parser
+Ignored blank;
+s =
+	s 's' ('a' 'b') |
+	s 's' ('a') |
+	s 's' ('a' 'c')? |
+	s 's' ('a' 'd')+ |
+	s 't' ('a' 'e')* |
+	Empty;

contrib/nitcc/tests/parser-plusize.input

@@ -0,0 +1,3 @@
+s ab
+s abab
+s abb

contrib/nitcc/tests/parser-plusize.sablecc

@@ -0,0 +1,8 @@
+Grammar gram;
+Lexer
+blank = #10 | #13 | #32;
+Parser
+Ignored blank;
+s = s 's' t | Empty;
+t = ab+ ;
+ab = 'a' 'b' ;

contrib/nitcc/tests/parser-questize.input

@@ -0,0 +1,3 @@
+sabc
+sac
+sabbc

contrib/nitcc/tests/parser-questize.sablecc

@@ -0,0 +1,7 @@
+Grammar gram;
+Lexer
+blank = #10 | #13 | #32;
+Parser
+Ignored blank;
+s = s 's' t | Empty;
+t = 'a' 'b'? 'c';

contrib/nitcc/tests/parser-separator.input

@@ -0,0 +1,7 @@
+s a b a
+s a b a b a
+s a
+s a c d e a c
+s a c d e a c d e a c
+s a c
+s a c d e

contrib/nitcc/tests/parser-separator.sablecc

@@ -0,0 +1,10 @@
+Grammar gram;
+Lexer
+blank = #10 | #13 | #32;
+Parser
+Ignored blank;
+s =
+	s 's' ('a' Separator 'b')+ |
+	s 's' ('a' 'c' Separator 'd' 'e')+ |
+	s 's' ('b' Separator 'c')* |
+	Empty;

contrib/nitcc/tests/parser-starize.input

@@ -0,0 +1,4 @@
+s
+sab
+sabab
+saba