Merge: lib/markdown2: introduce a new Markdown engine

This PR introduces a new Markdown engine called `markdown2`.

The current Markdown implementation is functional but does not follows the [CommonMark](http://commonmark.org/) specification as it passes only 273 tests of the 627 from the [CommonMark test suite](https://github.com/CommonMark/CommonMark).

Also there is a few issues with it as stated in #791, #1394, #1525 and #2507.

This new implementation aims at resolving these issues and follow the CommonMark specification.

Features:
* Markdown AST creation and rendering
* Rendering to HTML
* Rendering to Markdown
* Rendering to Man format
* Rendering to LaTeX
* Parsing of Github extended mode
* Parsing of wikilinks
* Parsing of LaTeX/Maths expression (upcoming in next PR)
* Respect of the CommonMark specification with only 10 tests failed (UTF-8 related)
* Extensive testing with a total of 980 test units...

I didn't remove the old markdown implementation since the benchmarks are not really good for now. Since `markdown2` uses a lot of regular expressions, performances can be an issue compared to `markdown`:

![screenshot from 2018-06-20 19 21 08](https://user-images.githubusercontent.com/583144/41689548-1a8bc5d0-74bf-11e8-899e-52b0eb093d57.png)

In the following PR some Nit tools will be migrated to the new implementation:
* nitunit
* nitiwiki
* nitdoc
* nitweb

Pull-Request: #2720

benchmarks/markdown/bench_markdown.sh

@@ -97,6 +97,30 @@ function bench_nitmd-o()
 }
 bench_nitmd-o
 
+function bench_nitmd2()
+{
+	name="$FUNCNAME"
+	skip_test "$name" && return
+	prepare_res $outdir/nitmd2.dat "nitmd2" "nitmd2"
+	for file in $bncdir/*.md; do
+		bench=`basename $file .md`
+		bench_command "$bench" "" "$engdir/nitmd2/nitmd2" "$file" "$s"
+	done
+}
+bench_nitmd2
+
+function bench_nitmd2-o()
+{
+	name="$FUNCNAME"
+	skip_test "$name" && return
+	prepare_res $outdir/nitmd2-o.dat "nitmd2-o" "nitmd2-o"
+	for file in $bncdir/*.md; do
+		bench=`basename $file .md`
+		bench_command "$bench" "" "$engdir/nitmd2/nitmd2-o" "$file" "$s"
+	done
+}
+bench_nitmd2-o
+
 function bench_txtmark()
 {
 	name="$FUNCNAME"

benchmarks/markdown/engines/Makefile

@@ -14,11 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-all: nitmd/nitmd txtmark/Txtmark.class markdown4j/Markdown4j.class
+all: nitmd/nitmd nitmd2/nitmd2 txtmark/Txtmark.class markdown4j/Markdown4j.class
 
 nitmd/nitmd:
 	make -C nitmd
 
+nitmd2/nitmd2:
+	make -C nitmd2
+
 txtmark/Txtmark.class:
 	make -C txtmark
 
@@ -30,6 +33,7 @@ pandoc/pandoc:
 
 clean:
 	make -C nitmd clean
+	make -C nitmd2 clean
 	make -C txtmark clean
 	make -C markdown4j clean
 	make -C pandoc clean

benchmarks/markdown/engines/nitmd2/Makefile

@@ -0,0 +1,32 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Copyright 2015 Alexandre Terrasa <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+NITC=../../../../bin/nitc
+
+all: nitmd2 nitmd2-o
+
+nitmd2:
+	$(NITC) nitmd2.nit
+
+nitmd2-o:
+	$(NITC) --semi-global nitmd2.nit -o $@
+
+test: all
+	./nitmd2 ../../benches/hello.md 5
+	./nitmd2-o ../../benches/hello.md 5
+
+clean:
+	rm -rf nitmd2 nitmd2-o

benchmarks/markdown/engines/nitmd2/nitmd2.nit

@@ -0,0 +1,26 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import markdown2
+
+var file = args.first
+var n = args[1].to_i
+
+var str = file.to_path.read_all
+var parser = new MdParser
+var renderer = new HtmlRenderer
+for i in [1..n] do
+	var doc = parser.parse(str)
+	print renderer.render(doc)
+end

lib/markdown2/markdown2.nit

@@ -0,0 +1,27 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module markdown2
+
+import markdown_block_parsing
+import markdown_html_rendering
+
+redef class String
+	fun md_to_html2: String do
+		var parser = new MdParser
+		var doc = parser.parse(self)
+		var renderer = new HtmlRenderer
+		return renderer.render(doc)
+	end
+end