Merge pull request #20 from hildjj/pest

Pest

.gitattributes

@@ -1,2 +1,22 @@
+*.cjs text eol=lf
+*.css text eol=lf
+*.csv text eol=lf
+*.html text eol=lf
+*.ico binary
+*.jpg binary
+*.js text eol=lf
+*.json text eol=lf
+*.map text eol=lf
+*.mjs text eol=lf
+*.mk text eol=lf
+*.pdf binary
+*.peg.js text eol=lf linguist-language=PEG.js
+*.peggy text eol=lf linguist-language=PEG.js
+*.pegjs text eol=lf linguist-language=PEG.js
+*.png binary
+*.rs text eol=lf
+*.ts text eol=lf
+*.tsv text eol=lf
+*.tsx text eol=lf
 examples/*.abnf eol=crlf
-
+Makefile text eol=lf

.gitignore

@@ -2,5 +2,6 @@
 coverage/
 examples/*.js
 examples/*.peggy
+examples/*.pest
 node_modules
 t.abnf

bin/abnf_gen.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import * as abnf from "../lib/abnf.js";
-import { Command } from "commander";
+import { Command, Option } from "commander";
 import { fileURLToPath } from "url";
 import fs from "fs";
 import path from "path";
@@ -11,8 +11,8 @@ const __dirname = path.dirname(__filename);
 
 const COMBINED = "XXXXXCOMBINEDXXXXX";
 
-function gen_peggy(rules, opts) {
-  const out = rules.toPeggy(opts);
+function genFormat(rules, opts) {
+  const out = rules.toFormat(opts);
   if (opts.output === "-") {
     console.log(out);
   } else {
@@ -22,16 +22,21 @@ function gen_peggy(rules, opts) {
 
 const program = new Command();
 program
-  .argument("[abnfFile...]", "ABNF files to turn into peggy grammars.")
-  .description("Create a Peggy grammar from an ABNF file")
+  .argument("[abnfFile...]", "ABNF files to turn into grammars.")
+  .description("Create a grammar from an ABNF file")
+  .addOption(
+    new Option("-f, --format <format>", "Output format")
+      .choices(["peggy", "pest"])
+      .default("peggy")
+  )
   .option(
     "-s, --startRule <ruleName>",
-    "Start rule for peggy grammar.  Defaults to first rule in ABNF grammar."
+    "Start rule for generated grammar.  Defaults to first rule in ABNF grammar."
   )
   .option("--stubs", "Generate stubs for rules that do not exist, rather than failing.")
   .option(
     "-o, --output <file>",
-    "Output peggy grammar file name.  Derived from input file name if not specified.",
+    "Output grammar file name.  Derived from input file name if not specified.",
     "stdin.peggy"
   )
   .option(
@@ -62,7 +67,7 @@ program
         if (cmd.getOptionValueSource("output") === "default") {
           const p = path.parse(f);
           delete p.base;
-          p.ext = ".peggy";
+          p.ext = `.${opts.format}`;
           cmd.setOptionValueWithSource("output", path.format(p), "first");
         }
         text = fs.readFileSync(f, "utf8");
@@ -78,8 +83,9 @@ program
     }
 
     try {
-      const rules = abnf.parseString(input, COMBINED);
-      gen_peggy(rules, opts);
+      const utf16 = opts.format === "peggy";
+      const rules = abnf.parseString(input, COMBINED, utf16);
+      genFormat(rules, opts);
     } catch (er) {
       if (typeof er.format === "function") {
         const line = er.location.start.line;

bin/abnf_test.js

@@ -35,7 +35,7 @@ program
       let text = null;
       try {
         const rules = await abnf.parseString(abnfSource, f);
-        text = rules.toPeggy(opts);
+        text = rules.toFormat(opts);
 
         const abnfOpts = {
           grammarSource: f,

examples/Makefile

@@ -1,20 +1,28 @@
 abnf = $(wildcard *.abnf)
 peggy = $(patsubst %.abnf,%.peggy,$(abnf))
+pest = $(patsubst %.abnf,%.pest,$(abnf))
 js = $(patsubst %.peggy,%.js,$(peggy))
 
 %.peggy: %.abnf
 	../bin/abnf_gen.js $< -uo $@
 
+%.pest: %.abnf
+	../bin/abnf_gen.js -f pest $< -uo $@
+	pest_debugger -g $@ < /dev/null
+
 .PRECIOUS: %.peggy
 %.js: %.peggy
 	npx peggy $< -o $@
 
 .PHONY: all
-all: $(js)
+all: $(js) $(pest)
 
 .PHONY: clean
 clean:
-	$(RM) $(js) $(peggy)
+	$(RM) $(js) $(peggy) $(pest)
 
 json.peggy: json.abnf
 	../bin/abnf_gen.js $< -cuo $@
+
+json.pest: json.abnf
+	../bin/abnf_gen.js $< -f pest -cuo $@

lib/abnf.js

@@ -2,11 +2,12 @@ import * as ast from "./ast.js";
 import * as fs from "fs";
 import { parse } from "./abnfp.js";
 
-export function parseString(str, grammarSource = "unknown") {
+export function parseString(str, grammarSource = "unknown", utf16 = true) {
   const text = str;
   try {
     return parse(text, {
       grammarSource,
+      utf16,
     });
   } catch (er) {
     er.grammarSource = grammarSource;
@@ -18,7 +19,7 @@ export function parseString(str, grammarSource = "unknown") {
   }
 }
 
-export function parseStream(input, grammarSource = "stdin") {
+export function parseStream(input, grammarSource = "stdin", utf16 = true) {
   return new Promise((resolve, reject) => {
     const bufs = [];
 
@@ -41,14 +42,14 @@ export function parseStream(input, grammarSource = "stdin") {
           break;
       }
       try {
-        resolve(parseString(s, grammarSource));
+        resolve(parseString(s, grammarSource, utf16));
       } catch (er) {
         reject(er);
       }
     });
   });
 }
 
-export function parseFile(input) {
-  return parseString(fs.readFileSync(input, "utf8"), input);
+export function parseFile(input, utf16 = true) {
+  return parseString(fs.readFileSync(input, "utf8"), input, utf16);
 }

lib/abnfp.js

@@ -321,7 +321,7 @@ function peg$parse(input, options) {
     );
   };
   var peg$f14 = function(min, max) {
-    return ast.Range.create(2, min, max, location());
+    return ast.Range.create(2, min, max, location(), options.utf16);
   };
   var peg$f15 = function(n) {
     return new ast.CaseSensitiveString(String.fromCodePoint(n), 2, location())
@@ -334,7 +334,7 @@ function peg$parse(input, options) {
     );
   };
   var peg$f17 = function(min, max) {
-    return ast.Range.create(10, min, max, location());
+    return ast.Range.create(10, min, max, location(), options.utf16);
   };
   var peg$f18 = function(n) {
     return new ast.CaseSensitiveString(String.fromCodePoint(n), 10, location())
@@ -347,7 +347,7 @@ function peg$parse(input, options) {
     );
   };
   var peg$f20 = function(min, max) {
-    return ast.Range.create(16, min, max, location());
+    return ast.Range.create(16, min, max, location(), options.utf16);
   };
   var peg$f21 = function(n) {
     return new ast.CaseSensitiveString(String.fromCodePoint(n), 10, location())

lib/abnfp.peggy

@@ -114,7 +114,7 @@ bin_val
     );
   }
   / "b"i min:binnum "-" max:binnum {
-    return ast.Range.create(2, min, max, location());
+    return ast.Range.create(2, min, max, location(), options.utf16);
   }
   / "b"i n:binnum {
     return new ast.CaseSensitiveString(String.fromCodePoint(n), 2, location())
@@ -129,7 +129,7 @@ dec_val
     );
   }
   / "d"i min:decnum "-" max:decnum {
-    return ast.Range.create(10, min, max, location());
+    return ast.Range.create(10, min, max, location(), options.utf16);
   }
   / "d"i n:decnum {
     return new ast.CaseSensitiveString(String.fromCodePoint(n), 10, location())
@@ -144,7 +144,7 @@ hex_val
     );
   }
   / "x"i min:hexnum "-" max:hexnum {
-    return ast.Range.create(16, min, max, location());
+    return ast.Range.create(16, min, max, location(), options.utf16);
   }
   / "x"i n:hexnum {
     return new ast.CaseSensitiveString(String.fromCodePoint(n), 10, location())