Add menhir table for packages

sphinx/conf.py

@@ -220,4 +220,4 @@
 
 # entry point for setup
 def setup(app):
-    app.add_stylesheet('css/fixes.css')
+    app.add_css_file('css/fixes.css')

sphinx/reference.rst

@@ -340,6 +340,22 @@ if not specified:
 * :code:`synopsis`: the one-line description of the package
 * :code:`version`: the version of this package
 
+For :code:`menhir` usage, you will find an optional :code:`[menhir]` talbe,
+with the following fields:
+* :code:`version`: the version of the menhir language configuration
+* :code:`parser`: a table with the following fields:
+  * :code:`modules`: an array of string with all your menhir modules
+  * :code:`merge-into`: an optional string for the name of the merged module,
+    by default the last value in the :code:`modules` array if not provided
+  * :code:`tokens`: an optional to add the :code:`--external-tokens` flag
+  * :code:`flags`: an optional string array with menhir flags
+  * :code:`infer`: an optional boolean which adds the :code:`(infer <bool>)`
+    rule to the :code:`dune` file
+* :code:`tokens`: an optional table that will add the `--only-tokens` flag
+  to the :code:`menhir` rule with the following fiels:
+  * :code:`modules`: a string array with all the modules to add to the rule
+  * :code:`flags`: an optional string array with all the flags to add to menhir
+
 Finally, there is a table :code:`[fields]` within a
 package. Currently, the following fields are used by skeletons:
 

sphinx/usecases.rst

@@ -8,18 +8,52 @@ This sections describes how the handle common use cases with :code:`drom`.
 Using :code:`menhir`
 --------------------
 
-Whereas :code:`ocamlyacc` is the legacy parser generator for OCaml, 
+Whereas :code:`ocamlyacc` is the legacy parser generator for OCaml,
 :code:`menhir` is much more powerful and probably more prevalent nowadays.
 
 For trivial usage where there is a simple :code:`parser.mly` file in your
 package, setting::
-  
+
   generators = ["menhir"]
 
-into the corresponding :code:`package.toml` will likely work. However, if 
+into the corresponding :code:`package.toml` will likely work. However, if
 you have multiple :code:`.mly` files, or
-need to specify custom :code:`menhir` flags, you'll probably need to tune
-the :code:`dune` generation. A good way to properly do that is to disable
+need to specify custom :code:`menhir` flags, you'll probably need to use the
+:code:`[menhir]` table in the corresponding :code:`package.toml`. For example::
+
+  [menhir]
+  version = "2.1"
+  parser = {
+    modules = ["tokens", "parser"],
+    merge-into = "parser",
+    tokens = "Tokens",
+    flags = [ "--table" ],
+    infer = true
+  }
+  tokens = {
+    modules = ["tokens"],
+    #flags = []
+  }
+
+
+This will generate the following rules in you :code:`dune` file::
+
+  (menhir
+    (modules tokens)
+    (flags --only-tokens))
+  (menhir
+    (modules tokens parser)
+    (merge_into parser)
+    (flags --table --external-tokens Tokens)
+    (infer true))
+
+and will add::
+
+  (using menhir 2.1)
+
+to the :code:`dune-project` file.
+
+You can also can tune the :code:`dune` generation. A good way to properly do that is to disable
 the default :code:`dune` generation for and add your own :code:`dune` stanzas
 for parsing generators. First, add :code:`(using menhir X.Y)` in the 
 :code:`drom.toml`'s ':code:`dune-project-header` field. For example::
@@ -35,8 +69,8 @@ Then replace the :code:`generators = ["menhir"]` line by the following::
 
   generators = []
 
-in :code:`package.toml` to disable the default :code:`dune` stanzas generation. 
-Finally, add the following :code:`dune` stanzas to your 
+in :code:`package.toml` to disable the default :code:`dune` stanzas generation.
+Finally, add the following :code:`dune` stanzas to your
 :code:`dune-trailer` field::
 
   [fields]
@@ -56,4 +90,4 @@ Finally, add the following :code:`dune` stanzas to your
 Of course, this is just a basic tuning and you can modify the flags or
 targets as needed. The overall result will likely fit in most
 of :code:`menhir` usages in :code:`drom` projects waiting for a better
-:code:`menhir` support.
+:code:`menhir` support.

src/drom_lib/dune.ml

@@ -25,7 +25,58 @@ let package_dune_files package =
   let p_generators =
     match package.p_generators with
     | None -> StringSet.of_list [ "ocamllex"; "ocamlyacc" ]
-    | Some generators -> generators
+    | Some generators ->
+        begin match package.p_menhir with
+          | Some { parser; tokens = menhir_tokens; _ }
+            when StringSet.mem "menhir" generators ->
+              let { modules; tokens; merge_into; flags; infer } = parser in
+              Printf.bprintf b "(menhir\n  (modules";
+              List.iter (fun module_ -> Printf.bprintf b " %s" module_) modules;
+              Printf.bprintf b ")";
+              begin match merge_into with
+                | Some merge_into ->
+                    Printf.bprintf b "\n  (merge_into %s)" merge_into
+                | None ->
+                    if List.length modules > 1 then
+                      let merge_into = List.rev modules |> List.hd in
+                      Printf.bprintf b "\n  (merge_into %s)" merge_into
+              end;
+              begin match flags with
+                | Some flags ->
+                    Printf.bprintf b "\n  (flags";
+                    List.iter (fun flag -> Printf.bprintf b " %s" flag) flags;
+                    begin match tokens with
+                      | Some tokens -> Printf.bprintf b "--external-token %s" tokens
+                      | None -> ()
+                    end;
+                    Printf.bprintf b ")";
+                | None -> ()
+              end;
+              begin match infer with
+                | Some infer -> Printf.bprintf b "\n  (infer %B)" infer
+                | None -> ()
+              end;
+              Printf.bprintf b ")\n";
+              begin match menhir_tokens with
+                | Some { modules; flags; } ->
+                    Printf.bprintf b "(menhir (modules";
+                    List.iter (fun module_ -> Printf.bprintf b " %s" module_) modules;
+                    Printf.bprintf b ")";
+                    begin match flags with
+                      | Some flags ->
+                          Printf.bprintf b "\n  (flags";
+                          List.iter (fun flag -> Printf.bprintf b " %s" flag) flags;
+                          Printf.bprintf b " --only-tokens)";
+                      | None -> ()
+                    end;
+                    Printf.bprintf b ")\n"
+                | None -> ()
+              end
+          | Some _ ->
+              Printf.eprintf "'menhir' table defined without menhir generator"
+          | None -> ()
+        end;
+        generators
   in
   ( match Sys.readdir package.dir with
   | exception _ -> ()
@@ -56,15 +107,18 @@ let package_dune_files package =
               Printf.bprintf b "\n  (mode %s)" mode;
               Printf.bprintf b ")\n"
           end else if StringSet.mem "menhir" p_generators then begin
-            Printf.bprintf b "(menhir (modules %s)"
-              (Filename.chop_suffix file ".mly");
-            List.iter
-              (fun ext ->
-                match StringMap.find ("menhir-" ^ ext) package.p_fields with
-                | exception Not_found -> ()
-                | s -> Printf.bprintf b "\n  (%s %s)" ext s )
-              [ "flags"; "into"; "infer" ];
-            Printf.bprintf b ")\n"
+            match package.p_menhir with
+            | None ->
+                Printf.bprintf b "(menhir (modules %s)"
+                  (Filename.chop_suffix file ".mly");
+                List.iter
+                  (fun ext ->
+                     match StringMap.find ("menhir-" ^ ext) package.p_fields with
+                     | exception Not_found -> ()
+                     | s -> Printf.bprintf b "\n  (%s %s)" ext s )
+                  [ "flags"; "into"; "infer" ];
+                Printf.bprintf b ")\n"
+            | Some _ -> ()
           end else
             Printf.eprintf "no generator for %s\n%!" file
         end )
@@ -148,7 +202,11 @@ let packages p =
   (* If menhir is used as a generator, prevents dune from modifying
      dune-project by adding this line ourselves. *)
   if StringSet.mem "menhir" p.generators then
-    Buffer.add_string b "(using menhir 2.0)\n";
+    begin match p.menhir_version with
+    | Some version ->
+        Printf.bprintf b "(using menhir %s)\n" version
+    | None -> Buffer.add_string b "(using menhir 2.0)\n"
+    end;
 
   List.iter add_package p.packages;
   Buffer.contents b

src/drom_lib/globals.ml

@@ -153,6 +153,7 @@ let rec dummy_project =
     share_dirs = [ "share" ];
     year = (Misc.date ()).Unix.tm_year;
     generators = StringSet.empty;
+    menhir_version = None;
     dune_version = current_dune_version ;
     project_create = false ;
   }
@@ -175,6 +176,7 @@ and dummy_package =
     p_fields = StringMap.empty;
     p_skeleton = None;
     p_generators = None;
+    p_menhir = None;
     p_skip = None;
     p_optional = None;
     p_preprocess = None

src/drom_lib/package.ml

@@ -193,6 +193,65 @@ let stringSet_encoding =
 
 let fields_encoding = EzToml.ENCODING.stringMap EzToml.string_encoding
 
+let menhir_parser_encoding =
+  EzToml.encoding
+    ~to_toml:(fun { modules; tokens; merge_into; flags; infer } ->
+        let table = EzToml.empty
+                    |> EzToml.put_string_list [ "modules" ] modules
+                    |> EzToml.put_string_option [ "tokens" ] tokens
+                    |> EzToml.put_string_option [ "merge-into" ] merge_into
+                    |> EzToml.put_string_list_option [ "flags" ] flags
+                    |> EzToml.put_bool_option [ "infer" ] infer
+        in
+        TTable table)
+    ~of_toml:(fun ~key v ->
+      match v with
+        | TTable table ->
+            let modules = EzToml.get_string_list_default table [ "modules" ] [ "parser" ] in
+            let tokens = EzToml.get_string_option table [ "tokens" ] in
+            let merge_into = EzToml.get_string_option table [ "merge-into" ] in
+            let flags = EzToml.get_string_list_option table [ "flags" ] in
+            let infer = EzToml.get_bool_option table [ "infer" ] in
+            { modules; tokens; merge_into; flags; infer; }
+        | _ ->
+            EzToml.expecting_type "table" key)
+
+let menhir_tokens_encoding =
+  EzToml.encoding
+    ~to_toml:(fun { modules; flags } ->
+        let table = EzToml.empty
+                    |> EzToml.put_string_list [ "modules" ] modules
+                    |> EzToml.put_string_list_option [ "flags" ] flags
+        in
+      TTable table)
+    ~of_toml:(fun ~key v ->
+        match v with
+        | TTable table ->
+            let modules = EzToml.get_string_list_default table [ "modules"] [ "tokens" ] in
+            let flags = EzToml.get_string_list_option table [ "flags" ] in
+            { modules; flags }
+        | _ ->
+            EzToml.expecting_type "table" key)
+
+let menhir_encoding =
+  EzToml.encoding
+    ~to_toml:(fun { version; parser; tokens } ->
+        let table = EzToml.empty
+                    |> EzToml.put_string [ "version" ] version
+                    |> EzToml.put_encoding menhir_parser_encoding [ "parser" ] parser
+                    |> EzToml.put_encoding_option menhir_tokens_encoding [ "encoding" ] tokens
+        in
+        TTable table)
+    ~of_toml:(fun ~key v ->
+        match v with
+        | TTable table ->
+            let version = EzToml.get_string table [ "version" ] in
+            let parser = EzToml.get_encoding menhir_parser_encoding table [ "parser" ] in
+            let tokens = EzToml.get_encoding_option menhir_tokens_encoding table [ "tokens" ] in
+            { version; parser; tokens; }
+        | _ ->
+            EzToml.expecting_type "talbe" key)
+
 let to_string pk =
   EzToml.CONST.(
     s_
@@ -226,6 +285,15 @@ let to_string pk =
             ]
           ~default:{|generators = [ "ocamllex", "menhir" ]|}
           (encoding_option stringSet_encoding pk.p_generators);
+        option "menhir"
+          ~comment:
+            [ "menhir options for the package";
+              "Example:";
+              {|version = "2.0"|};
+              {|parser = { modules = ["parser"]; tokens = "Tokens" }|};
+              {|tokens = { modules = ["tokens"]}|};
+            ]
+          (encoding_option menhir_encoding pk.p_menhir);
         option "pack-modules"
           ~comment:
             [ "whether all modules should be packed/wrapped (default is true)" ]
@@ -366,6 +434,9 @@ let of_toml ?default ?p_file table =
     EzToml.get_encoding_option stringSet_encoding table [ "generators" ]
       ?default:default.p_generators
   in
+  let p_menhir =
+    EzToml.get_encoding_option menhir_encoding table [ "menhir" ]
+  in
   let p_fields =
     EzToml.get_encoding_default fields_encoding table [ "fields" ]
       StringMap.empty
@@ -400,6 +471,7 @@ let of_toml ?default ?p_file table =
     p_fields;
     p_skeleton;
     p_generators;
+    p_menhir;
     p_skip;
     p_optional
   }

src/drom_lib/project.ml

@@ -531,6 +531,28 @@ let project_of_toml ?file ?default table =
         generators := StringSet.union !generators p_generators )
     packages;
   let generators = !generators in
+  let menhir_version =
+    List.fold_left
+      (fun acc p ->
+         match p.p_menhir with
+         | None -> acc
+         | Some { version; _ } ->
+             match acc with
+             | None ->
+                 begin try Scanf.sscanf version "%d.%d" (fun _ _ -> ())
+                   with Scanf.Scan_failure s ->
+                     Error.raise "In package %s, invalid menhir version: %s (error: %s)"
+                       p.name version s
+                 end;
+                 Some version
+             | Some acc_version ->
+                 if version <> acc_version then
+                   Error.raise "In package %s, menhir version is different from other packages \
+                                got %s when expecting %s" p.name version acc_version;
+                 acc)
+      None
+      packages
+  in
   let year = EzToml.get_int_default table [ project_key; "year" ] d.year in
 
   (* Check dune specification consistency :
@@ -637,6 +659,7 @@ let project_of_toml ?file ?default table =
       profile;
       fields;
       generators;
+      menhir_version;
       year;
       dune_version;
       project_create;