Rewrite codegen in Rust (#164)

This completely removes the dependency on Python, and especially the Python protobuf package.

pb-jelly-gen now depends on code that it generated for the various descriptors used by protoc as well as `rust/extensions.proto`. These are bundled into the crate using a newly added "single file" output mode to avoid needing to publish separate crates. The generated code is also checked in for bootstrapping.

For now this is mostly a 1:1 straight port of the original Python code. Boxing recursive fields was changed to use a side set as it's hard to mutate the original protos in-place (and pb-jelly support for extensions is read-only anyway).

Using pb-jelly-gen as a library no longer needs to create virtualenvs or run the plugin separately; instead it uses protoc's support for dumping parsed protos into a FileDescriptorSet and invokes the codegen "plugin" directly. A binary plugin target is still provided for direct use with protoc though.

.github/workflows/main.yml

@@ -17,27 +17,6 @@ jobs:
       - name: Run rustfmt
         run: rustup run nightly ci/run_rustfmt.sh
 
-  black:
-    name: Run black formatter on codegen.py
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-python@v2
-      - uses: actions/checkout@v2
-      - run: ci/run_black.sh
-
-  mypy:
-    name: Run mypy on codegen.py
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: ${{env.PROTOBUF_VER}}
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/setup-python@v2
-      - uses: actions/checkout@v2
-      - run: ci/run_mypy.sh
-
   pb-jelly-unit:
     name: pb-jelly unit tests
     runs-on: ubuntu-latest
@@ -68,7 +47,6 @@ jobs:
     runs-on: ${{matrix.plat}}
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
       - run: rustup update nightly
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
@@ -89,7 +67,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
       - run: rustup update nightly
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
@@ -108,7 +85,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
       - run: rustup update nightly
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
@@ -127,7 +103,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
       - run: rustup update nightly
       - run: rustup component add clippy --toolchain nightly
       - name: Install Protoc
@@ -143,3 +118,21 @@ jobs:
           rustup run nightly cargo run
           cd ..
           rustup run nightly cargo clippy -p proto_pbtest
+
+  gen_gen:
+    name: Generate pb-jelly-gen/src/protos.rs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: rustup update nightly
+      - run: rustup component add clippy --toolchain nightly
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          version: ${{env.PROTOBUF_VER}}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate protos
+        run: |
+          cd pb-jelly-gen
+          rustup run nightly bash regen_gen_protos.sh
+          git diff --exit-code

.gitignore

@@ -7,7 +7,3 @@ generated/
 .DS_Store
 __pycache__/
 *~
-
-pb-jelly-gen/codegen/.black_venv
-pb-jelly-gen/codegen/.mypy_venv
-pb-jelly-gen/codegen/proto/rust/extensions_pb2.pyi

.rustfmt.toml

@@ -22,3 +22,6 @@ match_block_trailing_comma = true
 # We prefer it already so just do it here
 use_try_shorthand = true
 use_field_init_shorthand = true
+
+imports_granularity = "Module"
+group_imports = "StdExternalCrate"

README.md

@@ -68,7 +68,7 @@ pb-jelly = "0.0.16"
 ##### `pb-jelly-gen`
 
 A framework for generating Rust structs and implementations for `.proto` files.
-In order to use pb-jelly, you need to add the pb-jelly-gen/codegen/codegen.py as a plugin to your protoc invocation.
+In order to use pb-jelly, you need to add the pb-jelly-gen as a plugin to your protoc invocation.
 
 We added some code here to handle the protoc invocation if you choose to use it.
 You'll need to add a generation crate (see `examples_gen` for an example)
@@ -86,18 +86,17 @@ Note that you can always invoke protoc on your own (for example if you are alrea
 with `--rust_out=codegen.py` as a plugin for rust.
 
 ### Generating Rust Code
-1. Install `protoc` - The protobuf compiler, this can be downloaded or built from source [`protobuf`](https://github.com/protocolbuffers/protobuf) or installed (mac) via `brew install protobuf`.
-2. `python3` - The codegen plugin used with `protoc` is written in Python3.
+1. Install `protoc`, the protobuf compiler.
+  - See [the upstream project](https://github.com/protocolbuffers/protobuf). Precompiled binaries can be found at their [releases page](https://github.com/protocolbuffers/protobuf/releases).
+  - On macOS, `protoc` can be installed via Homebrew: `brew install protobuf`.
 
 #### To generate with pb-jelly-gen
 3. Create an inner (build-step) crate which depends on pb-jelly-gen. [Example](https://github.com/dropbox/pb-jelly/tree/main/examples/examples_gen)
 4. `cargo run` in the directory of the inner generation crate
 
 #### To generate manually with protoc
-3. Create venv [optional] `python3 -m venv .pb_jelly_venv ; source .pb_jelly_venv/bin/activate`
-4. [Recommended] `python3 -m pip install protobuf==[same_version_as_your protoc]`
-5. Install `python3 -m pip install -e pb-jelly-gen/codegen`  (installs protoc-gen-rust into the venv)
-6. `protoc --rust_out=generated/ input.proto`
+1. `cargo build` in `pb-jelly-gen`
+2. `protoc --plugin=protoc-gen-jellyrust=pb-jelly-gen/target/debug/protoc-gen-jellyrust --jellyrust_out=generated/ input.proto`
 
 ## Example
 
@@ -158,8 +157,7 @@ Service Generation
     - protoc - part of Google's [protobuf tools](https://github.com/protocolbuffers/protobuf/)
         - macos: `brew install protobuf`
         - Linux (Fedora/CentOS/RHEL): `dnf install protobuf protobuf-devel`
-    - Install Python
-        - [if necessary] macos: `brew install python3`
+        - Linux (Ubuntu): `apt install protobuf-compiler`
 3. **pb-jelly** currently uses an experimental test framework that requires a nightly build of rust.
 	-  `rustup default nightly`
 4. `cd pb-test`

examples/examples_gen/Cargo.toml

@@ -8,3 +8,6 @@ publish = false
 [dependencies]
 #pb-jelly-gen = "0.0.16"  # If copying this example - use this
 pb-jelly-gen = { path = "../../pb-jelly-gen" }
+
+[patch.crates-io]
+pb-jelly = { path = "../../pb-jelly" }

examples/examples_gen/src/main.rs

@@ -6,7 +6,8 @@ fn main() -> std::io::Result<()> {
         .src_path("../protos")
         .include_path("../includes")
         .cleanup_out_path(true)
-        .gen_protos();
+        .gen_protos()
+        .expect("Failed to generate protos");
 
     Ok(())
 }

examples/src/zero_copy/main.rs

@@ -1,10 +1,11 @@
+use std::io::Cursor;
+
 use bytes::Bytes;
 use pb_jelly::{
     Lazy,
     Message,
 };
 use proto_zero_copy::basic::BytesMessage;
-use std::io::Cursor;
 
 fn main() -> std::io::Result<()> {
     // Create 1kb of Data

pb-jelly-gen/Cargo.toml

@@ -14,6 +14,14 @@ categories = ["encoding", "parsing", "web-programming"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-include_dir = "0.6"
 tempfile = "3.1.0"
 walkdir = "2"
+
+pb-jelly = { version = "0.0.16" }
+regex = "1.10.2"
+lazy_static = "1.4.0"
+indexmap = "2.0.2"
+
+# Override pb-jelly dependency for generated crates as well
+[patch.crates-io]
+pb-jelly = { path = "../pb-jelly" }

pb-jelly-gen/README.md

@@ -6,27 +6,35 @@ This crate provides a tool to generate [`Rust`](https://www.rust-lang.org/) code
 
 ### How To Use
 
-##### `python` + `protoc`
-The core of this crate is a python script `codegen.py` that is provided to the protobuf compiler, `protoc` as a plugin.
-
 You'll need the protobuf compiler which you can get by:
 1. Running `brew install protobuf` or...
 2. Download or build from source [`protobuf`](https://github.com/protocolbuffers/protobuf)
 
-Once you've completed the above steps, you should include this crate as a build-dependency in your `Cargo.toml` and then call the API of this crate from a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) files in the root of your repo.
+#### As a plugin for protoc
+
+A binary is included that can be passed directly to `protoc`:
+
+```
+% cargo build --bin protoc-gen-jellyrust
+% protoc --plugin=protoc-gen-jellyrust=target/debug/protoc-gen-jellyrust --jellyrust_out=out foo/bar.proto...
+```
+
+#### As a library
+
+Add this crate as a dependency in your `Cargo.toml` and then call `gen_protos`:
 
 ##### `Cargo.toml`
 ```
-[build-dependencies]
+[dependencies]
 pb-jelly-gen = "0.0.16"
 ```
 
-##### `build.rs`
+##### `main.rs`
 ```
 use pb_jelly_gen::gen_protos;
 
-fn main() -> std::io::Result<()> {
+fn main() {
     // Replace `./protos` with a path to your proto files.
-    gen_protos(vec!["./protos"])
+    gen_protos(vec!["./protos"]).unwrap()
 }
 ```