feat: support reproducible compilation and install

.gitignore

@@ -8,4 +8,7 @@ docs/build
 .venv
 benchmarks/results
 **/target/
-**/dist/
+**/dist/
+
+bindings/python/build
+bindings/python/src/outlines_core

README.md

@@ -11,5 +11,28 @@
 - dev build of python bindings `cd bindings/python && maturin develop`. If you have the conda `outlines-dev` environment activated, the outlines-core module is installed within the env automatically
 
 There's also a [justfile](https://github.com/casey/just) for running these easier:
+
 - `just dev-core`
 - `just dev-python`
+
+# Developer Notes
+
+Setup a virtual environment
+
+```bash
+uv venv
+source .venv/bin/activate
+```
+
+install the python bindings with
+
+```bash
+uv pip install bindings/python
+```
+
+# Testing
+
+```bash
+python -c "import outlines_core._lib;print(dir(outlines_core._lib))"
+python -c "import outlines_core._lib;print(outlines_core._lib.show_me_the_flag())"
+```

bindings/python/Cargo.toml

@@ -5,8 +5,6 @@ edition = "2021"
 
 [dependencies]
 pyo3 = "0.22.0"
-# outlines-core = { path = "../../outlines-core" }
-
 [profile.release-lto]
 inherits = "release"
 lto = true
@@ -15,3 +13,6 @@ lto = true
 name = "_lib"
 crate-type = ["cdylib"]
 path = "rust/lib.rs"
+
+[dependencies.outlines-core-rs]
+path = "../../outlines-core"

bindings/python/Manifest.in

@@ -4,3 +4,6 @@ graft tests
 include Cargo.toml
 global-exclude */__pycache__/*
 global-exclude *.pyc
+
+recursive-include rust *
+recursive-include outlines-core *

bindings/python/pyproject.toml

@@ -1,14 +1,17 @@
 [build-system]
-requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2", "setuptools-rust"]
+requires = ["setuptools", "wheel", "setuptools-rust"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "outlines_core"
-authors= [{name = "Outlines Developers"}]
+authors = [
+    { name = "Outlines Developers" },
+    { name = "Huggingface Developers" },
+]
 description = "Structured Text Generation in Rust"
 requires-python = ">=3.8"
-license = {text = "Apache-2.0"}
-keywords=[
+license = { text = "Apache-2.0" }
+keywords = [
     "machine learning",
     "deep learning",
     "language models",
@@ -24,17 +27,17 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
 dependencies = [
-   "interegular",
-   "numpy<2.0.0",
-   "cloudpickle",
-   "diskcache",
-   "pydantic>=2.0",
-   "numba",
-   "referencing",
-   "jsonschema",
-   "tqdm",
-   "datasets",
-   "typing_extensions",
+    "interegular",
+    "numpy<2.0.0",
+    "cloudpickle",
+    "diskcache",
+    "pydantic>=2.0",
+    "numba",
+    "referencing",
+    "jsonschema",
+    "tqdm",
+    "datasets",
+    "typing_extensions",
 ]
 dynamic = ["version"]
 
@@ -61,18 +64,14 @@ documentation = "https://outlines-dev.github.io/outlines-core/"
 repository = "https://github.com/outlines-dev/outlines-core/"
 
 [project.readme]
-file="README.md"
+file = "README.md"
 content-type = "text/markdown"
 
-[tool.setuptools]
-packages = ["outlines_core"]
-package-dir = {"" = "src"}
-
 [tool.setuptools.package-data]
 "outlines" = ["py.typed"]
 
 [tool.setuptools_scm]
-write_to = "src/outlines_core/_version.py"
+write_to = "py_src/outlines_core/_version.py"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
@@ -87,7 +86,7 @@ filterwarnings = [
 ]
 
 [tool.mypy]
-exclude=["examples"]
+exclude = ["examples"]
 enable_incomplete_feature = ["Unpack"]
 
 [[tool.mypy.overrides]]
@@ -109,26 +108,17 @@ module = [
 ignore_missing_imports = true
 
 [tool.coverage.run]
-omit = [
-    "src/outlines_core/_version.py",
-    "tests/*",
-]
+omit = ["py_src/outlines_core/_version.py", "tests/*"]
 branch = true
 
 [tool.coverage.report]
-omit = [
-    "tests/*",
-]
-exclude_lines = [
-    "pragma: no cover",
-    "if TYPE_CHECKING:",
-    "\\.\\.\\.",
-]
+omit = ["tests/*"]
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "\\.\\.\\."]
 show_missing = true
 
 [tool.diff_cover]
 compare_branch = "origin/main"
 diff_range_notation = ".."
 
 [[tool.setuptools-rust.ext-modules]]
-target = "outlines_core._lib"  # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package.
+target = "outlines_core._lib" # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package.

bindings/python/rust/lib.rs

@@ -4,16 +4,26 @@
 // use pyo3::types::{PyAnyMethods, PyDict, PyModule, PyModuleMethods, PySet};
 use pyo3::{pyfunction, pymodule, wrap_pyfunction, Bound, PyResult};
 
-
 #[pymodule]
 mod _lib {
+    use outlines_core_rs::FLAG;
     use pyo3::prelude::*;
 
     /// Formats the sum of two numbers as string.
     #[pyfunction]
     fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
         Ok((a + b).to_string())
     }
+
+    #[pyfunction]
+    fn show_me_the_flag() -> PyResult<String> {
+        Ok(FLAG.to_string())
+    }
+
+    #[pyfunction]
+    fn anotherone() -> PyResult<String> {
+        Ok("This is another one".to_string())
+    }
 }
 
 // #[pymodule]

outlines-core/src/lib.rs

@@ -3,6 +3,8 @@ use std::collections::BTreeSet;
 use std::sync::Arc;
 use std::thread;
 
+pub const FLAG: bool = true;
+
 pub fn create_fsm_index_end_to_end_rust(
     fsm_transitions: &BTreeMap<(i32, i32), i32>,
     alphabet_symbol_mapping: &BTreeMap<char, i32>,