From 5178c8f6276b24e811fecf07bee53ba65cdcbd53 Mon Sep 17 00:00:00 2001 From: drbh Date: Fri, 16 Aug 2024 13:43:16 -0400 Subject: [PATCH] feat: support reproducible compilation and install --- .gitignore | 5 ++- Cargo.lock | 1 + README.md | 23 +++++++++++++ bindings/python/Cargo.toml | 5 +-- bindings/python/Manifest.in | 3 ++ bindings/python/pyproject.toml | 60 ++++++++++++++-------------------- bindings/python/rust/lib.rs | 12 ++++++- outlines-core/src/lib.rs | 2 ++ 8 files changed, 72 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 00cae6e6..f3005168 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,7 @@ docs/build .venv benchmarks/results **/target/ -**/dist/ \ No newline at end of file +**/dist/ + +bindings/python/build +bindings/python/src/outlines_core \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index d38b8c72..298457c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,6 +133,7 @@ dependencies = [ name = "python-bindings" version = "0.1.0" dependencies = [ + "outlines-core-rs", "pyo3", ] diff --git a/README.md b/README.md index 18f13af6..4141d637 100644 --- a/README.md +++ b/README.md @@ -11,5 +11,28 @@ - dev build of python bindings `cd bindings/python && maturin develop`. If you have the conda `outlines-dev` environment activated, the outlines-core module is installed within the env automatically There's also a [justfile](https://github.com/casey/just) for running these easier: + - `just dev-core` - `just dev-python` + +# Developer Notes + +Setup a virtual environment + +```bash +uv venv +source .venv/bin/activate +``` + +install the python bindings with + +```bash +uv pip install bindings/python +``` + +# Testing + +```bash +python -c "import outlines_core._lib;print(dir(outlines_core._lib))" +python -c "import outlines_core._lib;print(outlines_core._lib.show_me_the_flag())" +``` diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index da2038be..1c936ccf 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -5,8 +5,6 @@ edition = "2021" [dependencies] pyo3 = "0.22.0" -# outlines-core = { path = "../../outlines-core" } - [profile.release-lto] inherits = "release" lto = true @@ -15,3 +13,6 @@ lto = true name = "_lib" crate-type = ["cdylib"] path = "rust/lib.rs" + +[dependencies.outlines-core-rs] +path = "../../outlines-core" diff --git a/bindings/python/Manifest.in b/bindings/python/Manifest.in index 855d922d..a4111c30 100644 --- a/bindings/python/Manifest.in +++ b/bindings/python/Manifest.in @@ -4,3 +4,6 @@ graft tests include Cargo.toml global-exclude */__pycache__/* global-exclude *.pyc + +recursive-include rust * +recursive-include outlines-core * diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 65a21838..d6b5c1d5 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,14 +1,17 @@ [build-system] -requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2", "setuptools-rust"] +requires = ["setuptools", "wheel", "setuptools-rust"] build-backend = "setuptools.build_meta" [project] name = "outlines_core" -authors= [{name = "Outlines Developers"}] +authors = [ + { name = "Outlines Developers" }, + { name = "Huggingface Developers" }, +] description = "Structured Text Generation in Rust" requires-python = ">=3.8" -license = {text = "Apache-2.0"} -keywords=[ +license = { text = "Apache-2.0" } +keywords = [ "machine learning", "deep learning", "language models", @@ -24,17 +27,17 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "interegular", - "numpy<2.0.0", - "cloudpickle", - "diskcache", - "pydantic>=2.0", - "numba", - "referencing", - "jsonschema", - "tqdm", - "datasets", - "typing_extensions", + "interegular", + "numpy<2.0.0", + "cloudpickle", + "diskcache", + "pydantic>=2.0", + "numba", + "referencing", + "jsonschema", + "tqdm", + "datasets", + "typing_extensions", ] dynamic = ["version"] @@ -61,18 +64,14 @@ documentation = "https://outlines-dev.github.io/outlines-core/" repository = "https://github.com/outlines-dev/outlines-core/" [project.readme] -file="README.md" +file = "README.md" content-type = "text/markdown" -[tool.setuptools] -packages = ["outlines_core"] -package-dir = {"" = "src"} - [tool.setuptools.package-data] "outlines" = ["py.typed"] [tool.setuptools_scm] -write_to = "src/outlines_core/_version.py" +write_to = "py_src/outlines_core/_version.py" [tool.pytest.ini_options] testpaths = ["tests"] @@ -87,7 +86,7 @@ filterwarnings = [ ] [tool.mypy] -exclude=["examples"] +exclude = ["examples"] enable_incomplete_feature = ["Unpack"] [[tool.mypy.overrides]] @@ -109,21 +108,12 @@ module = [ ignore_missing_imports = true [tool.coverage.run] -omit = [ - "src/outlines_core/_version.py", - "tests/*", -] +omit = ["py_src/outlines_core/_version.py", "tests/*"] branch = true [tool.coverage.report] -omit = [ - "tests/*", -] -exclude_lines = [ - "pragma: no cover", - "if TYPE_CHECKING:", - "\\.\\.\\.", -] +omit = ["tests/*"] +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "\\.\\.\\."] show_missing = true [tool.diff_cover] @@ -131,4 +121,4 @@ compare_branch = "origin/main" diff_range_notation = ".." [[tool.setuptools-rust.ext-modules]] -target = "outlines_core._lib" # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package. +target = "outlines_core._lib" # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package. diff --git a/bindings/python/rust/lib.rs b/bindings/python/rust/lib.rs index 69ad5356..cba0a30f 100644 --- a/bindings/python/rust/lib.rs +++ b/bindings/python/rust/lib.rs @@ -4,9 +4,9 @@ // use pyo3::types::{PyAnyMethods, PyDict, PyModule, PyModuleMethods, PySet}; use pyo3::{pyfunction, pymodule, wrap_pyfunction, Bound, PyResult}; - #[pymodule] mod _lib { + use outlines_core_rs::FLAG; use pyo3::prelude::*; /// Formats the sum of two numbers as string. @@ -14,6 +14,16 @@ mod _lib { fn sum_as_string(a: usize, b: usize) -> PyResult { Ok((a + b).to_string()) } + + #[pyfunction] + fn show_me_the_flag() -> PyResult { + Ok(FLAG.to_string()) + } + + #[pyfunction] + fn anotherone() -> PyResult { + Ok("This is another one".to_string()) + } } // #[pymodule] diff --git a/outlines-core/src/lib.rs b/outlines-core/src/lib.rs index 14e3dbb2..a81aa77f 100644 --- a/outlines-core/src/lib.rs +++ b/outlines-core/src/lib.rs @@ -3,6 +3,8 @@ use std::collections::BTreeSet; use std::sync::Arc; use std::thread; +pub const FLAG: bool = true; + pub fn create_fsm_index_end_to_end_rust( fsm_transitions: &BTreeMap<(i32, i32), i32>, alphabet_symbol_mapping: &BTreeMap,