diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 816919d96..eed34db6c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -6,10 +6,14 @@ on: - main paths-ignore: - "docs/**" + - "mkdocs.yml" + - "*.md" workflow_dispatch: pull_request: paths-ignore: - "docs/**" + - "mkdocs.yml" + - "*.md" name: Rust diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md new file mode 100644 index 000000000..601e9a865 --- /dev/null +++ b/docs/design_proposals/multi_environment_proposal.md @@ -0,0 +1,347 @@ +# Proposal Design: Multi Environment Support +## Objective +The aim is to introduce an environment set mechanism in the `pixi` package manager. +This mechanism will enable clear, conflict-free management of dependencies tailored to specific environments, while also maintaining the integrity of fixed lockfiles. + + +### Motivating Example +There are multiple scenarios where multiple environments are useful. + +- **Testing of multiple package versions**, e.g. `py39` and `py310` or polars `0.12` and `0.13`. +- **Smaller single tool environments**, e.g. `lint` or `docs`. +- **Large developer environments**, that combine all the smaller environments, e.g. `dev`. +- **Strict supersets of environments**, e.g. `prod` and `test-prod` where `test-prod` is a strict superset of `prod`. +- **Multiple machines from one project**, e.g. a `cuda` environment and a `cpu` environment. +- **And many more.** (Feel free to edit this document in our GitHub and add your use case.) + +This prepares `pixi` for use in large projects with multiple use-cases, multiple developers and different CI needs. + +## Design Considerations +1. **User-friendliness**: Pixi is a user focussed tool that goes beyond developers. The feature should have good error reporting and helpful documentation from the start. +2. **Keep it simple**: Not understanding the multiple environments feature shouldn't limit a user to use pixi. The feature should be "invisible" to the non-multi env use-cases. +3. **No Automatic Combinatorial**: To ensure the dependency resolution process remains manageable, the solution should avoid a combinatorial explosion of dependency sets. By making the environments user defined and not automatically inferred by testing a matrix of the features. +4. **Single environment Activation**: The design should allow only one environment to be active at any given time, simplifying the resolution process and preventing conflicts. +5. **Fixed Lockfiles**: It's crucial to preserve fixed lockfiles for consistency and predictability. Solutions must ensure reliability not just for authors but also for end-users, particularly at the time of lockfile creation. + +## Proposed Solution +!!! important + This is a proposal, not a final design. The proposal is open for discussion and will be updated based on the feedback. + +### Feature & Environment Set Definitions +Introduce environment sets into the `pixi.toml` this describes environments based on `feature`'s. Introduce features into the `pixi.toml` that can describe parts of environments. +As an environment goes beyond just `dependencies` the `features` should be described including the following fields: + +- `dependencies`: The conda package dependencies +- `pypi-dependencies`: The pypi package dependencies +- `system-requirements`: The system requirements of the environment +- `activation`: The activation information for the environment +- `platforms`: The platforms the environment can be run on. +- `channels`: The channels used to create the environment. Adding the `priority` field to the channels to allow concatenation of channels instead of overwriting. +- `target`: All the above features but also separated by targets. +- `tasks`: Feature specific tasks, tasks in one environment are selected as default tasks for the environment. + + +```toml title="Default features" linenums="1" +[dependencies] # short for [feature.default.dependencies] +python = "*" +numpy = "==2.3" + +[pypi-dependencies] # short for [feature.default.pypi-dependencies] +pandas = "*" + +[system-requirements] # short for [feature.default.system-requirements] +libc = "2.33" + +[activation] # short for [feature.default.activation] +scripts = ["activate.sh"] +``` + +```toml title="Different dependencies per feature" linenums="1" +[feature.py39.dependencies] +python = "~=3.9.0" +[feature.py310.dependencies] +python = "~=3.10.0" +[feature.test.dependencies] +pytest = "*" +``` + +```toml title="Full set of environment modification in one feature" linenums="1" +[feature.cuda] +dependencies = {cuda = "x.y.z", cudnn = "12.0"} +pypi-dependencies = {torch = "1.9.0"} +platforms = ["linux-64", "osx-arm64"] +activation = {scripts = ["cuda_activation.sh"]} +system-requirements = {cuda = "12"} +# Channels concatenate using a priority instead of overwrite, so the default channels are still used. +# Using the priority the concatenation is controlled, default is 0, so the default channels are used first. +channels = [{name = "nvidia", priority = "-1"}, "pytorch"] # Results in: ["nvidia", "conda-forge", "pytorch"] if the default is `conda-forge` +tasks = { warmup = "python warmup.py" } +target.osx-arm64 = {dependencies = {mlx = "x.y.z"}} +``` + + +```toml title="Define tasks as defaults of an environment" linenums="1" +[feature.test.tasks] +test = "pytest" + +[environments] +test = ["test"] + +# `pixi run test` == `pixi run --environments test test` +``` + +The environment definition should contain the following fields: + +- `features: Vec`: The features that are included in the environment set, which is also the default field in the environments. +- `solve-group: String`: The solve group is used to group environments together at the solve stage. +This is useful for environments that need to have the same dependencies but might extend them with additional dependencies. +For instance when testing a production environment with additional test dependencies. + +```toml title="Creating environments from features" linenums="1" +[environments] +# implicit: default = ["default"] +default = ["py39"] # implicit: default = ["py39", "default"] +py310 = ["py310"] # implicit: py310 = ["py310", "default"] +test = ["test"] # implicit: test = ["test", "default"] +test39 = ["test", "py39"] # implicit: test39 = ["test", "py39", "default"] +``` + +```toml title="Testing a production environment with additional dependencies" linenums="1" +[environments] +# Creating a `prod` environment which is the minimal set of dependencies used for production. +prod = {features = ["py39"], solve-group = "prod"} +# Creating a `test_prod` environment which is the `prod` environment plus the `test` feature. +test_prod = {features = ["py39", "test"], solve-group = "prod"} +# Using the `solve-group` to solve the `prod` and `test_prod` environments together +# Which makes sure the tested environment has the same version of the dependencies as the production environment. +``` + +```toml title="Creating environments without a default environment" linenums="1" +[dependencies] +# Keep empty or undefined to create an empty environment. + +[feature.base.dependencies] +python = "*" + +[feature.lint.dependencies] +pre-commit = "*" + +[environments] +# Create a custom default +default = ["base"] +# Create a custom environment which only has the `lint` feature as the default feature is empty. +lint = ["lint"] +``` + +### Lockfile Structure +Within the `pixi.lock` file, a package may now include an additional `environments` field, specifying the environment to which it belongs. +To avoid duplication the packages `environments` field may contain multiple environments so the lockfile is of minimal size. +```yaml +- platform: linux-64 + name: pre-commit + version: 3.3.3 + category: main + environments: + - dev + - test + - lint + ...: +- platform: linux-64 + name: python + version: 3.9.3 + category: main + environments: + - dev + - test + - lint + - py39 + - default + ...: +``` + + +### User Interface Environment Activation +Users can manually activate the desired environment via command line or configuration. +This approach guarantees a conflict-free environment by allowing only one feature set to be active at a time. +For the user the cli would look like this: + +```shell title="Default behavior" +pixi run python +# Runs python in the `default` environment +``` + +```shell title="Activating an specific environment" +pixi run -e test pytest +pixi run --environment test pytest +# Runs `pytest` in the `test` environment +``` + +```shell title="Activating a shell in an environment" +pixi shell -e cuda +pixi shell --environment cuda +# Starts a shell in the `cuda` environment +``` +```shell title="Running any command in an environment" +pixi run -e test any_command +# Runs any_command in the `test` environment which doesn't require to be predefined as a task. +``` + +```shell title="Interactive selection of environments if task is in multiple environments" +# In the scenario where test is a task in multiple environments, interactive selection should be used. +pixi run test +# Which env? +# 1. test +# 2. test39 +``` + +## Important links +- Initial writeup of the proposal: https://gist.github.com/0xbe7a/bbf8a323409be466fe1ad77aa6dd5428 +- GitHub project: https://github.com/orgs/prefix-dev/projects/10 + +## Real world example use cases +??? tip "Polarify test setup" + In `polarify` they want to test multiple versions combined with multiple versions of polars. + This is currently done by using a matrix in GitHub actions. + This can be replaced by using multiple environments. + + ```toml title="pixi.toml" + [project] + name = "polarify" + # ... + channels = ["conda-forge"] + platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] + + [tasks] + postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version-check -e ." + + [dependencies] + python = ">=3.9" + pip = "*" + polars = ">=0.14.24,<0.21" + + [feature.py39.dependencies] + python = "3.9.*" + [feature.py310.dependencies] + python = "3.10.*" + [feature.py311.dependencies] + python = "3.11.*" + [feature.py312.dependencies] + python = "3.12.*" + [feature.pl017.dependencies] + polars = "0.17.*" + [feature.pl018.dependencies] + polars = "0.18.*" + [feature.pl019.dependencies] + polars = "0.19.*" + [feature.pl020.dependencies] + polars = "0.20.*" + + [feature.test.dependencies] + pytest = "*" + pytest-md = "*" + pytest-emoji = "*" + hypothesis = "*" + [feature.test.tasks] + test = "pytest" + + [feature.lint.dependencies] + pre-commit = "*" + [feature.lint.tasks] + lint = "pre-commit run --all" + + [environments] + pl017 = ["pl017", "py39", "test"] + pl018 = ["pl018", "py39", "test"] + pl019 = ["pl019", "py39", "test"] + pl020 = ["pl020", "py39", "test"] + py39 = ["py39", "test"] + py310 = ["py310", "test"] + py311 = ["py311", "test"] + py312 = ["py312", "test"] + ``` + + ```yaml title=".github/workflows/test.yml" + jobs: + tests: + name: Test ${{ matrix.environment }} + runs-on: ubuntu-latest + strategy: + matrix: + environment: + - pl017 + - pl018 + - pl019 + - pl020 + - py39 + - py310 + - py311 + - py312 + steps: + - uses: actions/checkout@v4 + - uses: prefix-dev/setup-pixi@v0.5.0 + with: + # already installs the corresponding environment and caches it + environments: ${{ matrix.environment }} + - name: Install dependencies + run: | + pixi run --env ${{ matrix.environment }} postinstall + pixi run --env ${{ matrix.environment }} test + ``` +??? tip "Test vs Production example" + This is an example of a project that has a `test` feature and `prod` environment. + The `prod` environment is a production environment that contains the run dependencies. + The `test` feature is a set of dependencies and tasks that we want to put on top of the previously solved `prod` environment. + This is a common use case where we want to test the production environment with additional dependencies. + + ```toml title="pixi.toml" + [project] + name = "my-app" + # ... + channels = ["conda-forge"] + platforms = ["osx-arm64", "linux-64"] + + [tasks] + postinstall-e = "pip install --no-build-isolation --no-deps --disable-pip-version-check -e ." + postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version-check ." + dev = "uvicorn my_app.app:main --reload" + serve = "uvicorn my_app.app:main" + + [dependencies] + python = ">=3.12" + pip = "*" + pydantic = ">=2" + fastapi = ">=0.105.0" + sqlalchemy = ">=2,<3" + uvicorn = "*" + aiofiles = "*" + + [feature.test.dependencies] + pytest = "*" + pytest-md = "*" + pytest-asyncio = "*" + [feature.test.tasks] + test = "pytest --md=report.md" + + [environments] + # both default and prod will have exactly the same dependency versions when they share a dependency + default = {features = ["test"], solve-group = "prod-group"} + prod = {features = [], solve-group = "prod-group"} + ``` + In ci you would run the following commands: + ```shell + pixi run postinstall-e && pixi run test + ``` + Locally you would run the following command: + ```shell + pixi run postinstall-e && pixi run dev + ``` + + Then in a Dockerfile you would run the following command: + ```dockerfile title="Dockerfile" + FROM ghcr.io/prefix-dev/pixi:latest # this doesn't exist yet + WORKDIR /app + COPY . . + RUN pixi run --env prod postinstall + EXPOSE 8080 + CMD ["/usr/local/bin/pixi", "run", "--env", "prod", "serve"] + ``` diff --git a/mkdocs.yml b/mkdocs.yml index d5b5f05ea..9a3c41c0f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,8 @@ nav: - C++/Cmake: examples/cpp-sdl.md - OpenCV: examples/opencv.md - ROS2: examples/ros2-nav2.md + - Design Proposals: + - Multi Env: design_proposals/multi_environment_proposal.md - Community: Community.md - FAQ: FAQ.md