From f78f562497d90d494d715add6d026d84fba0cfcb Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 11:37:04 +0100 Subject: [PATCH 01/18] docs: add multi env proposal --- .../multi_environment_proposal.md | 182 ++++++++++++++++++ mkdocs.yml | 2 + 2 files changed, 184 insertions(+) create mode 100644 docs/design_proposals/multi_environment_proposal.md diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md new file mode 100644 index 000000000..ad344ba77 --- /dev/null +++ b/docs/design_proposals/multi_environment_proposal.md @@ -0,0 +1,182 @@ +# Proposal Design: Multi Environment Support +## Objective +The aim is to introduce an environment set mechanism in the `pixi` package manager. +This mechanism will enable clear, conflict-free management of dependencies tailored to specific environments, while also maintaining the integrity of fixed lockfiles. + + +### Motivating Example +There are multiple scenarios where multiple environments are useful. + +- **Testing of multiple package versions**, e.g. `py39` and `py310` or polars `0.12` and `0.13`. +- **Smaller single tool environments**, e.g. `lint` or `docs`. +- **Large developer environments**, that combine all the smaller environments, e.g. `dev`. +- **Strict supersets of environments**, e.g. `test` and `test39` where `test39` is a strict superset of `test`. +- **Multiple machines from one project**, e.g. a `cuda` environment and a `cpu` environment. +- **And many more.** (If you have a use-case please add it to the list, so we can make sure it's covered) + +This prepares `pixi` for the use in large projects with multiple use-cases, multiple developers and different CI needs. + +## Design Considerations +1. **User-friendliness**: Pixi is a user focussed tool this goes beyond developers. The feature should have good error reporting and helpful documentation from the start. This is opinionated so the user sided PR's should be checked by multiple developers. +2. **Keep it simple**: Not understanding the multiple environments feature shouldn't limit a user to use pixi. The feature should be "invisible" to the non-multi env use-cases. +3. **No Automatic Combinatorial**: To ensure the dependency resolution process remains manageable, the solution should avoid a combinatorial explosion of dependency sets. By making the environments user defined and not automatically inferred by testing a matrix of the features. +4. **Single Feature Activation**: The design should allow only one feature set to be active at any given time, simplifying the resolution process and preventing conflicts. +5. **Fixed Lockfiles**: It's crucial to preserve fixed lockfiles for consistency and predictability. Solutions must ensure reliability not just for authors but also for end-users, particularly at the time of lockfile creation. + +## Proposed Solution +!!! important + This is a proposal, not a final design. The proposal is open for discussion and will be updated based on the feedback. + +### Feature & Environment Set Definitions +Introduce environment sets into the `pixi.toml` this describes environments based on `feature`'s. Introduce features into the `pixi.toml` that can describe parts of environments. +As an environment goes beyond just `dependencies` the `features` should be described including the following fields: +- `dependencies`: The conda package dependencies +- `pypi-dependencies`: The pypi package dependencies +- `system-requirements`: The system requirements of the environment +- `activation`: The activation information for the environment +- `platforms`: The platforms the environment can be run on. +- `channels`: The channels used to create the environment. +- `target`: All the above features but also separated by targets. +- `tasks`: Feature specific tasks, tasks in one environment are selected as default tasks for the environment. + + +```toml title="Default features" linenums="1" +[dependencies] # short for [feature.default.dependencies] +python = "*" +numpy = "==2.3" + +[pypi-dependencies] # short for [feature.default.pypi-dependencies] +pandas = "*" + +[system-requirements] # short for [feature.default.system-requirements] +libc = "2.33" + +[activation] # short for [feature.default.activation] +scripts = ["activate.sh"] +``` + +```toml title="Different dependencies per feature" linenums="1" +[feature.py39.dependencies] +python = "~=3.9.0" +[feature.py310.dependencies] +python = "~=3.10.0" +[feature.test.dependencies] +pytest = "*" +``` + +```toml title="Full set of environment modification in one feature" linenums="1" +[feature.cuda] +dependencies = {cuda = "x.y.z", cudnn = "12.0"} +pypi-dependencies = {torch = "1.9.0"} +platforms = ["linux-64", "osx-arm64"] +activation = {scripts = ["cuda_activation.sh"]} +channels = ["conda-forge", "nvidia"] # Would concat instead of overwrite +tasks = { warmup = "python warmup.py" } +target.osx-arm64 = {dependencies = {mlx = "x.y.z"}} +``` + + +```toml title="Define tasks as defaults of an environment" linenums="1" +[feature.test.tasks] +test = "pytest" + +[environments] +test = ["test"] + +# `pixi run test` == `pixi run --envrionment test test` +``` + +The environment definition should contain the following fields: + +- `features: Vec`: The features that are included in the environment set, which is also the default field in the environments. +- `environments: Vec`: The environments that are included in the environment set. When environments is used, the extra features are **on top** of the included environments. + Environments are used as a locked base, so the features added to an environment are not allowed to change the locked set. This should result in a failure if the locked set is not compatible with the added features. +- `default-features: bool`: Whether the default features should be included in the environment set. + +```toml +[environments] +# `default` environment is now the `default` feature plus the py39 feature +default = ["py39"] +# `lint` environment is now the `lint` feature without the `default` feature or environment +lint = {features = ["lint"], default-features = "false"} +# `dev` environment is now the `default` feature plus the `test` feature, which makes the `default` envriroment is solved without the use of the test feature. +dev = {environments = ["default"], features = ["test"]} +``` + +```toml title="Creating environments from features" linenums="1" +[environments] +# implicit: default = ["default"] +default = ["py39"] # implicit: default = ["py39", "default"] +py310 = ["py310"] # implicit: py310 = ["py310", "default"] +test = ["test"] # implicit: test = ["test", "default"] +test39 = ["test", "py39"] # implicit: test39 = ["test", "py39", "default"] +lint = {features = ["lint"], default-features = "false"} # no implicit default +``` + +```toml title="Creating environments from environments" linenums="1" +[environments] +prod = ["py39"] +# Takes the `prod` environment and adds the `test` feature to it without modifying the `prod` environment requirements, solve should fail if requirements don't comply with locked set. +test_prod = {environments = ["prod"], features = ["test"]} +``` + +### Lockfile Structure +Within the `pixi.lock` file, a package may now include an additional `environments` field, specifying the environment to which it belongs. +To avoid duplication the packages `environments` field may contain multiple environments so the lockfile is of minimal size. +```yaml +- platform: linux-64 + name: pre-commit + version: 3.3.3 + category: main + environments: + - dev + - test + - lint + ...: +- platform: linux-64 + name: python + version: 3.9.3 + category: main + environments: + - dev + - test + - lint + - py39 + - default + ...: +``` + + +### User Interface Environment Activation +Users can manually activate the desired environment via command line or configuration. +This approach guarantees a conflict-free environment by allowing only one feature set to be active at a time. +For the user the cli would look like this: + +```shell title="Default behavior" +pixi run python +# Runs python in the `default` environment +``` + +```shell title="Activating an specific environment" +pixi run -e test pytest +pixi run --environment test pytest +# Runs `pytest` in the `test` environment +``` + +```shell title="Activating a shell in an environment" +pixi shell -e cuda +pixi shell --environment cuda +# Starts a shell in the `cuda` environment +``` +```shell title="Running any command in an environment" +pixi run -e test any_command +# Runs any_command in the `test` environment which doesn't require to be predefined as a task. +``` + +```shell title="Interactive selection of environments if task is in multiple environments" +# In the scenario where test is a task in multiple environments, interactive selection should be used. +pixi run test +# Which env? +# 1. test +# 2. test39 +``` diff --git a/mkdocs.yml b/mkdocs.yml index d5b5f05ea..9a3c41c0f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,8 @@ nav: - C++/Cmake: examples/cpp-sdl.md - OpenCV: examples/opencv.md - ROS2: examples/ros2-nav2.md + - Design Proposals: + - Multi Env: design_proposals/multi_environment_proposal.md - Community: Community.md - FAQ: FAQ.md From 28718eaf0bc193aff14187148ad6b9f0713a414c Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 13:07:45 +0100 Subject: [PATCH 02/18] Update docs/design_proposals/multi_environment_proposal.md Co-authored-by: Pavel Zwerschke --- docs/design_proposals/multi_environment_proposal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index ad344ba77..611a09c9d 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -10,7 +10,7 @@ There are multiple scenarios where multiple environments are useful. - **Testing of multiple package versions**, e.g. `py39` and `py310` or polars `0.12` and `0.13`. - **Smaller single tool environments**, e.g. `lint` or `docs`. - **Large developer environments**, that combine all the smaller environments, e.g. `dev`. -- **Strict supersets of environments**, e.g. `test` and `test39` where `test39` is a strict superset of `test`. +- **Strict supersets of environments**, e.g. `prod` and `test-prod` where `test-prod` is a strict superset of `prod`. - **Multiple machines from one project**, e.g. a `cuda` environment and a `cpu` environment. - **And many more.** (If you have a use-case please add it to the list, so we can make sure it's covered) From da9aef9adbf9eb48dd790dcc256031386b213eb1 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 13:07:56 +0100 Subject: [PATCH 03/18] Update docs/design_proposals/multi_environment_proposal.md Co-authored-by: Pavel Zwerschke --- docs/design_proposals/multi_environment_proposal.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index 611a09c9d..e56199ba1 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -30,6 +30,7 @@ This prepares `pixi` for the use in large projects with multiple use-cases, mult ### Feature & Environment Set Definitions Introduce environment sets into the `pixi.toml` this describes environments based on `feature`'s. Introduce features into the `pixi.toml` that can describe parts of environments. As an environment goes beyond just `dependencies` the `features` should be described including the following fields: + - `dependencies`: The conda package dependencies - `pypi-dependencies`: The pypi package dependencies - `system-requirements`: The system requirements of the environment From 2f50198f719fe47503f625f636a90df12091e1aa Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 11:47:07 +0100 Subject: [PATCH 04/18] add links --- docs/design_proposals/multi_environment_proposal.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index e56199ba1..fb93569a8 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -181,3 +181,7 @@ pixi run test # 1. test # 2. test39 ``` + +## Important links: +- Initial writeup of the proposal: https://gist.github.com/0xbe7a/bbf8a323409be466fe1ad77aa6dd5428 +- GitHub project: https://github.com/orgs/prefix-dev/projects/10 From 781e05a9499432f179275817a69442ba14318791 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 13:10:35 +0100 Subject: [PATCH 05/18] clearify the channels in features --- docs/design_proposals/multi_environment_proposal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index fb93569a8..867897779 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -71,7 +71,7 @@ dependencies = {cuda = "x.y.z", cudnn = "12.0"} pypi-dependencies = {torch = "1.9.0"} platforms = ["linux-64", "osx-arm64"] activation = {scripts = ["cuda_activation.sh"]} -channels = ["conda-forge", "nvidia"] # Would concat instead of overwrite +channels = ["nvidia"] # Would concat instead of overwrite, so the default channels are still used. tasks = { warmup = "python warmup.py" } target.osx-arm64 = {dependencies = {mlx = "x.y.z"}} ``` From 7ac8a5870a7dfa89db57f678228c7080f351c3ca Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 13:33:24 +0100 Subject: [PATCH 06/18] Update docs/design_proposals/multi_environment_proposal.md Co-authored-by: Bela Stoyan --- docs/design_proposals/multi_environment_proposal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index 867897779..06132e106 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -20,7 +20,7 @@ This prepares `pixi` for the use in large projects with multiple use-cases, mult 1. **User-friendliness**: Pixi is a user focussed tool this goes beyond developers. The feature should have good error reporting and helpful documentation from the start. This is opinionated so the user sided PR's should be checked by multiple developers. 2. **Keep it simple**: Not understanding the multiple environments feature shouldn't limit a user to use pixi. The feature should be "invisible" to the non-multi env use-cases. 3. **No Automatic Combinatorial**: To ensure the dependency resolution process remains manageable, the solution should avoid a combinatorial explosion of dependency sets. By making the environments user defined and not automatically inferred by testing a matrix of the features. -4. **Single Feature Activation**: The design should allow only one feature set to be active at any given time, simplifying the resolution process and preventing conflicts. +4. **Single environment Activation**: The design should allow only one environment to be active at any given time, simplifying the resolution process and preventing conflicts. 5. **Fixed Lockfiles**: It's crucial to preserve fixed lockfiles for consistency and predictability. Solutions must ensure reliability not just for authors but also for end-users, particularly at the time of lockfile creation. ## Proposed Solution From 2d46fd3249d4497a5077ecf2f83d35b5a77c7b87 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 13:20:33 +0100 Subject: [PATCH 07/18] Add real world example polarify, folded on website --- .../multi_environment_proposal.md | 92 ++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index 06132e106..13e414bcc 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -182,6 +182,96 @@ pixi run test # 2. test39 ``` -## Important links: +## Important links - Initial writeup of the proposal: https://gist.github.com/0xbe7a/bbf8a323409be466fe1ad77aa6dd5428 - GitHub project: https://github.com/orgs/prefix-dev/projects/10 + +## Real world example use cases +??? tip "Polarify test setup" + In `polarify` they want to test multiple versions combined with multiple versions of polars. + This is currently done by using a matrix in GitHub actions. + This can be replaced by using multiple environments. + + ```toml title="pixi.toml" + [project] + name = "polarify" + # ... + channels = ["conda-forge"] + platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] + + [tasks] + postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version-check -e ." + + [dependencies] + python = ">=3.9" + pip = "*" + polars = ">=0.14.24,<0.21" + + [feature.py39.dependencies] + python = "3.9.*" + [feature.py310.dependencies] + python = "3.10.*" + [feature.py311.dependencies] + python = "3.11.*" + [feature.py312.dependencies] + python = "3.12.*" + [feature.pl017.dependencies] + polars = "0.17.*" + [feature.pl018.dependencies] + polars = "0.18.*" + [feature.pl019.dependencies] + polars = "0.19.*" + [feature.pl020.dependencies] + polars = "0.20.*" + + [feature.test.dependencies] + pytest = "*" + pytest-md = "*" + pytest-emoji = "*" + hypothesis = "*" + [feature.test.tasks] + test = "pytest" + + [feature.lint.dependencies] + pre-commit = "*" + [feature.lint.tasks] + lint = "pre-commit run --all" + + [environments] + pl017 = ["pl017", "py39", "test"] + pl018 = ["pl018", "py39", "test"] + pl019 = ["pl019", "py39", "test"] + pl020 = ["pl020", "py39", "test"] + py39 = ["py39", "test"] + py310 = ["py310", "test"] + py311 = ["py311", "test"] + py312 = ["py312", "test"] + ``` + + ```yaml title=".github/workflows/test.yml" + jobs: + tests: + name: Test ${{ matrix.environment }} + runs-on: ubuntu-latest + strategy: + matrix: + environment: + - pl017 + - pl018 + - pl019 + - pl020 + - py39 + - py310 + - py311 + - py312 + steps: + - uses: actions/checkout@v4 + - uses: prefix-dev/setup-pixi@v0.5.0 + with: + # already installs the corresponding environment and caches it + environments: ${{ matrix.environment }} + - name: Install dependencies + run: | + pixi run --env ${{ matrix.environment }} postinstall + pixi run --env ${{ matrix.environment }} test + ``` From 3b5360b8d8cc72d3c4607c8bfe0467cafaac84ae Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 13:54:45 +0100 Subject: [PATCH 08/18] ci: avoid ci being run on docs --- .github/workflows/rust.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 816919d96..eed34db6c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -6,10 +6,14 @@ on: - main paths-ignore: - "docs/**" + - "mkdocs.yml" + - "*.md" workflow_dispatch: pull_request: paths-ignore: - "docs/**" + - "mkdocs.yml" + - "*.md" name: Rust From cbe85670916913b87341b40e552917a128d75b3a Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 14:27:08 +0100 Subject: [PATCH 09/18] Update docs/design_proposals/multi_environment_proposal.md Co-authored-by: Pavel Zwerschke --- .../multi_environment_proposal.md | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index 13e414bcc..74b547df8 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -250,28 +250,28 @@ pixi run test ```yaml title=".github/workflows/test.yml" jobs: - tests: - name: Test ${{ matrix.environment }} - runs-on: ubuntu-latest - strategy: - matrix: - environment: - - pl017 - - pl018 - - pl019 - - pl020 - - py39 - - py310 - - py311 - - py312 - steps: - - uses: actions/checkout@v4 - - uses: prefix-dev/setup-pixi@v0.5.0 - with: - # already installs the corresponding environment and caches it - environments: ${{ matrix.environment }} - - name: Install dependencies - run: | - pixi run --env ${{ matrix.environment }} postinstall - pixi run --env ${{ matrix.environment }} test + tests: + name: Test ${{ matrix.environment }} + runs-on: ubuntu-latest + strategy: + matrix: + environment: + - pl017 + - pl018 + - pl019 + - pl020 + - py39 + - py310 + - py311 + - py312 + steps: + - uses: actions/checkout@v4 + - uses: prefix-dev/setup-pixi@v0.5.0 + with: + # already installs the corresponding environment and caches it + environments: ${{ matrix.environment }} + - name: Install dependencies + run: | + pixi run --env ${{ matrix.environment }} postinstall + pixi run --env ${{ matrix.environment }} test ``` From c8b6b0976f6338d91546cc90ab422abd5e0c3563 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 16:01:43 +0100 Subject: [PATCH 10/18] Update docs/design_proposals/multi_environment_proposal.md Co-authored-by: Travis Hathaway --- docs/design_proposals/multi_environment_proposal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index 74b547df8..36f00fa45 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -14,7 +14,7 @@ There are multiple scenarios where multiple environments are useful. - **Multiple machines from one project**, e.g. a `cuda` environment and a `cpu` environment. - **And many more.** (If you have a use-case please add it to the list, so we can make sure it's covered) -This prepares `pixi` for the use in large projects with multiple use-cases, multiple developers and different CI needs. +This prepares `pixi` for use in large projects with multiple use-cases, multiple developers and different CI needs. ## Design Considerations 1. **User-friendliness**: Pixi is a user focussed tool this goes beyond developers. The feature should have good error reporting and helpful documentation from the start. This is opinionated so the user sided PR's should be checked by multiple developers. From dd9526bf435917d0b10280e5d5bb9998434d3ef6 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 17:02:29 +0100 Subject: [PATCH 11/18] Update docs/design_proposals/multi_environment_proposal.md --- docs/design_proposals/multi_environment_proposal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index 36f00fa45..fda2f7b81 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -17,7 +17,7 @@ There are multiple scenarios where multiple environments are useful. This prepares `pixi` for use in large projects with multiple use-cases, multiple developers and different CI needs. ## Design Considerations -1. **User-friendliness**: Pixi is a user focussed tool this goes beyond developers. The feature should have good error reporting and helpful documentation from the start. This is opinionated so the user sided PR's should be checked by multiple developers. +1. **User-friendliness**: Pixi is a user focussed tool that goes beyond developers. The feature should have good error reporting and helpful documentation from the start. 2. **Keep it simple**: Not understanding the multiple environments feature shouldn't limit a user to use pixi. The feature should be "invisible" to the non-multi env use-cases. 3. **No Automatic Combinatorial**: To ensure the dependency resolution process remains manageable, the solution should avoid a combinatorial explosion of dependency sets. By making the environments user defined and not automatically inferred by testing a matrix of the features. 4. **Single environment Activation**: The design should allow only one environment to be active at any given time, simplifying the resolution process and preventing conflicts. From aea25adefd5058e21f2e251478f91792a57adec5 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 20 Dec 2023 17:02:46 +0100 Subject: [PATCH 12/18] Update docs/design_proposals/multi_environment_proposal.md --- docs/design_proposals/multi_environment_proposal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index fda2f7b81..a55f32cf1 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -12,7 +12,7 @@ There are multiple scenarios where multiple environments are useful. - **Large developer environments**, that combine all the smaller environments, e.g. `dev`. - **Strict supersets of environments**, e.g. `prod` and `test-prod` where `test-prod` is a strict superset of `prod`. - **Multiple machines from one project**, e.g. a `cuda` environment and a `cpu` environment. -- **And many more.** (If you have a use-case please add it to the list, so we can make sure it's covered) +- **And many more.** (Feel free to edit this document in our GitHub and add your use case.) This prepares `pixi` for use in large projects with multiple use-cases, multiple developers and different CI needs. From 42b553834d2722ecfe3d241e5e771acb15dd88ad Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 21 Dec 2023 09:56:32 +0100 Subject: [PATCH 13/18] add prod vs test example --- .../multi_environment_proposal.md | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index a55f32cf1..ee7af9471 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -275,3 +275,60 @@ pixi run test pixi run --env ${{ matrix.environment }} postinstall pixi run --env ${{ matrix.environment }} test ``` +??? tip "Test vs Production example" + This is an example of a project that has a `test` feature and `prod` environment. + The `prod` environment is a production environment that contains the run dependencies. + The `test` feature is a set of dependencies and tasks that we want to put on top of the previously solved `prod` environment. + This is a common use case where we want to test the production environment with additional dependencies. + + ```toml title="pixi.toml" + [project] + name = "my-app" + # ... + channels = ["conda-forge"] + platforms = ["osx-arm64", "linux-64"] + + [tasks] + postinstall-e = "pip install --no-build-isolation --no-deps --disable-pip-version-check -e ." + postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version-check ." + dev = "uvicorn my_app.app:main --reload" + serve = "uvicorn my_app.app:main" + + [dependencies] + python = ">=3.12" + pip = "*" + pydantic = ">=2" + fastapi = ">=0.105.0" + sqlalchemy = ">=2,<3" + uvicorn = "*" + aiofiles = "*" + + [feature.test.dependencies] + pytest = "*" + pytest-md = "*" + pytest-asyncio = "*" + [feature.test.tasks] + test = "pytest --md=report.md" + + [environments] + default = ["test", {environment = "prod"}] # implicit ["default", "test"], overrides default environment + prod = [] # implicit for ["default"] + ``` + In ci you would run the following commands: + ```shell + pixi run postinstall-e && pixi run test + ``` + Locally you would run the following command: + ```shell + pixi run postinstall-e && pixi run dev + ``` + + Then in a Dockerfile you would run the following command: + ```dockerfile title="Dockerfile" + FROM ghcr.io/prefix-dev/pixi:latest # this doesn't exist yet + WORKDIR /app + COPY . . + RUN pixi run --env prod postinstall + EXPOSE 8080 + CMD ["/usr/local/bin/pixi", "run", "--env", "prod", "serve"] + ``` From d7b059b079cd8407a5ee7b40ac843b09d01c5800 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 21 Dec 2023 13:28:47 +0100 Subject: [PATCH 14/18] remove `default-features` field from the environments --- .../multi_environment_proposal.md | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index ee7af9471..0e6b73211 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -92,14 +92,11 @@ The environment definition should contain the following fields: - `features: Vec`: The features that are included in the environment set, which is also the default field in the environments. - `environments: Vec`: The environments that are included in the environment set. When environments is used, the extra features are **on top** of the included environments. Environments are used as a locked base, so the features added to an environment are not allowed to change the locked set. This should result in a failure if the locked set is not compatible with the added features. -- `default-features: bool`: Whether the default features should be included in the environment set. ```toml [environments] -# `default` environment is now the `default` feature plus the py39 feature +# `default` environment is now the `default` feature plus the `py39` feature default = ["py39"] -# `lint` environment is now the `lint` feature without the `default` feature or environment -lint = {features = ["lint"], default-features = "false"} # `dev` environment is now the `default` feature plus the `test` feature, which makes the `default` envriroment is solved without the use of the test feature. dev = {environments = ["default"], features = ["test"]} ``` @@ -111,7 +108,6 @@ default = ["py39"] # implicit: default = ["py39", "default"] py310 = ["py310"] # implicit: py310 = ["py310", "default"] test = ["test"] # implicit: test = ["test", "default"] test39 = ["test", "py39"] # implicit: test39 = ["test", "py39", "default"] -lint = {features = ["lint"], default-features = "false"} # no implicit default ``` ```toml title="Creating environments from environments" linenums="1" @@ -121,6 +117,23 @@ prod = ["py39"] test_prod = {environments = ["prod"], features = ["test"]} ``` +```toml title="Creating environments without a default environment" linenums="1" +[dependencies] +# Keep empty or undefined to create an empty environment. + +[feature.base.dependencies] +python = "*" + +[feature.lint.dependencies] +pre-commit = "*" + +[environments] +# Create a custom default +default = ["base"] +# Create a custom environment which only has the `lint` feature as the default feature is empty. +lint = ["lint"] +``` + ### Lockfile Structure Within the `pixi.lock` file, a package may now include an additional `environments` field, specifying the environment to which it belongs. To avoid duplication the packages `environments` field may contain multiple environments so the lockfile is of minimal size. From 4c8078969fb39a332114287048af9793411c8dcd Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Sun, 24 Dec 2023 11:48:05 +0100 Subject: [PATCH 15/18] replace `environments` with `solve-group` --- .../multi_environment_proposal.md | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index 0e6b73211..37158ff07 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -84,22 +84,15 @@ test = "pytest" [environments] test = ["test"] -# `pixi run test` == `pixi run --envrionment test test` +# `pixi run test` == `pixi run --environments test test` ``` The environment definition should contain the following fields: - `features: Vec`: The features that are included in the environment set, which is also the default field in the environments. -- `environments: Vec`: The environments that are included in the environment set. When environments is used, the extra features are **on top** of the included environments. - Environments are used as a locked base, so the features added to an environment are not allowed to change the locked set. This should result in a failure if the locked set is not compatible with the added features. - -```toml -[environments] -# `default` environment is now the `default` feature plus the `py39` feature -default = ["py39"] -# `dev` environment is now the `default` feature plus the `test` feature, which makes the `default` envriroment is solved without the use of the test feature. -dev = {environments = ["default"], features = ["test"]} -``` +- `solve-group: String`: The solve group is used to group environments together at the solve stage. +This is useful for environments that need to have the same dependencies but might extend them with additional dependencies. +For instance when testing a production environment with additional test dependencies. ```toml title="Creating environments from features" linenums="1" [environments] @@ -110,11 +103,14 @@ test = ["test"] # implicit: test = ["test", "default"] test39 = ["test", "py39"] # implicit: test39 = ["test", "py39", "default"] ``` -```toml title="Creating environments from environments" linenums="1" +```toml title="Testing a production environment with additional dependencies" linenums="1" [environments] -prod = ["py39"] -# Takes the `prod` environment and adds the `test` feature to it without modifying the `prod` environment requirements, solve should fail if requirements don't comply with locked set. -test_prod = {environments = ["prod"], features = ["test"]} +# Creating a `prod` environment which is the minimal set of dependencies used for production. +prod = {features = ["py39"], solve-group = "prod"} +# Creating a `test_prod` environment which is the `prod` environment plus the `test` feature. +test_prod = {features = ["py39", "test"], solve-group = "prod"} +# Using the `solve-group` to solve the `prod` and `test_prod` environments together +# Which makes sure the tested environment has the same version of the dependencies as the production environment. ``` ```toml title="Creating environments without a default environment" linenums="1" From 764c9a5b6ba1a413ce073f6f3ee5bdc514542034 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 26 Dec 2023 15:36:00 +0100 Subject: [PATCH 16/18] Update docs/design_proposals/multi_environment_proposal.md Co-authored-by: Pavel Zwerschke --- docs/design_proposals/multi_environment_proposal.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index 37158ff07..d231c4b47 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -320,8 +320,9 @@ pixi run test test = "pytest --md=report.md" [environments] - default = ["test", {environment = "prod"}] # implicit ["default", "test"], overrides default environment - prod = [] # implicit for ["default"] + # both default and prod will have exactly the same dependency versions when they share a dependency + default = {features = ["test"], solve-group = "prod-group"} + prod = {features = [], solve-group = "prod-group"} ``` In ci you would run the following commands: ```shell From 414a5c89117db7acc4effc8b0984d2c84f803d0d Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 2 Jan 2024 13:37:39 +0100 Subject: [PATCH 17/18] add system-requirements to cuda example --- docs/design_proposals/multi_environment_proposal.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index d231c4b47..ee4e32f6e 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -71,6 +71,7 @@ dependencies = {cuda = "x.y.z", cudnn = "12.0"} pypi-dependencies = {torch = "1.9.0"} platforms = ["linux-64", "osx-arm64"] activation = {scripts = ["cuda_activation.sh"]} +system-requirements = {cuda = "12"} channels = ["nvidia"] # Would concat instead of overwrite, so the default channels are still used. tasks = { warmup = "python warmup.py" } target.osx-arm64 = {dependencies = {mlx = "x.y.z"}} From cee41f6e8ad2d93263876f02e2032a4e7321b722 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Tue, 2 Jan 2024 16:36:43 +0100 Subject: [PATCH 18/18] add `priority` to the channels to control the concatenation --- docs/design_proposals/multi_environment_proposal.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/design_proposals/multi_environment_proposal.md b/docs/design_proposals/multi_environment_proposal.md index ee4e32f6e..601e9a865 100644 --- a/docs/design_proposals/multi_environment_proposal.md +++ b/docs/design_proposals/multi_environment_proposal.md @@ -36,7 +36,7 @@ As an environment goes beyond just `dependencies` the `features` should be descr - `system-requirements`: The system requirements of the environment - `activation`: The activation information for the environment - `platforms`: The platforms the environment can be run on. -- `channels`: The channels used to create the environment. +- `channels`: The channels used to create the environment. Adding the `priority` field to the channels to allow concatenation of channels instead of overwriting. - `target`: All the above features but also separated by targets. - `tasks`: Feature specific tasks, tasks in one environment are selected as default tasks for the environment. @@ -72,7 +72,9 @@ pypi-dependencies = {torch = "1.9.0"} platforms = ["linux-64", "osx-arm64"] activation = {scripts = ["cuda_activation.sh"]} system-requirements = {cuda = "12"} -channels = ["nvidia"] # Would concat instead of overwrite, so the default channels are still used. +# Channels concatenate using a priority instead of overwrite, so the default channels are still used. +# Using the priority the concatenation is controlled, default is 0, so the default channels are used first. +channels = [{name = "nvidia", priority = "-1"}, "pytorch"] # Results in: ["nvidia", "conda-forge", "pytorch"] if the default is `conda-forge` tasks = { warmup = "python warmup.py" } target.osx-arm64 = {dependencies = {mlx = "x.y.z"}} ```