feat: drop rattler_installs_packages for uv crates (#412)

Drops the dependency on `rattler_installs_packages` in favour of the uv crates. Note that while this works, the creates of uv do not officially support depending directly on the crates, however there is little overhead here, and is pinned to a fixed SHA. Closes #307 Closes #344 Closes #345 Due to compilation issues under llvm, this diff also switches to musl for the toolchain binaries. We need to keep llvm for proc macros, unfortunately. --- ### Changes are visible to end-users: yes - Searched for relevant documentation and updated as needed: yes - Breaking change (forces users to change their own code or config): no - Suggested release notes appear below: yes Drop dependency on `rattler_installs_packages` in favour of the uv crates. Produce statically linked toolchain binaries via musl. ### Test plan - New test cases added
aspect-build · Oct 24, 2024 · ca62068 · ca62068
1 parent 284162a
commit ca62068
Show file tree

Hide file tree

Showing 28 changed files with 13,312 additions and 7,417 deletions.
diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -17,7 +17,22 @@ readme = "README.md"
 rust-version = "1.81.0"
 
 [workspace.dependencies]
+clap = { version = "4.5.20", features = ["derive"] }
+itertools = "0.13.0"
 miette = { version = "7.2", features = ["fancy"] }
+tempfile = "3.13.0"
+thiserror = "1.0.64"
+uv-cache = { git = "https://github.com/astral-sh/uv", rev = "855c1917e1e0e2b48c38de71bebc845af016afae" }
+uv-distribution-filename = { git = "https://github.com/astral-sh/uv.git", rev = "855c1917e1e0e2b48c38de71bebc845af016afae" }
+uv-extract = { git = "https://github.com/astral-sh/uv.git", rev = "855c1917e1e0e2b48c38de71bebc845af016afae" }
+uv-install-wheel = { git = "https://github.com/astral-sh/uv", rev = "855c1917e1e0e2b48c38de71bebc845af016afae" }
+uv-pypi-types = { git = "https://github.com/astral-sh/uv.git", rev = "855c1917e1e0e2b48c38de71bebc845af016afae" }
+uv-python = { git = "https://github.com/astral-sh/uv", rev = "855c1917e1e0e2b48c38de71bebc845af016afae" }
+uv-virtualenv = { git = "https://github.com/astral-sh/uv", rev = "855c1917e1e0e2b48c38de71bebc845af016afae" }
+
+[patch.crates-io]
+reqwest-middleware = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "5e3eaf254b5bd481c75d2710eed055f95b756913", features = ["multipart"] }
+reqwest-retry = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "5e3eaf254b5bd481c75d2710eed055f95b756913" }
 
 [profile.release]
 strip = true
diff --git a/WORKSPACE b/WORKSPACE
@@ -49,6 +49,14 @@ load("@aspect_bazel_lib//lib:repositories.bzl", "register_coreutils_toolchains")
 
 register_coreutils_toolchains()
 
+load("@musl_toolchains//:repositories.bzl", "load_musl_toolchains")
+
+load_musl_toolchains(extra_target_compatible_with = ["@//tools/linkers:musl"])
+
+load("@musl_toolchains//:toolchains.bzl", "register_musl_toolchains")
+
+register_musl_toolchains()
+
 ############################################
 ## CC toolchain using llvm
 load("@toolchains_llvm//toolchain:deps.bzl", "bazel_toolchain_dependencies")
@@ -156,34 +164,123 @@ RUST_EDITION = "2021"
 
 RUST_VERSION = "1.81.0"
 
-rust_register_toolchains(
+# Declare cross-compilation toolchains
+rust_repository_set(
+    name = "apple_darwin_aarch64",
     edition = RUST_EDITION,
-    extra_target_triples = [
-        "x86_64-apple-darwin",
-    ],
+    exec_triple = "aarch64-apple-darwin",
+    # and cross-compile to these platforms:
+    extra_target_triples = {
+        "aarch64-apple-darwin": [
+            "@platforms//cpu:arm64",
+            "@platforms//os:macos",
+        ],
+        "aarch64-unknown-linux-musl": [
+            "@platforms//cpu:arm64",
+            "@platforms//os:linux",
+            "@//tools/linkers:musl",
+        ],
+        "x86_64-apple-darwin": [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:macos",
+        ],
+        "x86_64-unknown-linux-musl": [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:linux",
+            "@//tools/linkers:musl",
+        ],
+    },
     versions = [RUST_VERSION],
 )
 
-# Declare cross-compilation toolchains
 rust_repository_set(
-    name = "apple_darwin_86_64",
+    name = "apple_darwin_x86_64",
     edition = RUST_EDITION,
     exec_triple = "x86_64-apple-darwin",
     # and cross-compile to these platforms:
-    extra_target_triples = [
-        "aarch64-apple-darwin",
-    ],
+    extra_target_triples = {
+        "aarch64-apple-darwin": [
+            "@platforms//cpu:arm64",
+            "@platforms//os:macos",
+        ],
+        "aarch64-unknown-linux-musl": [
+            "@platforms//cpu:arm64",
+            "@platforms//os:linux",
+            "@//tools/linkers:musl",
+        ],
+        "x86_64-apple-darwin": [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:macos",
+        ],
+        "x86_64-unknown-linux-musl": [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:linux",
+            "@//tools/linkers:musl",
+        ],
+    },
     versions = [RUST_VERSION],
 )
 
 rust_repository_set(
-    name = "linux_x86_64",
+    name = "rust_linux_x86_64",
     edition = RUST_EDITION,
     exec_triple = "x86_64-unknown-linux-gnu",
-    # and cross-compile to these platforms:
-    extra_target_triples = [
-        "aarch64-unknown-linux-gnu",
-    ],
+    extra_target_triples = {
+        "aarch64-unknown-linux-gnu": [
+            "@platforms//cpu:arm64",
+            "@platforms//os:linux",
+            "@//tools/linkers:unknown",
+        ],
+        "aarch64-unknown-linux-musl": [
+            "@platforms//cpu:arm64",
+            "@platforms//os:linux",
+            "@//tools/linkers:musl",
+        ],
+        "x86_64-unknown-linux-gnu": [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:linux",
+            "@//tools/linkers:unknown",
+        ],
+        "x86_64-unknown-linux-musl": [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:linux",
+            "@//tools/linkers:musl",
+        ],
+    },
+    versions = [RUST_VERSION],
+)
+
+rust_repository_set(
+    name = "rust_linux_aarch64",
+    edition = RUST_EDITION,
+    exec_triple = "aarch64-unknown-linux-gnu",
+    extra_target_triples = {
+        "aarch64-unknown-linux-gnu": [
+            "@platforms//cpu:arm64",
+            "@platforms//os:linux",
+            "@//tools/linkers:unknown",
+        ],
+        "aarch64-unknown-linux-musl": [
+            "@platforms//cpu:arm64",
+            "@platforms//os:linux",
+            "@//tools/linkers:musl",
+        ],
+        "x86_64-unknown-linux-gnu": [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:linux",
+            "@//tools/linkers:unknown",
+        ],
+        "x86_64-unknown-linux-musl": [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:linux",
+            "@//tools/linkers:musl",
+        ],
+    },
+    versions = [RUST_VERSION],
+)
+
+rust_register_toolchains(
+    edition = RUST_EDITION,
     versions = [RUST_VERSION],
 )
 

diff --git a/py/private/py_binary.bzl b/py/private/py_binary.bzl
@@ -73,11 +73,6 @@ def _py_binary_rule_impl(ctx):
             "{{ARG_COLLISION_STRATEGY}}": ctx.attr.package_collisions,
             "{{ARG_PYTHON}}": to_rlocation_path(ctx, py_toolchain.python) if py_toolchain.runfiles_interpreter else py_toolchain.python.path,
             "{{ARG_VENV_NAME}}": ".{}.venv".format(ctx.attr.name),
-            "{{ARG_VENV_PYTHON_VERSION}}": "{}.{}.{}".format(
-                py_toolchain.interpreter_version_info.major,
-                py_toolchain.interpreter_version_info.minor,
-                py_toolchain.interpreter_version_info.micro,
-            ),
             "{{ARG_PTH_FILE}}": to_rlocation_path(ctx, site_packages_pth_file),
             "{{ENTRYPOINT}}": to_rlocation_path(ctx, ctx.file.main),
             "{{PYTHON_ENV}}": "\n".join(_dict_to_exports(env)).strip(),

diff --git a/py/private/py_unpacked_wheel.bzl b/py/private/py_unpacked_wheel.bzl
@@ -18,16 +18,12 @@ def _py_unpacked_wheel_impl(ctx):
         unpack_directory.path,
         "--wheel",
         ctx.file.src.path,
-        "--python",
-        py_toolchain.python,
         "--python-version",
         "{}.{}.{}".format(
             py_toolchain.interpreter_version_info.major,
             py_toolchain.interpreter_version_info.minor,
             py_toolchain.interpreter_version_info.micro,
         ),
-        "--package-name",
-        ctx.attr.py_package_name,
     ])
 
     ctx.actions.run(
@@ -72,9 +68,6 @@ _attrs = {
         allow_single_file = [".whl"],
         mandatory = True,
     ),
-    "py_package_name": attr.string(
-        mandatory = True,
-    ),
     # NB: this is read by _resolve_toolchain in py_semantics.
     "_interpreter_version_flag": attr.label(
         default = "//py:interpreter_version",

diff --git a/py/private/py_venv.bzl b/py/private/py_venv.bzl
@@ -54,11 +54,7 @@ def _py_venv_rule_imp(ctx):
             "{{ARG_PYTHON}}": to_rlocation_path(ctx, py_toolchain.python) if py_toolchain.runfiles_interpreter else py_toolchain.python.path,
             "{{ARG_COLLISION_STRATEGY}}": ctx.attr.package_collisions,
             "{{ARG_VENV_LOCATION}}": paths.join(ctx.attr.location, ctx.attr.venv_name),
-            "{{ARG_VENV_PYTHON_VERSION}}": "{}.{}.{}".format(
-                py_toolchain.interpreter_version_info.major,
-                py_toolchain.interpreter_version_info.minor,
-                py_toolchain.interpreter_version_info.micro,
-            ),
+            "{{ARG_VENV_NAME}}": ".{}.venv".format(ctx.attr.name),
             "{{ARG_PTH_FILE}}": to_rlocation_path(ctx, site_packages_pth_file),
             "{{EXEC_PYTHON_BIN}}": "python{}".format(
                 py_toolchain.interpreter_version_info.major,

diff --git a/py/private/run.tmpl.sh b/py/private/run.tmpl.sh
@@ -41,9 +41,9 @@ export VIRTUAL_ENV
 "${VENV_TOOL}" \
     --location "${VIRTUAL_ENV}" \
     --python "$(python_location)" \
-    --python-version "{{ARG_VENV_PYTHON_VERSION}}" \
     --pth-file "$(rlocation {{ARG_PTH_FILE}})" \
-    --collision-strategy "{{ARG_COLLISION_STRATEGY}}"
+    --collision-strategy "{{ARG_COLLISION_STRATEGY}}" \
+    --venv-name "{{ARG_VENV_NAME}}"
 
 PATH="${VIRTUAL_ENV}/bin:${PATH}"
 export PATH

diff --git a/py/private/toolchain/tools.bzl b/py/private/toolchain/tools.bzl
@@ -30,15 +30,15 @@ TOOLCHAIN_PLATFORMS = {
     ),
     "linux_amd64": struct(
         arch = "x86_64",
-        vendor_os_abi = "unknown-linux-gnu",
+        vendor_os_abi = "unknown-linux-musl",
         compatible_with = [
             "@platforms//os:linux",
             "@platforms//cpu:x86_64",
         ],
     ),
     "linux_arm64": struct(
         arch = "aarch64",
-        vendor_os_abi = "unknown-linux-gnu",
+        vendor_os_abi = "unknown-linux-musl",
         compatible_with = [
             "@platforms//os:linux",
             "@platforms//cpu:aarch64",

diff --git a/py/private/venv.tmpl.sh b/py/private/venv.tmpl.sh
@@ -27,9 +27,9 @@ VIRTUAL_ENV="$(alocation "${VENV_ROOT}/{{ARG_VENV_LOCATION}}")"
 "${VENV_TOOL}" \
     --location "${VIRTUAL_ENV}" \
     --python "$(alocation $(rlocation {{ARG_PYTHON}}))" \
-    --python-version "{{ARG_VENV_PYTHON_VERSION}}" \
     --pth-file "$(rlocation {{ARG_PTH_FILE}})" \
     --pth-entry-prefix "${RUNFILES_DIR}" \
-    --collision-strategy "{{ARG_COLLISION_STRATEGY}}"
+    --collision-strategy "{{ARG_COLLISION_STRATEGY}}" \
+    --venv-name "{{ARG_VENV_NAME}}"
 
 echo "Created virtualenv in ${VIRTUAL_ENV}"
diff --git a/py/tests/virtual/django/BUILD.bazel b/py/tests/virtual/django/BUILD.bazel
@@ -7,7 +7,6 @@ django_resolutions = resolutions.from_requirements(all_whl_requirements_by_packa
 py_unpacked_wheel(
     name = "django_wheel",
     src = "@django_4_2_4//file",
-    py_package_name = "Django",
 )
 
 compile_pip_requirements(

diff --git a/py/tools/pex/BUILD.bazel b/py/tools/pex/BUILD.bazel
@@ -3,13 +3,12 @@ load("//py:defs.bzl", "py_binary", "py_unpacked_wheel")
 py_unpacked_wheel(
     name = "pex_unpacked",
     src = "@rules_py_pex_2_3_1//file",
-    py_package_name = "pex"
 )
 
 py_binary(
     name = "pex",
     srcs = ["main.py"],
     main = "main.py",
+    visibility = ["//visibility:public"],
     deps = [":pex_unpacked"],
-    visibility = ["//visibility:public"]
-)
+)
diff --git a/py/tools/py/BUILD.bazel b/py/tools/py/BUILD.bazel
@@ -3,7 +3,6 @@ load("@rules_rust//rust:defs.bzl", "rust_library")
 rust_library(
     name = "py",
     srcs = [
-        "src/interpreter.rs",
         "src/lib.rs",
         "src/pth.rs",
         "src/unpack.rs",
@@ -14,7 +13,16 @@ rust_library(
         "//py/tools/venv_bin:__pkg__",
     ],
     deps = [
+        "@crate_index//:itertools",
         "@crate_index//:miette",
-        "@crate_index//:rattler_installs_packages",
+        "@crate_index//:tempfile",
+        "@crate_index//:thiserror",
+        "@crate_index//:uv-cache",
+        "@crate_index//:uv-distribution-filename",
+        "@crate_index//:uv-extract",
+        "@crate_index//:uv-install-wheel",
+        "@crate_index//:uv-pypi-types",
+        "@crate_index//:uv-python",
+        "@crate_index//:uv-virtualenv",
     ],
 )
diff --git a/py/tools/py/Cargo.toml b/py/tools/py/Cargo.toml
@@ -12,5 +12,14 @@ rust-version.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+itertools = { workspace = true }
 miette = { workspace = true }
-rattler_installs_packages = { git = "https://github.com/prefix-dev/rip", rev = "1b4d909496f68c292800ebbd3667c8682b01d218", default-features = false, features = ["rustls-tls"] }
+tempfile = { workspace = true }
+thiserror = { workspace = true }
+uv-cache = { workspace = true }
+uv-distribution-filename = { workspace = true }
+uv-extract = { workspace = true }
+uv-install-wheel = { workspace = true }
+uv-pypi-types = { workspace = true }
+uv-python = { workspace = true }
+uv-virtualenv = { workspace = true }
diff --git a/py/tools/py/src/lib.rs b/py/tools/py/src/lib.rs
@@ -1,10 +1,8 @@
-mod interpreter;
 mod pth;
 mod unpack;
 mod venv;
 
 pub use unpack::unpack_wheel;
 pub use venv::create_venv;
 
-pub(crate) use interpreter::Interpreter;
 pub use pth::{PthFile, SymlinkCollisionResolutionStrategy};