refactor(toolchain): use bazel to extract zstd archives (bazelbuild…

…#2412) Here we remove the dependence on the `zstd` archive downloaded from GH for extracting `zstd` toolchains because bazel now supports `zstd` extraction for a long time. This mainly cleans up the code which is again used a lot more due to bazelbuild#2386. Co-authored-by: Richard Levasseur <[email protected]>
aignas · Nov 18, 2024 · 79bd1f5 · 79bd1f5
1 parent b28db69
commit 79bd1f5
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 56 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -52,7 +52,7 @@ Unreleased changes template.
 
 {#v0-0-0-changed}
 ### Changed
-* Nothing changed.
+* (python_repository) Start honoring the `strip_prefix` field for `zstd` archives.
 
 {#v0-0-0-fixed}
 ### Fixed
@@ -89,6 +89,11 @@ Unreleased changes template.
 ### Removed
 * (publish) Remove deprecated `requirements.txt` for the `twine` dependencies.
   Please use `requirements_linux.txt` instead.
+* (python_repository) Use bazel's built in `zstd` support and remove attributes
+  for customizing the `zstd` binary to be used for `zstd` archives in the
+  {bzl:obj}`python_repository` repository_rule. This affects the
+  {bzl:obj}`python_register_toolchains` and
+  {bzl:obj}`python_register_multi_toolchains` callers in the `WORKSPACE`.
 
 {#v0-39-0}
 ## [0.39.0] - 2024-11-13

diff --git a/python/private/python.bzl b/python/private/python.bzl
@@ -462,7 +462,7 @@ def _get_toolchain_config(*, modules, _fail = fail):
             "strip_prefix": {
                 platform: item["strip_prefix"]
                 for platform in item["sha256"]
-            },
+            } if type(item["strip_prefix"]) == type("") else item["strip_prefix"],
             "url": {
                 platform: [item["url"]]
                 for platform in item["sha256"]

diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl
@@ -15,7 +15,7 @@
 """This file contains repository rules and macros to support toolchain registration.
 """
 
-load("//python:versions.bzl", "FREETHREADED", "PLATFORMS")
+load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY", "PLATFORMS")
 load(":auth.bzl", "get_auth")
 load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
 load(":text_util.bzl", "render")
@@ -72,51 +72,13 @@ def _python_repository_impl(rctx):
     urls = rctx.attr.urls or [rctx.attr.url]
     auth = get_auth(rctx, urls)
 
-    if release_filename.endswith(".zst"):
-        rctx.download(
+    if INSTALL_ONLY in release_filename:
+        rctx.download_and_extract(
             url = urls,
             sha256 = rctx.attr.sha256,
-            output = release_filename,
+            stripPrefix = rctx.attr.strip_prefix,
             auth = auth,
         )
-        unzstd = rctx.which("unzstd")
-        if not unzstd:
-            url = rctx.attr.zstd_url.format(version = rctx.attr.zstd_version)
-            rctx.download_and_extract(
-                url = url,
-                sha256 = rctx.attr.zstd_sha256,
-                auth = auth,
-            )
-            working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version)
-
-            repo_utils.execute_checked(
-                rctx,
-                op = "python_repository.MakeZstd",
-                arguments = [
-                    repo_utils.which_checked(rctx, "make"),
-                    "--jobs=4",
-                ],
-                timeout = 600,
-                quiet = True,
-                working_directory = working_directory,
-                logger = logger,
-            )
-            zstd = "{working_directory}/zstd".format(working_directory = working_directory)
-            unzstd = "./unzstd"
-            rctx.symlink(zstd, unzstd)
-
-        repo_utils.execute_checked(
-            rctx,
-            op = "python_repository.ExtractRuntime",
-            arguments = [
-                repo_utils.which_checked(rctx, "tar"),
-                "--extract",
-                "--strip-components=2",
-                "--use-compress-program={unzstd}".format(unzstd = unzstd),
-                "--file={}".format(release_filename),
-            ],
-            logger = logger,
-        )
     else:
         rctx.download_and_extract(
             url = urls,
@@ -125,6 +87,12 @@ def _python_repository_impl(rctx):
             auth = auth,
         )
 
+        # Strip the things that are not present in the INSTALL_ONLY builds
+        # NOTE: if the dirs are not present, we will not fail here
+        rctx.delete("python/build")
+        rctx.delete("python/licenses")
+        rctx.delete("python/PYTHON.json")
+
     patches = rctx.attr.patches
     if patches:
         for patch in patches:
@@ -378,15 +346,6 @@ function defaults (e.g. `single_version_override` for `MODULE.bazel` files.
         "urls": attr.string_list(
             doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.",
         ),
-        "zstd_sha256": attr.string(
-            default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
-        ),
-        "zstd_url": attr.string(
-            default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz",
-        ),
-        "zstd_version": attr.string(
-            default = "1.5.2",
-        ),
         "_rule_name": attr.string(default = "python_repository"),
     },
     environ = [REPO_DEBUG_ENV_VAR],

diff --git a/python/versions.bzl b/python/versions.bzl
@@ -20,6 +20,7 @@ MACOS_NAME = "mac os"
 LINUX_NAME = "linux"
 WINDOWS_NAME = "windows"
 FREETHREADED = "freethreaded"
+INSTALL_ONLY = "install_only"
 
 DEFAULT_RELEASE_BASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download"
 
@@ -52,7 +53,7 @@ TOOL_VERSIONS = {
             "x86_64-apple-darwin": "8d06bec08db8cdd0f64f4f05ee892cf2fcbc58cfb1dd69da2caab78fac420238",
             "x86_64-unknown-linux-gnu": "aec8c4c53373b90be7e2131093caa26063be6d9d826f599c935c0e1042af3355",
         },
-        "strip_prefix": "python",
+        "strip_prefix": "python/install",
     },
     "3.8.12": {
         "url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz",
@@ -579,7 +580,22 @@ TOOL_VERSIONS = {
             "x86_64-pc-windows-msvc-freethreaded": "bfd89f9acf866463bc4baf01733da5e767d13f5d0112175a4f57ba91f1541310",
             "x86_64-unknown-linux-gnu-freethreaded": "a73adeda301ad843cce05f31a2d3e76222b656984535a7b87696a24a098b216c",
         },
-        "strip_prefix": "python",
+        "strip_prefix": {
+            "aarch64-apple-darwin": "python",
+            "aarch64-unknown-linux-gnu": "python",
+            "ppc64le-unknown-linux-gnu": "python",
+            "s390x-unknown-linux-gnu": "python",
+            "x86_64-apple-darwin": "python",
+            "x86_64-pc-windows-msvc": "python",
+            "x86_64-unknown-linux-gnu": "python",
+            "aarch64-apple-darwin-freethreaded": "python/install",
+            "aarch64-unknown-linux-gnu-freethreaded": "python/install",
+            "ppc64le-unknown-linux-gnu-freethreaded": "python/install",
+            "s390x-unknown-linux-gnu-freethreaded": "python/install",
+            "x86_64-apple-darwin-freethreaded": "python/install",
+            "x86_64-pc-windows-msvc-freethreaded": "python/install",
+            "x86_64-unknown-linux-gnu-freethreaded": "python/install",
+        },
     },
 }
 
@@ -777,7 +793,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U
                 }[p],
             )
         else:
-            build = "install_only"
+            build = INSTALL_ONLY
 
         if WINDOWS_NAME in platform:
             build = "shared-" + build