fix(toolchain): symlink all toolchain files for the host toolchain (#…

…1745)

Previously we would only symlink the interpreter binary itself
when creating the hermetic host toolchain used in setting up the
`whl_library` repositories. This works on UNIX platforms and Windows
if they have the following in their `.bazelrc`:
```
startup --windows_enable_symlinks
```

In our CI we had the same lines but the users did not need to add them
until the `0.29.0` forced them to have them because we actually started
using symlinks on Windows. If the symlinks are not enabled on the host
platform `bazel` tries to be helpful and copies the files over instead
of making the links. We are leveraging this to just symlink all of the
contents of the python interpreter repository for the host platform to
the `_host` toolchain repository.

Fixes #1723

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 
@@ -19,7 +19,6 @@ build --incompatible_default_to_explicit_init_py
 
 # Windows makes use of runfiles for some rules
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # Make Bazel 6 use bzlmod by default
 common --enable_bzlmod

CHANGELOG.md

@@ -36,13 +36,18 @@ A brief description of the categories of changes:
 ### Fixed
 
 * (bzlmod) pip.parse now does not fail with an empty `requirements.txt`.
+
 * (py_wheel) Wheels generated by `py_wheel` now preserve executable bits when
   being extracted by `installer` and/or `pip`.
+
 * (coverage) During the running of lcov, the stdout/stderr was causing test
   failures.  By default, suppress output when generating lcov.  This can be
   overridden by setting 'VERBOSE_COVERAGE'.  This change only affect bazel
   7.x.x and above.
 
+* (toolchain) Changed the `host_toolchain` to symlink all files to support
+  Windows host environments without symlink support.
+
 ### Added
 
 * (py_wheel) Added `requires_file` and `extra_requires_files` attributes.

docs/sphinx/pypi-dependencies.md

@@ -27,6 +27,18 @@ use_repo(pip, "my_deps")
 For more documentation, including how the rules can update/create a requirements
 file, see the bzlmod examples under the {gh-path}`examples` folder.
 
+We are using a host-platform compatible toolchain by default to setup pip dependencies.
+During the setup phase, we create some symlinks, which may be inefficient on Windows
+by default. In that case use the following `.bazelrc` options to improve performance if
+you have admin privileges:
+```
+startup --windows_enable_symlinks
+```
+
+This will enable symlinks on Windows and help with bootstrap performance of setting up the 
+hermetic host python interpreter on this platform. Linux and OSX users should see no
+difference.
+
 ### Using a WORKSPACE file
 
 To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and

examples/build_file_generation/.bazelrc

@@ -2,7 +2,6 @@ test --test_output=errors --enable_runfiles
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # The bzlmod version of this example is in examples/bzlmod_build_file_generation
 # Once WORKSPACE support is dropped, this example can be entirely deleted.

examples/bzlmod/.bazelrc

@@ -6,5 +6,3 @@ test --test_output=errors --enable_runfiles
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-
-startup --windows_enable_symlinks

examples/bzlmod_build_file_generation/.bazelrc

@@ -2,7 +2,6 @@ test --test_output=errors --enable_runfiles
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 common --experimental_enable_bzlmod
 

examples/multi_python_versions/.bazelrc

@@ -2,6 +2,5 @@ test --test_output=errors
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 coverage --java_runtime_version=remotejdk_11

examples/pip_parse_vendored/.bazelrc

@@ -2,7 +2,6 @@ test --test_output=errors
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # Vendoring requirements.bzl files isn't necessary under bzlmod
 # When workspace support is dropped, this example can be removed.

gazelle/.bazelrc

@@ -10,7 +10,6 @@ build --incompatible_default_to_explicit_init_py
 
 # Windows makes use of runfiles for some rules
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # Do NOT implicitly create empty __init__.py files in the runfiles tree.
 # By default, these are created in every directory containing Python source code

python/private/repo_utils.bzl

@@ -223,7 +223,7 @@ def _outputs_to_str(result):
 repo_utils = struct(
     execute_checked = _execute_checked,
     execute_unchecked = _execute_unchecked,
-    execute_check_stdout = _execute_checked_stdout,
+    execute_checked_stdout = _execute_checked_stdout,
     is_repo_debug_enabled = _is_repo_debug_enabled,
     debug_print = _debug_print,
     which_checked = _which_checked,

python/private/toolchains_repo.bzl

@@ -238,15 +238,64 @@ exports_files(["python"], visibility = ["//visibility:public"])
 
     (os_name, arch) = get_host_os_arch(rctx)
     host_platform = get_host_platform(os_name, arch)
-    host_python = rctx.path(
-        Label(
-            "@@{py_repository}_{host_platform}//:python".format(
-                py_repository = rctx.attr.name[:-len("_host")],
-                host_platform = host_platform,
-            ),
-        ),
+    repo = "@@{py_repository}_{host_platform}".format(
+        py_repository = rctx.attr.name[:-len("_host")],
+        host_platform = host_platform,
+    )
+
+    rctx.report_progress("Symlinking interpreter files to the target platform")
+    host_python_repo = rctx.path(Label("{repo}//:BUILD.bazel".format(repo = repo)))
+
+    # The interpreter might not work on platfroms that don't have symlink support if
+    # we just symlink the interpreter itself. rctx.symlink does a copy in such cases
+    # so we can just attempt to symlink all of the directories in the host interpreter
+    # repo, which should be faster than re-downloading it.
+    for p in host_python_repo.dirname.readdir():
+        if p.basename in [
+            # ignore special files created by the repo rule automatically
+            "BUILD.bazel",
+            "MODULE.bazel",
+            "REPO.bazel",
+            "WORKSPACE",
+            "WORKSPACE.bazel",
+            "WORKSPACE.bzlmod",
+        ]:
+            continue
+
+        # symlink works on all platforms that bazel supports, so it should work on
+        # UNIX and Windows with and without symlink support. For better performance
+        # users should enable the symlink startup option, however that requires admin
+        # privileges.
+        rctx.symlink(p, p.basename)
+
+    is_windows = (os_name == WINDOWS_NAME)
+    python_binary = "python.exe" if is_windows else "python"
+
+    # Ensure that we can run the interpreter and check that we are not
+    # using the host interpreter.
+    python_tester_contents = """\
+from pathlib import Path
+import sys
+
+python = Path(sys.executable)
+want_python = str(Path("{python}").resolve())
+got_python = str(Path(sys.executable).resolve())
+
+assert want_python == got_python, \
+    "Expected to use a different interpreter:\\nwant: '{{}}'\\n got: '{{}}'".format(
+        want_python,
+        got_python,
+    )
+""".format(repo = repo.strip("@"), python = python_binary)
+    python_tester = rctx.path("python_tester.py")
+    rctx.file(python_tester, python_tester_contents)
+    repo_utils.execute_checked(
+        rctx,
+        op = "CheckHostInterpreter",
+        arguments = [rctx.path(python_binary), python_tester],
     )
-    rctx.symlink(host_python, "python")
+    if not rctx.delete(python_tester):
+        fail("Failed to delete the python tester")
 
 host_toolchain = repository_rule(
     _host_toolchain_impl,

tests/integration/compile_pip_requirements/.bazelrc

@@ -2,4 +2,3 @@ test --test_output=errors
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks

tests/integration/ignore_root_user_error/.bazelrc

@@ -4,4 +4,3 @@ test --test_output=errors
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks

tests/integration/pip_parse/.bazelrc

@@ -1,7 +1,6 @@
 # Bazel configuration flags
 
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
 try-import %workspace%/user.bazelrc

tests/integration/pip_repository_entry_points/.bazelrc

@@ -1,7 +1,6 @@
 # Bazel configuration flags
 
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
 try-import %workspace%/user.bazelrc