Merge remote-tracking branch 'upstream/main' into use-com_google_prot…

…obuf

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

CHANGELOG.md

@@ -70,13 +70,16 @@ Unreleased changes template.
   bzlmod extension.
 * (bzlmod) `pip.parse.parse_all_requirements_files` attribute has been removed.
   See notes in the previous versions about what to do.
+* (deps) rules_cc 0.0.16 -- this rules_cc version requires protobuf to be
+  setup.
+* (deps) protobuf 29.0-rc2 (workspace; bzlmod already specifying that version)
 
 Other changes:
 * (python_repository) Start honoring the `strip_prefix` field for `zstd` archives.
 * (pypi) {bzl:obj}`pip_parse.extra_hub_aliases` now works in WORKSPACE files.
-* (deps) rules_cc 0.0.16
 * (deps) bazel_skylib 1.7.0 (workspace; bzlmod already specifying that version)
-* (deps) protobuf 29.0-rc2 (workspace; bzlmod already specifying that version)
+* (binaries/tests) For {obj}`--bootstrap_impl=script`, a binary-specific (but
+  otherwise empty) virtual env is used to customize `sys.path` initialization.
 
 {#v0-0-0-fixed}
 ### Fixed
@@ -86,6 +89,9 @@ Other changes:
   Fixes ([2337](https://github.com/bazelbuild/rules_python/issues/2337)).
 * (uv): Correct the sha256sum for the `uv` binary for aarch64-apple-darwin.
   Fixes ([2411](https://github.com/bazelbuild/rules_python/issues/2411)).
+* (binaries/tests) ({obj}`--bootstrap_impl=scipt`) Using `sys.executable` will
+  use the same `sys.path` setup as the calling binary.
+  ([2169](https://github.com/bazelbuild/rules_python/issues/2169)).
 
 {#v0-0-0-added}
 ### Added
@@ -100,6 +106,9 @@ Other changes:
   for the latest toolchain versions for each minor Python version. You can control
   the toolchain selection by using the
   {bzl:obj}`//python/config_settings:py_linux_libc` build flag.
+* (providers) Added {obj}`py_runtime_info.site_init_template` and
+  {obj}`PyRuntimeInfo.site_init_template` for specifying the template to use to
+  initialize the interpreter via venv startup hooks.
 
 {#v0-0-0-removed}
 ### Removed

python/private/BUILD.bazel

@@ -702,6 +702,14 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 
+filegroup(
+    name = "site_init_template",
+    srcs = ["site_init_template.py"],
+    # Not actually public. Only public because it's an implicit dependency of
+    # py_runtime.
+    visibility = ["//visibility:public"],
+)
+
 # NOTE: Windows builds don't use this bootstrap. Instead, a native Windows
 # program locates some Python exe and runs `python.exe foo.zip` which
 # runs the __main__.py in the zip file.

python/private/py_executable_bazel.bzl

@@ -81,6 +81,9 @@ the `srcs` of Python targets as required.
         "_py_toolchain_type": attr.label(
             default = TARGET_TOOLCHAIN_TYPE,
         ),
+        "_python_version_flag": attr.label(
+            default = "//python/config_settings:python_version",
+        ),
         "_windows_launcher_maker": attr.label(
             default = "@bazel_tools//tools/launcher:launcher_maker",
             cfg = "exec",
@@ -177,13 +180,22 @@ def _create_executable(
     else:
         base_executable_name = executable.basename
 
+    venv = None
+
     # The check for stage2_bootstrap_template is to support legacy
     # BuiltinPyRuntimeInfo providers, which is likely to come from
     # @bazel_tools//tools/python:autodetecting_toolchain, the toolchain used
     # for workspace builds when no rules_python toolchain is configured.
     if (BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT and
         runtime_details.effective_runtime and
         hasattr(runtime_details.effective_runtime, "stage2_bootstrap_template")):
+        venv = _create_venv(
+            ctx,
+            output_prefix = base_executable_name,
+            imports = imports,
+            runtime_details = runtime_details,
+        )
+
         stage2_bootstrap = _create_stage2_bootstrap(
             ctx,
             output_prefix = base_executable_name,
@@ -192,11 +204,12 @@ def _create_executable(
             imports = imports,
             runtime_details = runtime_details,
         )
-        extra_runfiles = ctx.runfiles([stage2_bootstrap])
+        extra_runfiles = ctx.runfiles([stage2_bootstrap] + venv.files_without_interpreter)
         zip_main = _create_zip_main(
             ctx,
             stage2_bootstrap = stage2_bootstrap,
             runtime_details = runtime_details,
+            venv = venv,
         )
     else:
         stage2_bootstrap = None
@@ -272,6 +285,7 @@ def _create_executable(
             zip_file = zip_file,
             stage2_bootstrap = stage2_bootstrap,
             runtime_details = runtime_details,
+            venv = venv,
         )
     elif bootstrap_output:
         _create_stage1_bootstrap(
@@ -282,6 +296,7 @@ def _create_executable(
             is_for_zip = False,
             imports = imports,
             main_py = main_py,
+            venv = venv,
         )
     else:
         # Otherwise, this should be the Windows case of launcher + zip.
@@ -296,13 +311,20 @@ def _create_executable(
                 build_zip_enabled = build_zip_enabled,
             ))
 
+    # The interpreter is added this late in the process so that it isn't
+    # added to the zipped files.
+    if venv:
+        extra_runfiles = extra_runfiles.merge(ctx.runfiles([venv.interpreter]))
     return create_executable_result_struct(
         extra_files_to_build = depset(extra_files_to_build),
         output_groups = {"python_zip_file": depset([zip_file])},
         extra_runfiles = extra_runfiles,
     )
 
-def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details):
+def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv):
+    python_binary = _runfiles_root_path(ctx, venv.interpreter.short_path)
+    python_binary_actual = _runfiles_root_path(ctx, venv.interpreter_actual_path)
+
     # The location of this file doesn't really matter. It's added to
     # the zip file as the top-level __main__.py file and not included
     # elsewhere.
@@ -311,7 +333,8 @@ def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details):
         template = runtime_details.effective_runtime.zip_main_template,
         output = output,
         substitutions = {
-            "%python_binary%": runtime_details.executable_interpreter_path,
+            "%python_binary%": python_binary,
+            "%python_binary_actual%": python_binary_actual,
             "%stage2_bootstrap%": "{}/{}".format(
                 ctx.workspace_name,
                 stage2_bootstrap.short_path,
@@ -321,6 +344,82 @@ def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details):
     )
     return output
 
+# Create a venv the executable can use.
+# For venv details and the venv startup process, see:
+# * https://docs.python.org/3/library/venv.html
+# * https://snarky.ca/how-virtual-environments-work/
+# * https://github.com/python/cpython/blob/main/Modules/getpath.py
+# * https://github.com/python/cpython/blob/main/Lib/site.py
+def _create_venv(ctx, output_prefix, imports, runtime_details):
+    venv = "_{}.venv".format(output_prefix.lstrip("_"))
+
+    # The pyvenv.cfg file must be present to trigger the venv site hooks.
+    # Because it's paths are expected to be absolute paths, we can't reliably
+    # put much in it. See https://github.com/python/cpython/issues/83650
+    pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv))
+    ctx.actions.write(pyvenv_cfg, "")
+
+    runtime = runtime_details.effective_runtime
+    if runtime.interpreter:
+        py_exe_basename = paths.basename(runtime.interpreter.short_path)
+
+        # Even though ctx.actions.symlink() is used, using
+        # declare_symlink() is required to ensure that the resulting file
+        # in runfiles is always a symlink. An RBE implementation, for example,
+        # may choose to write what symlink() points to instead.
+        interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename))
+        interpreter_actual_path = runtime.interpreter.short_path
+        parent = "/".join([".."] * (interpreter_actual_path.count("/") + 1))
+        rel_path = parent + "/" + interpreter_actual_path
+        ctx.actions.symlink(output = interpreter, target_path = rel_path)
+    else:
+        py_exe_basename = paths.basename(runtime.interpreter_path)
+        interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename))
+        ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path)
+        interpreter_actual_path = runtime.interpreter_path
+
+    if runtime.interpreter_version_info:
+        version = "{}.{}".format(
+            runtime.interpreter_version_info.major,
+            runtime.interpreter_version_info.minor,
+        )
+    else:
+        version_flag = ctx.attr._python_version_flag[config_common.FeatureFlagInfo].value
+        version_flag_parts = version_flag.split(".")[0:2]
+        version = "{}.{}".format(*version_flag_parts)
+
+    # See site.py logic: free-threaded builds append "t" to the venv lib dir name
+    if "t" in runtime.abi_flags:
+        version += "t"
+
+    site_packages = "{}/lib/python{}/site-packages".format(venv, version)
+    pth = ctx.actions.declare_file("{}/bazel.pth".format(site_packages))
+    ctx.actions.write(pth, "import _bazel_site_init\n")
+
+    site_init = ctx.actions.declare_file("{}/_bazel_site_init.py".format(site_packages))
+    computed_subs = ctx.actions.template_dict()
+    computed_subs.add_joined("%imports%", imports, join_with = ":", map_each = _map_each_identity)
+    ctx.actions.expand_template(
+        template = runtime.site_init_template,
+        output = site_init,
+        substitutions = {
+            "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False",
+            "%site_init_runfiles_path%": "{}/{}".format(ctx.workspace_name, site_init.short_path),
+            "%workspace_name%": ctx.workspace_name,
+        },
+        computed_substitutions = computed_subs,
+    )
+
+    return struct(
+        interpreter = interpreter,
+        # Runfiles-relative path or absolute path
+        interpreter_actual_path = interpreter_actual_path,
+        files_without_interpreter = [pyvenv_cfg, pth, site_init],
+    )
+
+def _map_each_identity(v):
+    return v
+
 def _create_stage2_bootstrap(
         ctx,
         *,
@@ -363,6 +462,13 @@ def _create_stage2_bootstrap(
     )
     return output
 
+def _runfiles_root_path(ctx, path):
+    # The ../ comes from short_path for files in other repos.
+    if path.startswith("../"):
+        return path[3:]
+    else:
+        return "{}/{}".format(ctx.workspace_name, path)
+
 def _create_stage1_bootstrap(
         ctx,
         *,
@@ -371,12 +477,24 @@ def _create_stage1_bootstrap(
         stage2_bootstrap = None,
         imports = None,
         is_for_zip,
-        runtime_details):
+        runtime_details,
+        venv = None):
     runtime = runtime_details.effective_runtime
 
+    if venv:
+        python_binary_path = _runfiles_root_path(ctx, venv.interpreter.short_path)
+    else:
+        python_binary_path = runtime_details.executable_interpreter_path
+
+    if is_for_zip and venv:
+        python_binary_actual = _runfiles_root_path(ctx, venv.interpreter_actual_path)
+    else:
+        python_binary_actual = ""
+
     subs = {
         "%is_zipfile%": "1" if is_for_zip else "0",
-        "%python_binary%": runtime_details.executable_interpreter_path,
+        "%python_binary%": python_binary_path,
+        "%python_binary_actual%": python_binary_actual,
         "%target%": str(ctx.label),
         "%workspace_name%": ctx.workspace_name,
     }
@@ -447,6 +565,7 @@ def _create_windows_exe_launcher(
     )
 
 def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfiles):
+    """Create a Python zipapp (zip with __main__.py entry point)."""
     workspace_name = ctx.workspace_name
     legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx)
 
@@ -524,7 +643,14 @@ def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles):
         zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path))
     return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path)
 
-def _create_executable_zip_file(ctx, *, output, zip_file, stage2_bootstrap, runtime_details):
+def _create_executable_zip_file(
+        ctx,
+        *,
+        output,
+        zip_file,
+        stage2_bootstrap,
+        runtime_details,
+        venv):
     prelude = ctx.actions.declare_file(
         "{}_zip_prelude.sh".format(output.basename),
         sibling = output,
@@ -536,6 +662,7 @@ def _create_executable_zip_file(ctx, *, output, zip_file, stage2_bootstrap, runt
             stage2_bootstrap = stage2_bootstrap,
             runtime_details = runtime_details,
             is_for_zip = True,
+            venv = venv,
         )
     else:
         ctx.actions.write(prelude, "#!/usr/bin/env python3\n")