[DRAFT] Use alternative index url when installing build dependencies

pyodide_build/build_env.py

@@ -16,7 +16,6 @@
 from pyodide_build import __version__
 from pyodide_build.common import search_pyproject_toml, to_bool, xbuildenv_dirname
 from pyodide_build.config import ConfigManager
-from pyodide_build.recipe import load_all_recipes
 
 RUST_BUILD_PRELUDE = """
 rustup toolchain install ${RUST_TOOLCHAIN} && rustup default ${RUST_TOOLCHAIN}
@@ -170,25 +169,6 @@ def get_hostsitepackages() -> str:
     return get_build_flag("HOSTSITEPACKAGES")
 
 
-@functools.cache
-def get_unisolated_packages() -> list[str]:
-    PYODIDE_ROOT = get_pyodide_root()
-
-    unisolated_file = PYODIDE_ROOT / "unisolated.txt"
-    if unisolated_file.exists():
-        # in xbuild env, read from file
-        unisolated_packages = unisolated_file.read_text().splitlines()
-    else:
-        unisolated_packages = []
-        recipe_dir = PYODIDE_ROOT / "packages"
-        recipes = load_all_recipes(recipe_dir)
-        for name, config in recipes.items():
-            if config.build.cross_build_env:
-                unisolated_packages.append(name)
-
-    return unisolated_packages
-
-
 def platform() -> str:
     emscripten_version = get_build_flag("PYODIDE_EMSCRIPTEN_VERSION")
     version = emscripten_version.replace(".", "_")

pyodide_build/buildpkg.py

@@ -537,15 +537,9 @@ def _package_wheel(
                 / f"lib/{python_dir}/site-packages"
             )
             if self.build_metadata.cross_build_env:
-                subprocess.run(
-                    ["pip", "install", "-t", str(host_site_packages), f"{name}=={ver}"],
-                    check=True,
-                )
-
-            for cross_build_file in self.build_metadata.cross_build_files:
-                shutil.copy(
-                    (wheel_dir / cross_build_file),
-                    host_site_packages / cross_build_file,
+                shutil.copytree(
+                    wheel_dir,
+                    host_site_packages,
                 )
 
             try:

pyodide_build/common.py

@@ -447,3 +447,38 @@ def to_bool(value: str) -> bool:
     Convert a string to a boolean value. Useful for parsing environment variables.
     """
     return value.lower() not in {"", "0", "false", "no", "off"}
+
+
+def get_host_platform():
+    """
+    Return a string that identifies the current platform.
+    Simplified version of get_host_platform in pypa/distlib.
+    """
+    if os.name != "posix":
+        raise ValueError(f"only posix platforms are supported, got {os.name}")
+
+    # Set for cross builds explicitly
+    if "_PYTHON_HOST_PLATFORM" in os.environ:
+        return os.environ["_PYTHON_HOST_PLATFORM"]
+
+    # Try to distinguish various flavours of Unix
+    (osname, host, release, version, machine) = os.uname()
+
+    # Convert the OS name to lowercase, remove '/' characters, and translate
+    # spaces (for "Power Macintosh")
+    osname = osname.lower().replace("/", "")
+    machine = machine.replace(" ", "_").replace("/", "-")
+
+    if osname[:5] == "linux":
+        return f"{osname}-{machine}"
+    elif osname[:6] == "darwin":
+        import _osx_support
+        import sysconfig
+
+        osname, release, machine = _osx_support.get_platform_osx(
+            sysconfig.get_config_vars(), osname, release, machine
+        )
+    else:
+        raise ValueError(f"unsupported os: {osname}")
+
+    return f"{osname}-{release}-{machine}"

pyodide_build/pypabuild.py

@@ -4,9 +4,8 @@
 import subprocess as sp
 import sys
 import traceback
-from collections.abc import Callable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Generator, Iterator, Mapping, Sequence
 from contextlib import contextmanager
-from itertools import chain
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Literal, cast
@@ -18,9 +17,7 @@
 from pyodide_build import _f2c_fixes, common, pywasmcross
 from pyodide_build.build_env import (
     get_build_flag,
-    get_hostsitepackages,
     get_pyversion,
-    get_unisolated_packages,
     platform,
 )
 from pyodide_build.io import _BuildSpecExports
@@ -54,6 +51,8 @@
     "gfortran": "FC",  # https://mesonbuild.com/Reference-tables.html#compiler-and-linker-selection-variables
 }
 
+HOST_ARCH = common.get_host_platform().replace("-", "_").replace(".", "_")
+
 
 def _gen_runner(
     cross_build_env: Mapping[str, str],
@@ -103,13 +102,6 @@ def symlink_unisolated_packages(env: DefaultIsolatedEnv) -> None:
 
     env_site_packages.mkdir(parents=True, exist_ok=True)
     shutil.copy(sysconfigdata_path, env_site_packages)
-    host_site_packages = Path(get_hostsitepackages())
-    for name in get_unisolated_packages():
-        for path in chain(
-            host_site_packages.glob(f"{name}*"), host_site_packages.glob(f"_{name}*")
-        ):
-            (env_site_packages / path.name).unlink(missing_ok=True)
-            (env_site_packages / path.name).symlink_to(path)
 
 
 def remove_avoided_requirements(
@@ -127,7 +119,7 @@ def install_reqs(env: DefaultIsolatedEnv, reqs: set[str]) -> None:
     env.install(
         remove_avoided_requirements(
             reqs,
-            get_unisolated_packages() + AVOIDED_REQUIREMENTS,
+            AVOIDED_REQUIREMENTS,
         )
     )
 
@@ -153,8 +145,25 @@ def _build_in_isolated_env(
 
         # first install the build dependencies
         symlink_unisolated_packages(env)
-        install_reqs(env, builder.build_system_requires)
-        installed_requires_for_build = False
+        index_url_for_cross_build = get_build_flag("BUILD_DEPENDENCY_INDEX_URL")
+        installed_build_system_requires = False  # build depdency for in pyproject.toml
+        installed_backend_requires = (
+            False  # dependencies defined by the backend for a given distribution
+        )
+
+        with switch_index_url(index_url_for_cross_build):
+            try:
+                install_reqs(env, builder.build_system_requires)
+                installed_build_system_requires = True
+            except Exception:
+                print(
+                    f"Failed to install build dependencies from {index_url_for_cross_build}, falling back to default index url"
+                )
+
+        # Disabled for testing
+        # if not installed_build_system_requires:
+        #     install_reqs(env, builder.build_system_requires)
+
         try:
             build_reqs = builder.get_requires_for_build(
                 distribution,
@@ -163,10 +172,10 @@ def _build_in_isolated_env(
             pass
         else:
             install_reqs(env, build_reqs)
-            installed_requires_for_build = True
+            installed_backend_requires = True
 
         with common.replace_env(build_env):
-            if not installed_requires_for_build:
+            if not installed_backend_requires:
                 build_reqs = builder.get_requires_for_build(
                     distribution,
                     config_settings,
@@ -242,6 +251,31 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]:
     return env
 
 
+@contextmanager
+def switch_index_url(index_url: str) -> Generator[None, None, None]:
+    """
+    Switch index URL that pip locates the packages.
+    This function is expected to be used during the process of
+    installing package build dependencies.
+
+    Parameters
+    ----------
+    index_url: index URL to switch to
+    """
+
+    env = {
+        "PIP_INDEX_URL": index_url,
+        "PIP_PLATFORM": " ".join(
+            [
+                f"pyodide_{get_build_flag("PYODIDE_ABI_VERSION")}_wasm32",
+                HOST_ARCH,
+            ]
+        ),
+    }
+
+    yield common.replace_env(env)
+
+
 @contextmanager
 def get_build_env(
     env: dict[str, str],

pyodide_build/tests/conftest.py

@@ -62,7 +62,6 @@ def reset_cache():
     def _reset():
         build_env.get_pyodide_root.cache_clear()
         build_env.get_build_environment_vars.cache_clear()
-        build_env.get_unisolated_packages.cache_clear()
 
     _reset()