Change package query order when custom index URLs are given (#83)

CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [0.5.0] - 2023/09/19
+
+### Changed
+
+- When custom index URLs are set by `micropip.set_index_urls` or by `micropip.install(index_urls=...)`,
+  micropip will now query packages from the custom index first,
+  and then from pyodide lockfile.
+  [#83](https://github.com/pyodide/micropip/pull/83)
+
 - Made micropip.freeze correctly list dependencies of manually installed packages.
   [#79](https://github.com/pyodide/micropip/pull/79)
 

micropip/_commands/index_urls.py

@@ -24,4 +24,4 @@ def set_index_urls(urls: list[str] | str) -> None:
     if isinstance(urls, str):
         urls = [urls]
 
-    package_index.INDEX_URLS = urls
+    package_index.INDEX_URLS = urls[:]

micropip/_commands/install.py

@@ -4,6 +4,7 @@
 
 from packaging.markers import default_environment
 
+from .. import package_index
 from .._compat import loadPackage, to_js
 from ..constants import FAQ_URLS
 from ..logging import setup_logging
@@ -125,6 +126,9 @@ async def install(
 
     wheel_base = Path(getsitepackages()[0])
 
+    if index_urls is None:
+        index_urls = package_index.INDEX_URLS[:]
+
     transaction = Transaction(
         ctx=ctx,
         ctx_extras=[],

micropip/transaction.py

@@ -186,6 +186,13 @@ class Transaction:
 
     verbose: bool | int = False
 
+    def __post_init__(self):
+        # If index_urls is None, pyodide-lock.json have to be searched first.
+        # TODO: when PyPI starts to support hosting WASM wheels, this might be deleted.
+        self.search_pyodide_lock_first = (
+            self.index_urls == package_index.DEFAULT_INDEX_URLS
+        )
+
     async def gather_requirements(
         self,
         requirements: list[str],
@@ -285,36 +292,57 @@ def eval_marker(e: dict[str, str]) -> bool:
             logger.info(f"Requirement already satisfied: {req} ({ver})")
             return
 
-        # If there's a Pyodide package that matches the version constraint, use
-        # the Pyodide package instead of the one on PyPI
+        try:
+            if self.search_pyodide_lock_first:
+                if self._add_requirement_from_pyodide_lock(req):
+                    return
+
+                await self._add_requirement_from_package_index(req)
+            else:
+                try:
+                    await self._add_requirement_from_package_index(req)
+                except ValueError:
+                    # If the requirement is not found in package index,
+                    # we still have a chance to find it from pyodide lockfile.
+                    if not self._add_requirement_from_pyodide_lock(req):
+                        raise
+        except ValueError:
+            self.failed.append(req)
+            if not self.keep_going:
+                raise
+
+    def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool:
+        """
+        Find requirement from pyodide-lock.json. If the requirement is found,
+        add it to the package list and return True. Otherwise, return False.
+        """
         if req.name in REPODATA_PACKAGES and req.specifier.contains(
             REPODATA_PACKAGES[req.name]["version"], prereleases=True
         ):
             version = REPODATA_PACKAGES[req.name]["version"]
             self.pyodide_packages.append(
                 PackageMetadata(name=req.name, version=str(version), source="pyodide")
             )
-            return
+            return True
+
+        return False
 
+    async def _add_requirement_from_package_index(self, req: Requirement):
+        """
+        Find requirement from package index. If the requirement is found,
+        add it to the package list and return True. Otherwise, return False.
+        """
         metadata = await package_index.query_package(
             req.name, self.fetch_kwargs, index_urls=self.index_urls
         )
 
-        try:
-            wheel = find_wheel(metadata, req)
-        except ValueError:
-            self.failed.append(req)
-            if not self.keep_going:
-                raise
-            else:
-                return
+        wheel = find_wheel(metadata, req)
 
         # Maybe while we were downloading pypi_json some other branch
         # installed the wheel?
         satisfied, ver = self.check_version_satisfied(req)
         if satisfied:
             logger.info(f"Requirement already satisfied: {req} ({ver})")
-            return
 
         await self.add_wheel(wheel, req.extras, specifier=str(req.specifier))
 

pyproject.toml

@@ -19,7 +19,7 @@ test = [
   "pytest-httpserver",
   "pytest-pyodide",
   "pytest-cov",
-  "build<1.0.0",
+  "build==0.7.0",
 ]
 
 

tests/test_transaction.py

@@ -292,3 +292,68 @@ def test_last_version_and_best_tag_from_pypi(
     wheel = find_wheel(metadata, requirement)
 
     assert str(wheel.version) == new_version
+
+
+def test_search_pyodide_lock_first():
+    from micropip import package_index
+    from micropip.transaction import Transaction
+
+    t = Transaction(
+        ctx={},
+        ctx_extras=[],
+        keep_going=True,
+        deps=True,
+        pre=True,
+        fetch_kwargs={},
+        verbose=False,
+        index_urls=package_index.DEFAULT_INDEX_URLS,
+    )
+    assert t.search_pyodide_lock_first is True
+
+    t = Transaction(
+        ctx={},
+        ctx_extras=[],
+        keep_going=True,
+        deps=True,
+        pre=True,
+        fetch_kwargs={},
+        verbose=False,
+        index_urls=["https://my.custom.index.com"],
+    )
+    assert t.search_pyodide_lock_first is False
+
+
+@pytest.mark.asyncio
+async def test_index_url_priority(
+    mock_importlib, wheel_base, monkeypatch, mock_package_index_simple_json_api
+):
+    # Test that if the index_urls are provided, package should be searched in
+    # the index_urls first before searching in Pyodide lock file.
+    from micropip.transaction import Transaction
+
+    # add_wheel is called only when the package is found in the index_urls
+    add_wheel_called = None
+
+    async def mock_add_wheel(self, wheel, extras, *, specifier=""):
+        nonlocal add_wheel_called
+        add_wheel_called = wheel
+
+    monkeypatch.setattr(Transaction, "add_wheel", mock_add_wheel)
+
+    mock_index_url = mock_package_index_simple_json_api(pkgs=["black"])
+
+    t = Transaction(
+        keep_going=True,
+        deps=False,
+        pre=False,
+        ctx={},
+        ctx_extras=[],
+        fetch_kwargs={},
+        index_urls=mock_index_url,
+    )
+
+    await t.add_requirement("black")
+    assert add_wheel_called is not None
+    assert add_wheel_called.name == "black"
+    # 23.7.0 is the latest version of black in the mock index
+    assert str(add_wheel_called.version) == "23.7.0"