Change package query order when custom index URLs are given

CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [0.4.1] - 2023/09/XX
+
+### Changed
+
+- When custom index URLs are set by `micropip.set_index_urls` or by `micropip.install(index_urls=...)`,
+  micropip will now query packages from the custom index first,
+  and then from pyodide lockfile.
+  [#83](https://github.com/pyodide/micropip/pull/83)
+
 ## [0.4.0] - 2023/07/25
 
 ### Added

micropip/_commands/install.py

@@ -4,6 +4,7 @@
 
 from packaging.markers import default_environment
 
+from .. import package_index
 from .._compat import loadPackage, to_js
 from ..constants import FAQ_URLS
 from ..logging import setup_logging
@@ -125,6 +126,9 @@ async def install(
 
     wheel_base = Path(getsitepackages()[0])
 
+    if index_urls is None:
+        index_urls = package_index.INDEX_URLS
+
     transaction = Transaction(
         ctx=ctx,
         ctx_extras=[],

micropip/transaction.py

@@ -186,6 +186,11 @@ class Transaction:
 
     verbose: bool | int = False
 
+    def __post_init__(self):
+        # If index_urls is None, pyodide-lock.json have to be searched first.
+        # TODO: when PyPI starts to support hosting WASM wheels, this might be deleted.
+        self.search_pyodide_lock_first = self.index_urls is None
+
     async def gather_requirements(
         self,
         requirements: list[str],
@@ -285,17 +290,41 @@ def eval_marker(e: dict[str, str]) -> bool:
             logger.info(f"Requirement already satisfied: {req} ({ver})")
             return
 
-        # If there's a Pyodide package that matches the version constraint, use
-        # the Pyodide package instead of the one on PyPI
+        if self.search_pyodide_lock_first:
+            if self._add_requirement_from_pyodide_lock(req):
+                return
+
+            await self._add_requirement_from_package_index(req)
+        else:
+            try:
+                await self._add_requirement_from_package_index(req)
+            except ValueError:
+                # If the requirement is not found in package index,
+                # we still have a chance to find it from pyodide lockfile.
+                if not self._add_requirement_from_pyodide_lock(req):
+                    raise
+
+    def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool:
+        """
+        Find requirement from pyodide-lock.json. If the requirement is found,
+        add it to the package list and return True. Otherwise, return False.
+        """
         if req.name in REPODATA_PACKAGES and req.specifier.contains(
             REPODATA_PACKAGES[req.name]["version"], prereleases=True
         ):
             version = REPODATA_PACKAGES[req.name]["version"]
             self.pyodide_packages.append(
                 PackageMetadata(name=req.name, version=str(version), source="pyodide")
             )
-            return
+            return True
 
+        return False
+
+    async def _add_requirement_from_package_index(self, req: Requirement) -> bool:
+        """
+        Find requirement from package index. If the requirement is found,
+        add it to the package list and return True. Otherwise, return False.
+        """
         metadata = await package_index.query_package(
             req.name, self.fetch_kwargs, index_urls=self.index_urls
         )
@@ -307,16 +336,17 @@ def eval_marker(e: dict[str, str]) -> bool:
             if not self.keep_going:
                 raise
             else:
-                return
+                return False
 
         # Maybe while we were downloading pypi_json some other branch
         # installed the wheel?
         satisfied, ver = self.check_version_satisfied(req)
         if satisfied:
             logger.info(f"Requirement already satisfied: {req} ({ver})")
-            return
+            return True
 
         await self.add_wheel(wheel, req.extras, specifier=str(req.specifier))
+        return True
 
     async def add_wheel(
         self,

pyproject.toml

@@ -19,7 +19,7 @@ test = [
   "pytest-httpserver",
   "pytest-pyodide",
   "pytest-cov",
-  "build",
+  "build==0.7.0",
 ]
 
 

tests/test_transaction.py

@@ -292,3 +292,67 @@ def test_last_version_and_best_tag_from_pypi(
     wheel = find_wheel(metadata, requirement)
 
     assert str(wheel.version) == new_version
+
+
+def test_search_pyodide_lock_first():
+    from micropip.transaction import Transaction
+
+    t = Transaction(
+        ctx={},
+        ctx_extras=[],
+        keep_going=True,
+        deps=True,
+        pre=True,
+        fetch_kwargs={},
+        verbose=False,
+        index_urls=None,
+    )
+    assert t.search_pyodide_lock_first is True
+
+    t = Transaction(
+        ctx={},
+        ctx_extras=[],
+        keep_going=True,
+        deps=True,
+        pre=True,
+        fetch_kwargs={},
+        verbose=False,
+        index_urls=["https://my.custom.index.com"],
+    )
+    assert t.search_pyodide_lock_first is False
+
+
+@pytest.mark.asyncio
+async def test_index_url_priority(
+    mock_importlib, wheel_base, monkeypatch, mock_package_index_simple_json_api
+):
+    # Test that if the index_urls are provided, package should be searched in
+    # the index_urls first before searching in Pyodide lock file.
+    from micropip.transaction import Transaction
+
+    # add_wheel is called only when the package is found in the index_urls
+    add_wheel_called = None
+
+    async def mock_add_wheel(self, wheel, extras, *, specifier=""):
+        nonlocal add_wheel_called
+        add_wheel_called = wheel
+
+    monkeypatch.setattr(Transaction, "add_wheel", mock_add_wheel)
+
+    mock_index_url = mock_package_index_simple_json_api(pkgs=["black"])
+
+    t = Transaction(
+        keep_going=True,
+        deps=False,
+        pre=False,
+        ctx={},
+        ctx_extras=[],
+        fetch_kwargs={},
+        index_urls=mock_index_url,
+    )
+
+    await t.add_requirement("black")
+    assert add_wheel_called is not None
+    assert add_wheel_called.name == "black"
+    # 23.7.0 is the latest version of black in the mock index
+    assert str(add_wheel_called.version) == "23.7.0"