From b607b06bc38f7b6f5346692b006990b0245b6417 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Tue, 19 Sep 2023 22:39:05 +0900 Subject: [PATCH] Change package query order when custom index URLs are given (#83) --- CHANGELOG.md | 9 +++++ micropip/_commands/index_urls.py | 2 +- micropip/_commands/install.py | 4 ++ micropip/transaction.py | 52 +++++++++++++++++++------ pyproject.toml | 2 +- tests/test_transaction.py | 65 ++++++++++++++++++++++++++++++++ 6 files changed, 120 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3748109..5c430e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## [0.5.0] - 2023/09/19 + +### Changed + +- When custom index URLs are set by `micropip.set_index_urls` or by `micropip.install(index_urls=...)`, + micropip will now query packages from the custom index first, + and then from pyodide lockfile. + [#83](https://github.com/pyodide/micropip/pull/83) + - Made micropip.freeze correctly list dependencies of manually installed packages. [#79](https://github.com/pyodide/micropip/pull/79) diff --git a/micropip/_commands/index_urls.py b/micropip/_commands/index_urls.py index 60cc9b1..aae9cce 100644 --- a/micropip/_commands/index_urls.py +++ b/micropip/_commands/index_urls.py @@ -24,4 +24,4 @@ def set_index_urls(urls: list[str] | str) -> None: if isinstance(urls, str): urls = [urls] - package_index.INDEX_URLS = urls + package_index.INDEX_URLS = urls[:] diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index dc856ec..bc93c1f 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -4,6 +4,7 @@ from packaging.markers import default_environment +from .. import package_index from .._compat import loadPackage, to_js from ..constants import FAQ_URLS from ..logging import setup_logging @@ -125,6 +126,9 @@ async def install( wheel_base = Path(getsitepackages()[0]) + if index_urls is None: + index_urls = package_index.INDEX_URLS[:] + transaction = Transaction( ctx=ctx, ctx_extras=[], diff --git a/micropip/transaction.py b/micropip/transaction.py index 431612b..c2fffbd 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -186,6 +186,13 @@ class Transaction: verbose: bool | int = False + def __post_init__(self): + # If index_urls is None, pyodide-lock.json have to be searched first. + # TODO: when PyPI starts to support hosting WASM wheels, this might be deleted. + self.search_pyodide_lock_first = ( + self.index_urls == package_index.DEFAULT_INDEX_URLS + ) + async def gather_requirements( self, requirements: list[str], @@ -285,8 +292,30 @@ def eval_marker(e: dict[str, str]) -> bool: logger.info(f"Requirement already satisfied: {req} ({ver})") return - # If there's a Pyodide package that matches the version constraint, use - # the Pyodide package instead of the one on PyPI + try: + if self.search_pyodide_lock_first: + if self._add_requirement_from_pyodide_lock(req): + return + + await self._add_requirement_from_package_index(req) + else: + try: + await self._add_requirement_from_package_index(req) + except ValueError: + # If the requirement is not found in package index, + # we still have a chance to find it from pyodide lockfile. + if not self._add_requirement_from_pyodide_lock(req): + raise + except ValueError: + self.failed.append(req) + if not self.keep_going: + raise + + def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: + """ + Find requirement from pyodide-lock.json. If the requirement is found, + add it to the package list and return True. Otherwise, return False. + """ if req.name in REPODATA_PACKAGES and req.specifier.contains( REPODATA_PACKAGES[req.name]["version"], prereleases=True ): @@ -294,27 +323,26 @@ def eval_marker(e: dict[str, str]) -> bool: self.pyodide_packages.append( PackageMetadata(name=req.name, version=str(version), source="pyodide") ) - return + return True + + return False + async def _add_requirement_from_package_index(self, req: Requirement): + """ + Find requirement from package index. If the requirement is found, + add it to the package list and return True. Otherwise, return False. + """ metadata = await package_index.query_package( req.name, self.fetch_kwargs, index_urls=self.index_urls ) - try: - wheel = find_wheel(metadata, req) - except ValueError: - self.failed.append(req) - if not self.keep_going: - raise - else: - return + wheel = find_wheel(metadata, req) # Maybe while we were downloading pypi_json some other branch # installed the wheel? satisfied, ver = self.check_version_satisfied(req) if satisfied: logger.info(f"Requirement already satisfied: {req} ({ver})") - return await self.add_wheel(wheel, req.extras, specifier=str(req.specifier)) diff --git a/pyproject.toml b/pyproject.toml index fd5f54b..463f537 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ test = [ "pytest-httpserver", "pytest-pyodide", "pytest-cov", - "build<1.0.0", + "build==0.7.0", ] diff --git a/tests/test_transaction.py b/tests/test_transaction.py index b22dddf..b2ebb1d 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -292,3 +292,68 @@ def test_last_version_and_best_tag_from_pypi( wheel = find_wheel(metadata, requirement) assert str(wheel.version) == new_version + + +def test_search_pyodide_lock_first(): + from micropip import package_index + from micropip.transaction import Transaction + + t = Transaction( + ctx={}, + ctx_extras=[], + keep_going=True, + deps=True, + pre=True, + fetch_kwargs={}, + verbose=False, + index_urls=package_index.DEFAULT_INDEX_URLS, + ) + assert t.search_pyodide_lock_first is True + + t = Transaction( + ctx={}, + ctx_extras=[], + keep_going=True, + deps=True, + pre=True, + fetch_kwargs={}, + verbose=False, + index_urls=["https://my.custom.index.com"], + ) + assert t.search_pyodide_lock_first is False + + +@pytest.mark.asyncio +async def test_index_url_priority( + mock_importlib, wheel_base, monkeypatch, mock_package_index_simple_json_api +): + # Test that if the index_urls are provided, package should be searched in + # the index_urls first before searching in Pyodide lock file. + from micropip.transaction import Transaction + + # add_wheel is called only when the package is found in the index_urls + add_wheel_called = None + + async def mock_add_wheel(self, wheel, extras, *, specifier=""): + nonlocal add_wheel_called + add_wheel_called = wheel + + monkeypatch.setattr(Transaction, "add_wheel", mock_add_wheel) + + mock_index_url = mock_package_index_simple_json_api(pkgs=["black"]) + + t = Transaction( + keep_going=True, + deps=False, + pre=False, + ctx={}, + ctx_extras=[], + fetch_kwargs={}, + index_urls=mock_index_url, + ) + + await t.add_requirement("black") + assert add_wheel_called is not None + assert add_wheel_called.name == "black" + # 23.7.0 is the latest version of black in the mock index + assert str(add_wheel_called.version) == "23.7.0"