From 5966cfdb5e19f62daf6d796182f6bd7e1cd85783 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Mon, 11 Sep 2023 22:21:41 +0900 Subject: [PATCH 01/12] Search index urls first if given --- micropip/transaction.py | 40 +++++++++++++++++++++--- tests/test_transaction.py | 64 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/micropip/transaction.py b/micropip/transaction.py index 431612b..a09f7ec 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -186,6 +186,11 @@ class Transaction: verbose: bool | int = False + def __post_init__(self): + # If index_urls is None, pyodide-lock.json have to be searched first. + # TODO: when PyPI starts to support hosting WASM wheels, this might be deleted. + self.search_pyodide_lock_first = self.index_urls is None + async def gather_requirements( self, requirements: list[str], @@ -285,8 +290,25 @@ def eval_marker(e: dict[str, str]) -> bool: logger.info(f"Requirement already satisfied: {req} ({ver})") return - # If there's a Pyodide package that matches the version constraint, use - # the Pyodide package instead of the one on PyPI + if self.search_pyodide_lock_first: + if self._add_requirement_from_pyodide_lock(req): + return + + await self._add_requirement_from_package_index(req) + else: + try: + await self._add_requirement_from_package_index(req) + except ValueError: + # If the requirement is not found in package index, + # we still have a chance to find it from pyodide lockfile. + if not self._add_requirement_from_pyodide_lock(req): + raise + + def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: + """ + Find requirement from pyodide-lock.json. If the requirement is found, + add it to the package list and return True. Otherwise, return False. + """ if req.name in REPODATA_PACKAGES and req.specifier.contains( REPODATA_PACKAGES[req.name]["version"], prereleases=True ): @@ -294,8 +316,15 @@ def eval_marker(e: dict[str, str]) -> bool: self.pyodide_packages.append( PackageMetadata(name=req.name, version=str(version), source="pyodide") ) - return + return True + return False + + async def _add_requirement_from_package_index(self, req: Requirement) -> bool: + """ + Find requirement from package index. If the requirement is found, + add it to the package list and return True. Otherwise, return False. + """ metadata = await package_index.query_package( req.name, self.fetch_kwargs, index_urls=self.index_urls ) @@ -307,16 +336,17 @@ def eval_marker(e: dict[str, str]) -> bool: if not self.keep_going: raise else: - return + return False # Maybe while we were downloading pypi_json some other branch # installed the wheel? satisfied, ver = self.check_version_satisfied(req) if satisfied: logger.info(f"Requirement already satisfied: {req} ({ver})") - return + return True await self.add_wheel(wheel, req.extras, specifier=str(req.specifier)) + return True async def add_wheel( self, diff --git a/tests/test_transaction.py b/tests/test_transaction.py index b22dddf..b91b29a 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -292,3 +292,67 @@ def test_last_version_and_best_tag_from_pypi( wheel = find_wheel(metadata, requirement) assert str(wheel.version) == new_version + + +def test_search_pyodide_lock_first(): + from micropip.transaction import Transaction + + t = Transaction( + ctx={}, + ctx_extras=[], + keep_going=True, + deps=True, + pre=True, + fetch_kwargs={}, + verbose=False, + index_urls=None, + ) + assert t.search_pyodide_lock_first is True + + t = Transaction( + ctx={}, + ctx_extras=[], + keep_going=True, + deps=True, + pre=True, + fetch_kwargs={}, + verbose=False, + index_urls=["https://my.custom.index.com"], + ) + assert t.search_pyodide_lock_first is False + + +@pytest.mark.asyncio +async def test_index_url_priority( + mock_importlib, wheel_base, monkeypatch, mock_package_index_simple_json_api +): + # Test that if the index_urls are provided, package should be searched in + # the index_urls first before searching in Pyodide lock file. + from micropip.transaction import Transaction + + # add_wheel is called only when the package is found in the index_urls + add_wheel_called = None + + async def mock_add_wheel(self, wheel, extras, *, specifier=""): + nonlocal add_wheel_called + add_wheel_called = wheel + + monkeypatch.setattr(Transaction, "add_wheel", mock_add_wheel) + + mock_index_url = mock_package_index_simple_json_api(pkgs=["black"]) + + t = Transaction( + keep_going=True, + deps=False, + pre=False, + ctx={}, + ctx_extras=[], + fetch_kwargs={}, + index_urls=mock_index_url, + ) + + await t.add_requirement("black") + assert add_wheel_called is not None + assert add_wheel_called.name == "black" + # 23.7.0 is the latest version of black in the mock index + assert str(add_wheel_called.version) == "23.7.0" From bcb71851c69714404cb9cb3b509dfe995789c77e Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Mon, 11 Sep 2023 22:33:29 +0900 Subject: [PATCH 02/12] Set index_url before creating transaction --- micropip/_commands/install.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index dc856ec..439b12c 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -8,6 +8,7 @@ from ..constants import FAQ_URLS from ..logging import setup_logging from ..transaction import Transaction +from .. import package_index async def install( @@ -125,6 +126,9 @@ async def install( wheel_base = Path(getsitepackages()[0]) + if index_urls is None: + index_urls = package_index.INDEX_URLS + transaction = Transaction( ctx=ctx, ctx_extras=[], From d922fdb557da2cbf02a41f0a3eb3a0b6eff6d13a Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Mon, 11 Sep 2023 22:34:23 +0900 Subject: [PATCH 03/12] Pin build version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6deaefb..463f537 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ test = [ "pytest-httpserver", "pytest-pyodide", "pytest-cov", - "build", + "build==0.7.0", ] From 578d7180eaa3dcca66c0f5b2ce549aeeb57bc2f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:34:40 +0000 Subject: [PATCH 04/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- micropip/_commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index 439b12c..155eab0 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -4,11 +4,11 @@ from packaging.markers import default_environment +from .. import package_index from .._compat import loadPackage, to_js from ..constants import FAQ_URLS from ..logging import setup_logging from ..transaction import Transaction -from .. import package_index async def install( From c3b4e385c53230abd05dfd653ba79fd64958029c Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Mon, 11 Sep 2023 22:38:06 +0900 Subject: [PATCH 05/12] Update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0db4b22..5131b51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## [0.4.1] - 2023/09/XX + +### Changed + +- When custom index URLs are set by `micropip.set_index_urls` or by `micropip.install(index_urls=...)`, + micropip will now query packages from the custom index first, + and then from pyodide lockfile. + [#83](https://github.com/pyodide/micropip/pull/83) + ## [0.4.0] - 2023/07/25 ### Added From df9cc45bbdd8bb886055a26bc8bc21df5653aef3 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 19 Sep 2023 21:19:34 +0900 Subject: [PATCH 06/12] Copy index urls when setting --- micropip/_commands/index_urls.py | 2 +- micropip/_commands/install.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/micropip/_commands/index_urls.py b/micropip/_commands/index_urls.py index 60cc9b1..aae9cce 100644 --- a/micropip/_commands/index_urls.py +++ b/micropip/_commands/index_urls.py @@ -24,4 +24,4 @@ def set_index_urls(urls: list[str] | str) -> None: if isinstance(urls, str): urls = [urls] - package_index.INDEX_URLS = urls + package_index.INDEX_URLS = urls[:] diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index 155eab0..bc93c1f 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -127,7 +127,7 @@ async def install( wheel_base = Path(getsitepackages()[0]) if index_urls is None: - index_urls = package_index.INDEX_URLS + index_urls = package_index.INDEX_URLS[:] transaction = Transaction( ctx=ctx, From b924c5b0240333b35889fcbb496b0a6647e6232c Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 19 Sep 2023 21:29:45 +0900 Subject: [PATCH 07/12] Fix mistake in initialization --- micropip/transaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropip/transaction.py b/micropip/transaction.py index a09f7ec..0e03423 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -189,7 +189,7 @@ class Transaction: def __post_init__(self): # If index_urls is None, pyodide-lock.json have to be searched first. # TODO: when PyPI starts to support hosting WASM wheels, this might be deleted. - self.search_pyodide_lock_first = self.index_urls is None + self.search_pyodide_lock_first = self.index_urls == package_index.DEFAULT_INDEX_URLS async def gather_requirements( self, From 1ae1a466c5f9a5806270738657760f643a7da22e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 12:30:32 +0000 Subject: [PATCH 08/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- micropip/transaction.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/micropip/transaction.py b/micropip/transaction.py index 0e03423..fe9ac90 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -189,7 +189,9 @@ class Transaction: def __post_init__(self): # If index_urls is None, pyodide-lock.json have to be searched first. # TODO: when PyPI starts to support hosting WASM wheels, this might be deleted. - self.search_pyodide_lock_first = self.index_urls == package_index.DEFAULT_INDEX_URLS + self.search_pyodide_lock_first = ( + self.index_urls == package_index.DEFAULT_INDEX_URLS + ) async def gather_requirements( self, From 2599964fa1c1fe438f03d74565c10c3327b5eebb Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 19 Sep 2023 21:35:52 +0900 Subject: [PATCH 09/12] Fix test --- tests/test_transaction.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_transaction.py b/tests/test_transaction.py index b91b29a..7cacbc7 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -296,6 +296,7 @@ def test_last_version_and_best_tag_from_pypi( def test_search_pyodide_lock_first(): from micropip.transaction import Transaction + from micropip import package_index t = Transaction( ctx={}, @@ -305,7 +306,7 @@ def test_search_pyodide_lock_first(): pre=True, fetch_kwargs={}, verbose=False, - index_urls=None, + index_urls=package_index.DEFAULT_INDEX_URLS, ) assert t.search_pyodide_lock_first is True From d8cf2c1487cff7de378a6ca6779e6148184a7985 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 12:37:08 +0000 Subject: [PATCH 10/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_transaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 7cacbc7..b2ebb1d 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -295,8 +295,8 @@ def test_last_version_and_best_tag_from_pypi( def test_search_pyodide_lock_first(): - from micropip.transaction import Transaction from micropip import package_index + from micropip.transaction import Transaction t = Transaction( ctx={}, From 53cebc32a713be6baacd9f527b9c14378b2a85bb Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 19 Sep 2023 21:46:48 +0900 Subject: [PATCH 11/12] Move appending failed list --- micropip/transaction.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/micropip/transaction.py b/micropip/transaction.py index fe9ac90..a3663a6 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -292,19 +292,23 @@ def eval_marker(e: dict[str, str]) -> bool: logger.info(f"Requirement already satisfied: {req} ({ver})") return - if self.search_pyodide_lock_first: - if self._add_requirement_from_pyodide_lock(req): - return + try: + if self.search_pyodide_lock_first: + if self._add_requirement_from_pyodide_lock(req): + return - await self._add_requirement_from_package_index(req) - else: - try: await self._add_requirement_from_package_index(req) - except ValueError: - # If the requirement is not found in package index, - # we still have a chance to find it from pyodide lockfile. - if not self._add_requirement_from_pyodide_lock(req): - raise + else: + try: + await self._add_requirement_from_package_index(req) + except ValueError: + # If the requirement is not found in package index, + # we still have a chance to find it from pyodide lockfile. + if not self._add_requirement_from_pyodide_lock(req): + raise + except ValueError: + self.failed.append(req) + raise def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: """ @@ -334,7 +338,6 @@ async def _add_requirement_from_package_index(self, req: Requirement) -> bool: try: wheel = find_wheel(metadata, req) except ValueError: - self.failed.append(req) if not self.keep_going: raise else: From 730a8341e901788f1b324fc9ea09dde7f3a8cdab Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 19 Sep 2023 22:13:21 +0900 Subject: [PATCH 12/12] Fix keep going --- micropip/transaction.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/micropip/transaction.py b/micropip/transaction.py index a3663a6..c2fffbd 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -308,7 +308,8 @@ def eval_marker(e: dict[str, str]) -> bool: raise except ValueError: self.failed.append(req) - raise + if not self.keep_going: + raise def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: """ @@ -326,7 +327,7 @@ def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: return False - async def _add_requirement_from_package_index(self, req: Requirement) -> bool: + async def _add_requirement_from_package_index(self, req: Requirement): """ Find requirement from package index. If the requirement is found, add it to the package list and return True. Otherwise, return False. @@ -335,23 +336,15 @@ async def _add_requirement_from_package_index(self, req: Requirement) -> bool: req.name, self.fetch_kwargs, index_urls=self.index_urls ) - try: - wheel = find_wheel(metadata, req) - except ValueError: - if not self.keep_going: - raise - else: - return False + wheel = find_wheel(metadata, req) # Maybe while we were downloading pypi_json some other branch # installed the wheel? satisfied, ver = self.check_version_satisfied(req) if satisfied: logger.info(f"Requirement already satisfied: {req} ({ver})") - return True await self.add_wheel(wheel, req.extras, specifier=str(req.specifier)) - return True async def add_wheel( self,