From 5d4a8696f7daf302f060ffded34f9a84455fcffc Mon Sep 17 00:00:00 2001 From: Andrew Moon Date: Mon, 8 Jul 2024 22:17:18 +0900 Subject: [PATCH] Port `install` into `PackageManager` (#119) * feat: initialize repodata * feat: port install * fix: relative imports --- micropip/_commands/install.py | 172 +------------------------------ micropip/install.py | 185 ++++++++++++++++++++++++++++++++++ micropip/package_manager.py | 46 ++++++--- tests/test_package_manager.py | 47 ++++++++- 4 files changed, 265 insertions(+), 185 deletions(-) create mode 100644 micropip/install.py diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index 8e7e947..c395311 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -1,14 +1,6 @@ -import asyncio -import importlib -from pathlib import Path +from micropip import package_index -from packaging.markers import default_environment - -from .. import package_index -from .._compat import loadPackage, to_js -from ..constants import FAQ_URLS -from ..logging import setup_logging -from ..transaction import Transaction +from ..install import install as _install async def install( @@ -21,165 +13,9 @@ async def install( *, verbose: bool | int = False, ) -> None: - """Install the given package and all of its dependencies. - - If a package is not found in the Pyodide repository it will be loaded from - PyPI. Micropip can only load pure Python wheels or wasm32/emscripten wheels - built by Pyodide. - - When used in web browsers, downloads from PyPI will be cached. When run in - Node.js, packages are currently not cached, and will be re-downloaded each - time ``micropip.install`` is run. - - Parameters - ---------- - requirements : - - A requirement or list of requirements to install. Each requirement is a - string, which should be either a package name or a wheel URI: - - - If the requirement does not end in ``.whl``, it will be interpreted as - a package name. A package with this name must either be present - in the Pyodide lock file or on PyPI. - - - If the requirement ends in ``.whl``, it is a wheel URI. The part of - the requirement after the last ``/`` must be a valid wheel name in - compliance with the `PEP 427 naming convention - `_. - - - If a wheel URI starts with ``emfs:``, it will be interpreted as a path - in the Emscripten file system (Pyodide's file system). E.g., - ``emfs:../relative/path/wheel.whl`` or ``emfs:/absolute/path/wheel.whl``. - In this case, only .whl files are supported. - - - If a wheel URI requirement starts with ``http:`` or ``https:`` it will - be interpreted as a URL. - - - In node, you can access the native file system using a URI that starts - with ``file:``. In the browser this will not work. - - keep_going : - - This parameter decides the behavior of the micropip when it encounters a - Python package without a pure Python wheel while doing dependency - resolution: - - - If ``False``, an error will be raised on first package with a missing - wheel. - - - If ``True``, the micropip will keep going after the first error, and - report a list of errors at the end. - - deps : - - If ``True``, install dependencies specified in METADATA file for each - package. Otherwise do not install dependencies. - - credentials : - - This parameter specifies the value of ``credentials`` when calling the - `fetch() `__ - function which is used to download the package. - - When not specified, ``fetch()`` is called without ``credentials``. - - pre : - - If ``True``, include pre-release and development versions. By default, - micropip only finds stable versions. - - index_urls : - - A list of URLs or a single URL to use as the package index when looking - up packages. If None, *https://pypi.org/pypi/{package_name}/json* is used. - - - The index URL should support the - `JSON API `__ . - - - The index URL may contain the placeholder {package_name} which will be - replaced with the package name when looking up a package. If it does not - contain the placeholder, the package name will be appended to the URL. - - - If a list of URLs is provided, micropip will try each URL in order until - it finds a package. If no package is found, an error will be raised. - - verbose : - Print more information about the process. - By default, micropip is silent. Setting ``verbose=True`` will print - similar information as pip. - """ - logger = setup_logging(verbose) - - ctx = default_environment() - if isinstance(requirements, str): - requirements = [requirements] - - fetch_kwargs = dict() - - if credentials: - fetch_kwargs["credentials"] = credentials - - # Note: getsitepackages is not available in a virtual environment... - # See https://github.com/pypa/virtualenv/issues/228 (issue is closed but - # problem is not fixed) - from site import getsitepackages - - wheel_base = Path(getsitepackages()[0]) - if index_urls is None: index_urls = package_index.INDEX_URLS[:] - transaction = Transaction( - ctx=ctx, # type: ignore[arg-type] - ctx_extras=[], - keep_going=keep_going, - deps=deps, - pre=pre, - fetch_kwargs=fetch_kwargs, - verbose=verbose, - index_urls=index_urls, + return await _install( + requirements, keep_going, deps, credentials, pre, index_urls, verbose=verbose ) - await transaction.gather_requirements(requirements) - - if transaction.failed: - failed_requirements = ", ".join([f"'{req}'" for req in transaction.failed]) - raise ValueError( - f"Can't find a pure Python 3 wheel for: {failed_requirements}\n" - f"See: {FAQ_URLS['cant_find_wheel']}\n" - ) - - package_names = [pkg.name for pkg in transaction.pyodide_packages] + [ - pkg.name for pkg in transaction.wheels - ] - - if package_names: - logger.info("Installing collected packages: " + ", ".join(package_names)) - - wheel_promises = [] - # Install built-in packages - pyodide_packages = transaction.pyodide_packages - if len(pyodide_packages): - # Note: branch never happens in out-of-browser testing because in - # that case REPODATA_PACKAGES is empty. - wheel_promises.append( - asyncio.ensure_future( - loadPackage(to_js([name for [name, _, _] in pyodide_packages])) - ) - ) - - # Now install PyPI packages - for wheel in transaction.wheels: - # detect whether the wheel metadata is from PyPI or from custom location - # wheel metadata from PyPI has SHA256 checksum digest. - wheel_promises.append(wheel.install(wheel_base)) - - await asyncio.gather(*wheel_promises) - - packages = [f"{pkg.name}-{pkg.version}" for pkg in transaction.pyodide_packages] + [ - f"{pkg.name}-{pkg.version}" for pkg in transaction.wheels - ] - - if packages: - logger.info("Successfully installed " + ", ".join(packages)) - - importlib.invalidate_caches() diff --git a/micropip/install.py b/micropip/install.py new file mode 100644 index 0000000..4183c86 --- /dev/null +++ b/micropip/install.py @@ -0,0 +1,185 @@ +import asyncio +import importlib +from pathlib import Path + +from packaging.markers import default_environment + +from . import package_index +from ._compat import loadPackage, to_js +from .constants import FAQ_URLS +from .logging import setup_logging +from .transaction import Transaction + + +async def install( + requirements: str | list[str], + keep_going: bool = False, + deps: bool = True, + credentials: str | None = None, + pre: bool = False, + index_urls: list[str] | str | None = None, + *, + verbose: bool | int = False, +) -> None: + """Install the given package and all of its dependencies. + + If a package is not found in the Pyodide repository it will be loaded from + PyPI. Micropip can only load pure Python wheels or wasm32/emscripten wheels + built by Pyodide. + + When used in web browsers, downloads from PyPI will be cached. When run in + Node.js, packages are currently not cached, and will be re-downloaded each + time ``micropip.install`` is run. + + Parameters + ---------- + requirements : + + A requirement or list of requirements to install. Each requirement is a + string, which should be either a package name or a wheel URI: + + - If the requirement does not end in ``.whl``, it will be interpreted as + a package name. A package with this name must either be present + in the Pyodide lock file or on PyPI. + + - If the requirement ends in ``.whl``, it is a wheel URI. The part of + the requirement after the last ``/`` must be a valid wheel name in + compliance with the `PEP 427 naming convention + `_. + + - If a wheel URI starts with ``emfs:``, it will be interpreted as a path + in the Emscripten file system (Pyodide's file system). E.g., + ``emfs:../relative/path/wheel.whl`` or ``emfs:/absolute/path/wheel.whl``. + In this case, only .whl files are supported. + + - If a wheel URI requirement starts with ``http:`` or ``https:`` it will + be interpreted as a URL. + + - In node, you can access the native file system using a URI that starts + with ``file:``. In the browser this will not work. + + keep_going : + + This parameter decides the behavior of the micropip when it encounters a + Python package without a pure Python wheel while doing dependency + resolution: + + - If ``False``, an error will be raised on first package with a missing + wheel. + + - If ``True``, the micropip will keep going after the first error, and + report a list of errors at the end. + + deps : + + If ``True``, install dependencies specified in METADATA file for each + package. Otherwise do not install dependencies. + + credentials : + + This parameter specifies the value of ``credentials`` when calling the + `fetch() `__ + function which is used to download the package. + + When not specified, ``fetch()`` is called without ``credentials``. + + pre : + + If ``True``, include pre-release and development versions. By default, + micropip only finds stable versions. + + index_urls : + + A list of URLs or a single URL to use as the package index when looking + up packages. If None, *https://pypi.org/pypi/{package_name}/json* is used. + + - The index URL should support the + `JSON API `__ . + + - The index URL may contain the placeholder {package_name} which will be + replaced with the package name when looking up a package. If it does not + contain the placeholder, the package name will be appended to the URL. + + - If a list of URLs is provided, micropip will try each URL in order until + it finds a package. If no package is found, an error will be raised. + + verbose : + Print more information about the process. + By default, micropip is silent. Setting ``verbose=True`` will print + similar information as pip. + """ + logger = setup_logging(verbose) + + ctx = default_environment() + if isinstance(requirements, str): + requirements = [requirements] + + fetch_kwargs = dict() + + if credentials: + fetch_kwargs["credentials"] = credentials + + # Note: getsitepackages is not available in a virtual environment... + # See https://github.com/pypa/virtualenv/issues/228 (issue is closed but + # problem is not fixed) + from site import getsitepackages + + wheel_base = Path(getsitepackages()[0]) + + if index_urls is None: + index_urls = package_index.INDEX_URLS[:] + + transaction = Transaction( + ctx=ctx, # type: ignore[arg-type] + ctx_extras=[], + keep_going=keep_going, + deps=deps, + pre=pre, + fetch_kwargs=fetch_kwargs, + verbose=verbose, + index_urls=index_urls, + ) + await transaction.gather_requirements(requirements) + + if transaction.failed: + failed_requirements = ", ".join([f"'{req}'" for req in transaction.failed]) + raise ValueError( + f"Can't find a pure Python 3 wheel for: {failed_requirements}\n" + f"See: {FAQ_URLS['cant_find_wheel']}\n" + ) + + package_names = [pkg.name for pkg in transaction.pyodide_packages] + [ + pkg.name for pkg in transaction.wheels + ] + + if package_names: + logger.info("Installing collected packages: " + ", ".join(package_names)) + + wheel_promises = [] + # Install built-in packages + pyodide_packages = transaction.pyodide_packages + if len(pyodide_packages): + # Note: branch never happens in out-of-browser testing because in + # that case REPODATA_PACKAGES is empty. + wheel_promises.append( + asyncio.ensure_future( + loadPackage(to_js([name for [name, _, _] in pyodide_packages])) + ) + ) + + # Now install PyPI packages + for wheel in transaction.wheels: + # detect whether the wheel metadata is from PyPI or from custom location + # wheel metadata from PyPI has SHA256 checksum digest. + wheel_promises.append(wheel.install(wheel_base)) + + await asyncio.gather(*wheel_promises) + + packages = [f"{pkg.name}-{pkg.version}" for pkg in transaction.pyodide_packages] + [ + f"{pkg.name}-{pkg.version}" for pkg in transaction.wheels + ] + + if packages: + logger.info("Successfully installed " + ", ".join(packages)) + + importlib.invalidate_caches() diff --git a/micropip/package_manager.py b/micropip/package_manager.py index 42f4108..c3b874c 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -3,11 +3,13 @@ List, ) -from micropip import package_index -from micropip._commands import mock_package -from micropip.freeze import freeze_lockfile -from micropip.list import list_installed_packages -from micropip.package import PackageDict +from . import package_index +from ._commands import mock_package +from ._compat import REPODATA_INFO, REPODATA_PACKAGES +from .freeze import freeze_lockfile +from .install import install +from .list import list_installed_packages +from .package import PackageDict class PackageManager: @@ -16,21 +18,39 @@ class PackageManager: Each Manager instance holds its own local state that is independent of other instances. - - TODO: Implement all of the following global commands to utilize local state. """ def __init__(self) -> None: - self.index_urls = package_index.DEFAULT_INDEX_URLS + self.index_urls = package_index.DEFAULT_INDEX_URLS[:] - # TODO: initialize the compatibility layer - self.repodata_packages: dict[str, dict[str, Any]] = {} - self.repodata_info: dict[str, str] = {} + self.repodata_packages: dict[str, dict[str, Any]] = REPODATA_PACKAGES + self.repodata_info: dict[str, str] = REPODATA_INFO pass - def install(self): - raise NotImplementedError() + async def install( + self, + requirements: str | list[str], + keep_going: bool = False, + deps: bool = True, + credentials: str | None = None, + pre: bool = False, + index_urls: list[str] | str | None = None, + *, + verbose: bool | int = False, + ): + if index_urls is None: + index_urls = self.index_urls + + return await install( + requirements, + keep_going, + deps, + credentials, + pre, + index_urls, + verbose=verbose, + ) def list(self) -> PackageDict: return list_installed_packages(self.repodata_packages) diff --git a/tests/test_package_manager.py b/tests/test_package_manager.py index a1669af..8d70714 100644 --- a/tests/test_package_manager.py +++ b/tests/test_package_manager.py @@ -1,6 +1,7 @@ import json import pytest +from conftest import mock_fetch_cls import micropip.package_index as package_index from micropip.package_manager import PackageManager @@ -61,10 +62,48 @@ def test_freeze(): } -@pytest.mark.skip(reason="Not implemented") -def test_list(): +@pytest.mark.asyncio +async def test_list(mock_fetch: mock_fetch_cls): manager = get_test_package_manager() - _package_dict = manager.list() + dummy = "dummy" + mock_fetch.add_pkg_version(dummy) + dummy_url = f"https://dummy.com/{dummy}-1.0.0-py3-none-any.whl" - # TODO: implement test after implementing manager.install() + await manager.install(dummy_url) + + pkg_list = manager.list() + + assert dummy in pkg_list + assert pkg_list[dummy].source.lower() == dummy_url + + +@pytest.mark.asyncio +async def test_custom_index_url(mock_package_index_json_api, monkeypatch): + manager = get_test_package_manager() + + mock_server_fake_package = mock_package_index_json_api( + pkgs=["fake-pkg-micropip-test"] + ) + + _wheel_url = "" + + async def _mock_fetch_bytes(url, *args): + nonlocal _wheel_url + _wheel_url = url + return b"fake wheel" + + from micropip import wheelinfo + + monkeypatch.setattr(wheelinfo, "fetch_bytes", _mock_fetch_bytes) + + manager.set_index_urls([mock_server_fake_package]) + + try: + await manager.install("fake-pkg-micropip-test") + except Exception: + # We just check that the custom index url was used + # install will fail because the package is not real, but it doesn't matter. + pass + + assert "fake_pkg_micropip_test-1.0.0-py2.py3-none-any.whl" in _wheel_url