diff --git a/CHANGELOG.md b/CHANGELOG.md index 0db4b22..3748109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Made micropip.freeze correctly list dependencies of manually installed packages. + [#79](https://github.com/pyodide/micropip/pull/79) + ## [0.4.0] - 2023/07/25 ### Added diff --git a/micropip/_commands/freeze.py b/micropip/_commands/freeze.py index 6b27cc4..4a49dc5 100644 --- a/micropip/_commands/freeze.py +++ b/micropip/_commands/freeze.py @@ -6,6 +6,7 @@ from packaging.utils import canonicalize_name from .._compat import REPODATA_INFO, REPODATA_PACKAGES +from .._utils import fix_package_dependencies def freeze() -> str: @@ -20,7 +21,6 @@ def freeze() -> str: You can use your custom lock file by passing an appropriate url to the ``lockFileURL`` of :js:func:`~globalThis.loadPyodide`. """ - packages = deepcopy(REPODATA_PACKAGES) for dist in importlib.metadata.distributions(): name = dist.name @@ -33,6 +33,9 @@ def freeze() -> str: assert sha256 imports = (dist.read_text("top_level.txt") or "").split() requires = dist.read_text("PYODIDE_REQUIRES") + if not requires: + fix_package_dependencies(name) + requires = dist.read_text("PYODIDE_REQUIRES") if requires: depends = json.loads(requires) else: diff --git a/micropip/_utils.py b/micropip/_utils.py index 04cfb89..9d6bc35 100644 --- a/micropip/_utils.py +++ b/micropip/_utils.py @@ -1,14 +1,18 @@ import functools +import json from importlib.metadata import Distribution from pathlib import Path from sysconfig import get_platform +from packaging.requirements import Requirement from packaging.tags import Tag from packaging.tags import sys_tags as sys_tags_orig -from packaging.utils import BuildTag, InvalidWheelFilename +from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name from packaging.utils import parse_wheel_filename as parse_wheel_filename_orig from packaging.version import InvalidVersion, Version +from ._compat import REPODATA_PACKAGES + def get_dist_info(dist: Distribution) -> Path: """ @@ -181,3 +185,79 @@ def platform_to_version(platform: str) -> str: ) raise ValueError(f"Wheel interpreter version '{tag.interpreter}' is not supported.") + + +def fix_package_dependencies( + package_name: str, *, extras: list[str | None] | None = None +) -> None: + """Check and fix the list of dependencies for this package + + If you have manually installed a package and dependencies from wheels, + the dependencies will not be correctly setup in the package list + or the pyodide lockfile generated by freezing. This method checks + if the dependencies are correctly set in the package list and will + add missing dependencies. + + Parameters + ---------- + package_name (string): + The name of the package to check. + + extras (list): + List of extras for this package. + + """ + if package_name in REPODATA_PACKAGES: + # don't check things that are in original repository + return + + dist = Distribution.from_name(package_name) + + package_requires = dist.requires + if package_requires is None: + # no dependencies - we're good to go + return + + url = dist.read_text("PYODIDE_URL") + + # If it wasn't installed with micropip / pyodide, then we + # can't do anything with it. + if url is None: + return + + # Get current list of pyodide requirements + requires = dist.read_text("PYODIDE_REQUIRES") + + if requires: + depends = json.loads(requires) + else: + depends = [] + + if extras is None: + extras = [None] + else: + extras = extras + [None] + for r in package_requires: + req = Requirement(r) + req_extras = req.extras + req_marker = req.marker + req_name = canonicalize_name(req.name) + needs_requirement = False + if req_marker is not None: + for e in extras: + if req_marker.evaluate(None if e is None else {"extra": e}): + needs_requirement = True + break + else: + needs_requirement = True + + if needs_requirement: + fix_package_dependencies(req_name, extras=list(req_extras)) + + if req_name not in depends: + depends.append(req_name) + + # write updated depends to PYODIDE_DEPENDS + (get_dist_info(dist) / "PYODIDE_REQUIRES").write_text( + json.dumps(sorted(x for x in depends)) + ) diff --git a/pyproject.toml b/pyproject.toml index 6deaefb..fd5f54b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ test = [ "pytest-httpserver", "pytest-pyodide", "pytest-cov", - "build", + "build<1.0.0", ] diff --git a/tests/conftest.py b/tests/conftest.py index 4e587d4..2f2647a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -117,6 +117,13 @@ def wheel_base(monkeypatch): def mock_importlib(monkeypatch, wheel_base): import importlib.metadata + def _mock_importlib_from_name(name: str) -> Distribution: + dists = _mock_importlib_distributions() + for dist in dists: + if dist.name == name: + return dist + raise PackageNotFoundError(name) + def _mock_importlib_version(name: str) -> str: dists = _mock_importlib_distributions() for dist in dists: @@ -131,6 +138,9 @@ def _mock_importlib_distributions(): monkeypatch.setattr( importlib.metadata, "distributions", _mock_importlib_distributions ) + monkeypatch.setattr( + importlib.metadata.Distribution, "from_name", _mock_importlib_from_name + ) class Wildcard: diff --git a/tests/test_freeze.py b/tests/test_freeze.py index 5aa25d8..50b8eb8 100644 --- a/tests/test_freeze.py +++ b/tests/test_freeze.py @@ -3,7 +3,7 @@ @pytest.mark.asyncio -async def test_freeze(mock_fetch: mock_fetch_cls) -> None: +async def test_freeze(mock_fetch: mock_fetch_cls, mock_importlib: None) -> None: import micropip dummy = "dummy" @@ -30,3 +30,37 @@ async def test_freeze(mock_fetch: mock_fetch_cls) -> None: assert pkg_metadata["imports"] == toplevel[0] assert dep1_metadata["imports"] == toplevel[1] assert dep2_metadata["imports"] == toplevel[2] + + +@pytest.mark.asyncio +async def test_freeze_fix_depends( + mock_fetch: mock_fetch_cls, mock_importlib: None +) -> None: + import micropip + + dummy = "dummy" + dep1 = "dep1" + dep2 = "dep2" + toplevel = [["abc", "def", "geh"], ["c", "h", "i"], ["a12", "b13"]] + + mock_fetch.add_pkg_version(dummy, requirements=[dep1, dep2], top_level=toplevel[0]) + mock_fetch.add_pkg_version(dep1, top_level=toplevel[1]) + mock_fetch.add_pkg_version(dep2, top_level=toplevel[2]) + + await micropip.install(dummy, deps=False) + await micropip.install(dep1, deps=False) + await micropip.install(dep2, deps=False) + + import json + + lockfile = json.loads(micropip.freeze()) + + pkg_metadata = lockfile["packages"][dummy] + dep1_metadata = lockfile["packages"][dep1] + dep2_metadata = lockfile["packages"][dep2] + assert pkg_metadata["depends"] == [dep1, dep2] + assert dep1_metadata["depends"] == [] + assert dep2_metadata["depends"] == [] + assert pkg_metadata["imports"] == toplevel[0] + assert dep1_metadata["imports"] == toplevel[1] + assert dep2_metadata["imports"] == toplevel[2]