Options to fix dependencies of manually loaded packages (#79)

Right now, if you import packages from wheel files without automatically getting dependencies, micropip.freeze won't correctly show dependencies for the packages.

This patch fixes up dependencies for manually loaded wheels, adding a method micropip.check_package_dependencies to check and fix dependencies for a package.

It also uses the same function to make micropip.freeze work even if you just manually load a wheel and all dependencies.

Still not sure whether this would be better in pyodide-lock, but I think given micropip.freeze exists, this patch that makes it work in this case seems reasonably sensible.

CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+- Made micropip.freeze correctly list dependencies of manually installed packages.
+  [#79](https://github.com/pyodide/micropip/pull/79)
+
 ## [0.4.0] - 2023/07/25
 
 ### Added

micropip/_commands/freeze.py

@@ -6,6 +6,7 @@
 from packaging.utils import canonicalize_name
 
 from .._compat import REPODATA_INFO, REPODATA_PACKAGES
+from .._utils import fix_package_dependencies
 
 
 def freeze() -> str:
@@ -20,7 +21,6 @@ def freeze() -> str:
     You can use your custom lock file by passing an appropriate url to the
     ``lockFileURL`` of :js:func:`~globalThis.loadPyodide`.
     """
-
     packages = deepcopy(REPODATA_PACKAGES)
     for dist in importlib.metadata.distributions():
         name = dist.name
@@ -33,6 +33,9 @@ def freeze() -> str:
         assert sha256
         imports = (dist.read_text("top_level.txt") or "").split()
         requires = dist.read_text("PYODIDE_REQUIRES")
+        if not requires:
+            fix_package_dependencies(name)
+            requires = dist.read_text("PYODIDE_REQUIRES")
         if requires:
             depends = json.loads(requires)
         else:

micropip/_utils.py

@@ -1,14 +1,18 @@
 import functools
+import json
 from importlib.metadata import Distribution
 from pathlib import Path
 from sysconfig import get_platform
 
+from packaging.requirements import Requirement
 from packaging.tags import Tag
 from packaging.tags import sys_tags as sys_tags_orig
-from packaging.utils import BuildTag, InvalidWheelFilename
+from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name
 from packaging.utils import parse_wheel_filename as parse_wheel_filename_orig
 from packaging.version import InvalidVersion, Version
 
+from ._compat import REPODATA_PACKAGES
+
 
 def get_dist_info(dist: Distribution) -> Path:
     """
@@ -181,3 +185,79 @@ def platform_to_version(platform: str) -> str:
         )
 
     raise ValueError(f"Wheel interpreter version '{tag.interpreter}' is not supported.")
+
+
+def fix_package_dependencies(
+    package_name: str, *, extras: list[str | None] | None = None
+) -> None:
+    """Check and fix the list of dependencies for this package
+
+    If you have manually installed a package and dependencies from wheels,
+    the dependencies will not be correctly setup in the package list
+    or the pyodide lockfile generated by freezing. This method checks
+    if the dependencies are correctly set in the package list and will
+    add missing dependencies.
+
+    Parameters
+    ----------
+    package_name (string):
+        The name of the package to check.
+
+    extras (list):
+        List of extras for this package.
+
+    """
+    if package_name in REPODATA_PACKAGES:
+        # don't check things that are in original repository
+        return
+
+    dist = Distribution.from_name(package_name)
+
+    package_requires = dist.requires
+    if package_requires is None:
+        # no dependencies - we're good to go
+        return
+
+    url = dist.read_text("PYODIDE_URL")
+
+    # If it wasn't installed with micropip / pyodide, then we
+    # can't do anything with it.
+    if url is None:
+        return
+
+    # Get current list of pyodide requirements
+    requires = dist.read_text("PYODIDE_REQUIRES")
+
+    if requires:
+        depends = json.loads(requires)
+    else:
+        depends = []
+
+    if extras is None:
+        extras = [None]
+    else:
+        extras = extras + [None]
+    for r in package_requires:
+        req = Requirement(r)
+        req_extras = req.extras
+        req_marker = req.marker
+        req_name = canonicalize_name(req.name)
+        needs_requirement = False
+        if req_marker is not None:
+            for e in extras:
+                if req_marker.evaluate(None if e is None else {"extra": e}):
+                    needs_requirement = True
+                    break
+        else:
+            needs_requirement = True
+
+        if needs_requirement:
+            fix_package_dependencies(req_name, extras=list(req_extras))
+
+            if req_name not in depends:
+                depends.append(req_name)
+
+    # write updated depends to PYODIDE_DEPENDS
+    (get_dist_info(dist) / "PYODIDE_REQUIRES").write_text(
+        json.dumps(sorted(x for x in depends))
+    )

pyproject.toml

@@ -19,7 +19,7 @@ test = [
   "pytest-httpserver",
   "pytest-pyodide",
   "pytest-cov",
-  "build",
+  "build<1.0.0",
 ]
 
 

tests/conftest.py

@@ -117,6 +117,13 @@ def wheel_base(monkeypatch):
 def mock_importlib(monkeypatch, wheel_base):
     import importlib.metadata
 
+    def _mock_importlib_from_name(name: str) -> Distribution:
+        dists = _mock_importlib_distributions()
+        for dist in dists:
+            if dist.name == name:
+                return dist
+        raise PackageNotFoundError(name)
+
     def _mock_importlib_version(name: str) -> str:
         dists = _mock_importlib_distributions()
         for dist in dists:
@@ -131,6 +138,9 @@ def _mock_importlib_distributions():
     monkeypatch.setattr(
         importlib.metadata, "distributions", _mock_importlib_distributions
     )
+    monkeypatch.setattr(
+        importlib.metadata.Distribution, "from_name", _mock_importlib_from_name
+    )
 
 
 class Wildcard:

tests/test_freeze.py

@@ -3,7 +3,7 @@
 
 
 @pytest.mark.asyncio
-async def test_freeze(mock_fetch: mock_fetch_cls) -> None:
+async def test_freeze(mock_fetch: mock_fetch_cls, mock_importlib: None) -> None:
     import micropip
 
     dummy = "dummy"
@@ -30,3 +30,37 @@ async def test_freeze(mock_fetch: mock_fetch_cls) -> None:
     assert pkg_metadata["imports"] == toplevel[0]
     assert dep1_metadata["imports"] == toplevel[1]
     assert dep2_metadata["imports"] == toplevel[2]
+
+
+@pytest.mark.asyncio
+async def test_freeze_fix_depends(
+    mock_fetch: mock_fetch_cls, mock_importlib: None
+) -> None:
+    import micropip
+
+    dummy = "dummy"
+    dep1 = "dep1"
+    dep2 = "dep2"
+    toplevel = [["abc", "def", "geh"], ["c", "h", "i"], ["a12", "b13"]]
+
+    mock_fetch.add_pkg_version(dummy, requirements=[dep1, dep2], top_level=toplevel[0])
+    mock_fetch.add_pkg_version(dep1, top_level=toplevel[1])
+    mock_fetch.add_pkg_version(dep2, top_level=toplevel[2])
+
+    await micropip.install(dummy, deps=False)
+    await micropip.install(dep1, deps=False)
+    await micropip.install(dep2, deps=False)
+
+    import json
+
+    lockfile = json.loads(micropip.freeze())
+
+    pkg_metadata = lockfile["packages"][dummy]
+    dep1_metadata = lockfile["packages"][dep1]
+    dep2_metadata = lockfile["packages"][dep2]
+    assert pkg_metadata["depends"] == [dep1, dep2]
+    assert dep1_metadata["depends"] == []
+    assert dep2_metadata["depends"] == []
+    assert pkg_metadata["imports"] == toplevel[0]
+    assert dep1_metadata["imports"] == toplevel[1]
+    assert dep2_metadata["imports"] == toplevel[2]