Options to fix dependencies of manually loaded packages

micropip/_commands/freeze.py

@@ -6,6 +6,7 @@
 from packaging.utils import canonicalize_name
 
 from .._compat import REPODATA_INFO, REPODATA_PACKAGES
+from .._utils import fix_package_dependencies
 
 
 def freeze() -> str:
@@ -20,7 +21,6 @@ def freeze() -> str:
     You can use your custom lock file by passing an appropriate url to the
     ``lockFileURL`` of :js:func:`~globalThis.loadPyodide`.
     """
-
     packages = deepcopy(REPODATA_PACKAGES)
     for dist in importlib.metadata.distributions():
         name = dist.name
@@ -33,6 +33,9 @@ def freeze() -> str:
         assert sha256
         imports = (dist.read_text("top_level.txt") or "").split()
         requires = dist.read_text("PYODIDE_REQUIRES")
+        if not requires:
+            fix_package_dependencies(name)
+            requires = dist.read_text("PYODIDE_REQUIRES")
         if requires:
             depends = json.loads(requires)
         else:

micropip/_utils.py

@@ -1,14 +1,18 @@
 import functools
+import json
 from importlib.metadata import Distribution
 from pathlib import Path
 from sysconfig import get_platform
 
+from packaging.requirements import Requirement
 from packaging.tags import Tag
 from packaging.tags import sys_tags as sys_tags_orig
-from packaging.utils import BuildTag, InvalidWheelFilename
+from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name
 from packaging.utils import parse_wheel_filename as parse_wheel_filename_orig
 from packaging.version import InvalidVersion, Version
 
+from ._compat import REPODATA_PACKAGES
+
 
 def get_dist_info(dist: Distribution) -> Path:
     """
@@ -181,3 +185,75 @@ def platform_to_version(platform: str) -> str:
         )
 
     raise ValueError(f"Wheel interpreter version '{tag.interpreter}' is not supported.")
+
+
+def fix_package_dependencies(package_name, *, extras=None):
+    """Check and fix the list of dependencies for this package
+
+    If you have manually installed a package and dependencies from wheels,
+    the dependencies will not be correctly setup in the package list
+    or the pyodide lockfile generated by freezing. This method checks
+    if the dependencies are correctly set in the package list and will
+    add missing dependencies.
+
+    Parameters
+    ----------
+    package_name (string):
+        The name of the package to check.
+
+    extras (list):
+        List of extras for this package.
+
+    """
+    if package_name in REPODATA_PACKAGES:
+        # don't check things that are in original repository
+        return
+
+    dist = Distribution.from_name(package_name)
+
+    package_requires = dist.requires
+    if package_requires is None:
+        # no dependencies - we're good to go
+        return
+
+    url = dist.read_text("PYODIDE_URL")
+
+    # If it wasn't installed with micropip / pyodide, then we
+    # can't do anything with it.
+    if url is None:
+        return
+
+    # Get current list of pyodide requirements
+    requires = dist.read_text("PYODIDE_REQUIRES")
+
+    if requires:
+        depends = json.loads(requires)
+    else:
+        depends = []
+
+    if extras is None:
+        extras = [None]
+    else:
+        extras.append(None)
+    for r in package_requires:
+        req = Requirement(r)
+        req_extras = req.extras
+        req_marker = req.marker
+        req_name = canonicalize_name(req.name)
+        needs_requirement = False
+        if req_marker is not None:
+            for e in extras:
+                if req_marker.evaluate(None if e is None else {"extra": e}):
+                    needs_requirement = True
+                    break
+        else:
+            needs_requirement = True
+
+        if needs_requirement:
+            fix_package_dependencies(req_name, extras=list(req_extras))
+
+            if req_name not in depends:
+                depends.append(req_name)
+
+    # write updated depends to PYODIDE_DEPENDS
+    (dist._path / "PYODIDE_REQUIRES").write_text(json.dumps(sorted(x for x in depends)))

pyproject.toml

@@ -19,7 +19,7 @@ test = [
   "pytest-httpserver",
   "pytest-pyodide",
   "pytest-cov",
-  "build",
+  "build<1.0.0",
 ]
 
 

tests/conftest.py

@@ -117,6 +117,13 @@ def wheel_base(monkeypatch):
 def mock_importlib(monkeypatch, wheel_base):
     import importlib.metadata
 
+    def _mock_importlib_from_name(name: str) -> Distribution:
+        dists = _mock_importlib_distributions()
+        for dist in dists:
+            if dist.name == name:
+                return dist
+        raise PackageNotFoundError(name)
+
     def _mock_importlib_version(name: str) -> str:
         dists = _mock_importlib_distributions()
         for dist in dists:
@@ -131,6 +138,9 @@ def _mock_importlib_distributions():
     monkeypatch.setattr(
         importlib.metadata, "distributions", _mock_importlib_distributions
     )
+    monkeypatch.setattr(
+        importlib.metadata.Distribution, "from_name", _mock_importlib_from_name
+    )
 
 
 class Wildcard:

tests/test_freeze.py

@@ -3,7 +3,7 @@
 
 
 @pytest.mark.asyncio
-async def test_freeze(mock_fetch: mock_fetch_cls) -> None:
+async def test_freeze(mock_fetch: mock_fetch_cls, mock_importlib: None) -> None:
     import micropip
 
     dummy = "dummy"
@@ -30,3 +30,37 @@ async def test_freeze(mock_fetch: mock_fetch_cls) -> None:
     assert pkg_metadata["imports"] == toplevel[0]
     assert dep1_metadata["imports"] == toplevel[1]
     assert dep2_metadata["imports"] == toplevel[2]
+
+
+@pytest.mark.asyncio
+async def test_freeze_fix_depends(
+    mock_fetch: mock_fetch_cls, mock_importlib: None
+) -> None:
+    import micropip
+
+    dummy = "dummy"
+    dep1 = "dep1"
+    dep2 = "dep2"
+    toplevel = [["abc", "def", "geh"], ["c", "h", "i"], ["a12", "b13"]]
+
+    mock_fetch.add_pkg_version(dummy, requirements=[dep1, dep2], top_level=toplevel[0])
+    mock_fetch.add_pkg_version(dep1, top_level=toplevel[1])
+    mock_fetch.add_pkg_version(dep2, top_level=toplevel[2])
+
+    await micropip.install(dummy, deps=False)
+    await micropip.install(dep1, deps=False)
+    await micropip.install(dep2, deps=False)
+
+    import json
+
+    lockfile = json.loads(micropip.freeze())
+
+    pkg_metadata = lockfile["packages"][dummy]
+    dep1_metadata = lockfile["packages"][dep1]
+    dep2_metadata = lockfile["packages"][dep2]
+    assert pkg_metadata["depends"] == [dep1, dep2]
+    assert dep1_metadata["depends"] == []
+    assert dep2_metadata["depends"] == []
+    assert pkg_metadata["imports"] == toplevel[0]
+    assert dep1_metadata["imports"] == toplevel[1]
+    assert dep2_metadata["imports"] == toplevel[2]