pyodide pack: Add support for packages installed with micropip

CHANGELOG.md

@@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  - Add support for stdlib bundling in `pyodide pack`
    [#27](https://github.com/pyodide/pyodide-pack/pull/27)
 
+ - Add support for packages installed via micropip in `pyodide pack`
+   [#31](https://github.com/pyodide/pyodide-pack/pull/31)
+
 
 ### Changed
 

examples/micropip_deps/app.py

@@ -0,0 +1,6 @@
+# Examples of a package that is installed by micropip from PyPI
+
+import snowballstemmer
+
+stemmer = snowballstemmer.stemmer("english")
+print(stemmer.stemWords("An example of stemming".split()))

examples/micropip_deps/requirements.txt

@@ -0,0 +1 @@
+snowballstemmer

examples/test_examples.py

@@ -46,6 +46,15 @@ def test_all(example_dir, tmp_path):
 
     stdout_str = stdout.getvalue()
     assert "Bundle validation successful" in stdout_str
+    using_micropip = (
+        "Failed to load packages with loadPackage, re-trying with micropip."
+        in stdout_str
+    )
+
+    if example_dir.name == ["micropip_deps"]:
+        assert using_micropip
+    else:
+        assert not using_micropip
     assert (tmp_path / "python_stdlib_stripped.zip").exists()
     # Better than 20% size reduction on the stdlib
     assert (tmp_path / "python_stdlib_stripped.zip").stat().st_size < (

pyodide_pack/_utils.py

@@ -3,6 +3,7 @@
 import shutil
 import sys
 import tempfile
+import urllib.request
 from contextlib import contextmanager
 from pathlib import Path
 
@@ -117,6 +118,13 @@ def _get_packages_from_lockfile(
     for key, val in loaded_packages.items():
         if val == "default channel":
             file_name = pyodide_lock.packages[key].file_name
+        elif val == "pypi":
+            url = pyodide_lock.packages[key].file_name
+            file_name = os.path.basename(url)
+            # Cache wheels in node_modules/pyodide
+            # Will raise an exception if the URL is not valid
+            with urllib.request.urlopen(url) as response:
+                (package_dir / file_name).write_bytes(response.read())
         else:
             # Otherwise loaded from custom URL
             # TODO: this branch needs testing

pyodide_pack/cli.py

@@ -89,7 +89,10 @@ def main(
 
     package_dir = ROOT_DIR / "node_modules" / "pyodide"
 
-    pyodide_lock = PyodideLockSpec.from_json(package_dir / "pyodide-lock.json")
+    if "pyodide_lock" in db:
+        pyodide_lock = PyodideLockSpec(**json.loads(db["pyodide_lock"]))
+    else:
+        pyodide_lock = PyodideLockSpec.from_json(package_dir / "pyodide-lock.json")
 
     packages = _get_packages_from_lockfile(
         pyodide_lock, db["loaded_packages"], package_dir

pyodide_pack/js/discovery.js

@@ -19,7 +19,14 @@ async function main() {
     return open_orig(path, flags, mode, fd_start, fd_end);
   };
 
-  await pyodide.loadPackage({{packages}});
+  try {
+  	await pyodide.loadPackage({{packages}});
+  } catch (e) {
+	console.log("Failed to load packages with loadPackage, re-trying with micropip.");
+	await pyodide.loadPackage("micropip");
+	let micropip = pyodide.pyimport("micropip");
+	await micropip.install({{packages}});
+  }
 
   // Monkeypatching findObject calls used in dlopen
   let findObjectCalls = [];
@@ -42,6 +49,9 @@ async function main() {
   obj.loaded_packages = pyodide.loadedPackages;
   obj.find_object_calls = findObjectCalls;
   obj.sys_modules = sysModules;
+  if ("micropip" in pyodide.loadedPackages) {
+    obj.pyodide_lock = pyodide.pyimport("micropip").freeze();
+  }
   // For some reason there is a double / in the path
   obj.stdlib_prefix = pyodide.pyimport('sysconfig').get_path('stdlib').replace('//', '/');
   let jsonString = JSON.stringify(obj);