From dca16912156e0273515ebb68500dabbe8d2e41e1 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Tue, 26 Sep 2023 23:32:48 +0200 Subject: [PATCH 1/2] Add support for packages installed with micropip --- CHANGELOG.md | 3 +++ examples/micropip_deps/app.py | 6 ++++++ examples/micropip_deps/requirements.txt | 1 + pyodide_pack/_utils.py | 7 +++++++ pyodide_pack/cli.py | 5 ++++- pyodide_pack/js/discovery.js | 12 +++++++++++- 6 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 examples/micropip_deps/app.py create mode 100644 examples/micropip_deps/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 61c2d90..8bf8e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add support for stdlib bundling in `pyodide pack` [#27](https://github.com/pyodide/pyodide-pack/pull/27) + - Add support for packages installed via micropip in `pyodide pack` + [#31](https://github.com/pyodide/pyodide-pack/pull/31) + ### Changed diff --git a/examples/micropip_deps/app.py b/examples/micropip_deps/app.py new file mode 100644 index 0000000..fdb9a7c --- /dev/null +++ b/examples/micropip_deps/app.py @@ -0,0 +1,6 @@ +# Examples of a package that is installed by micropip from PyPI + +import snowballstemmer + +stemmer = snowballstemmer.stemmer("english") +print(stemmer.stemWords("An example of stemming".split())) diff --git a/examples/micropip_deps/requirements.txt b/examples/micropip_deps/requirements.txt new file mode 100644 index 0000000..a78d13e --- /dev/null +++ b/examples/micropip_deps/requirements.txt @@ -0,0 +1 @@ +snowballstemmer diff --git a/pyodide_pack/_utils.py b/pyodide_pack/_utils.py index 94ea360..fe289d3 100644 --- a/pyodide_pack/_utils.py +++ b/pyodide_pack/_utils.py @@ -3,6 +3,7 @@ import shutil import sys import tempfile +import urllib.request from contextlib import contextmanager from pathlib import Path @@ -117,6 +118,12 @@ def _get_packages_from_lockfile( for key, val in loaded_packages.items(): if val == "default channel": file_name = pyodide_lock.packages[key].file_name + elif val == "pypi": + url = pyodide_lock.packages[key].file_name + file_name = os.path.basename(url) + # Will raise an exception if the URL is not valid + with urllib.request.urlopen(url) as response: + (package_dir / file_name).write_bytes(response.read()) else: # Otherwise loaded from custom URL # TODO: this branch needs testing diff --git a/pyodide_pack/cli.py b/pyodide_pack/cli.py index 8a18f9e..c514228 100644 --- a/pyodide_pack/cli.py +++ b/pyodide_pack/cli.py @@ -89,7 +89,10 @@ def main( package_dir = ROOT_DIR / "node_modules" / "pyodide" - pyodide_lock = PyodideLockSpec.from_json(package_dir / "pyodide-lock.json") + if "pyodide_lock" in db: + pyodide_lock = PyodideLockSpec(**json.loads(db["pyodide_lock"])) + else: + pyodide_lock = PyodideLockSpec.from_json(package_dir / "pyodide-lock.json") packages = _get_packages_from_lockfile( pyodide_lock, db["loaded_packages"], package_dir diff --git a/pyodide_pack/js/discovery.js b/pyodide_pack/js/discovery.js index 83ed16a..ae700bb 100644 --- a/pyodide_pack/js/discovery.js +++ b/pyodide_pack/js/discovery.js @@ -19,7 +19,14 @@ async function main() { return open_orig(path, flags, mode, fd_start, fd_end); }; - await pyodide.loadPackage({{packages}}); + try { + await pyodide.loadPackage({{packages}}); + } catch (e) { + console.log("Failed to load packages, trying to load micropip"); + await pyodide.loadPackage("micropip"); + let micropip = pyodide.pyimport("micropip"); + await micropip.install({{packages}}); + } // Monkeypatching findObject calls used in dlopen let findObjectCalls = []; @@ -42,6 +49,9 @@ async function main() { obj.loaded_packages = pyodide.loadedPackages; obj.find_object_calls = findObjectCalls; obj.sys_modules = sysModules; + if ("micropip" in pyodide.loadedPackages) { + obj.pyodide_lock = pyodide.pyimport("micropip").freeze(); + } // For some reason there is a double / in the path obj.stdlib_prefix = pyodide.pyimport('sysconfig').get_path('stdlib').replace('//', '/'); let jsonString = JSON.stringify(obj); From cc3887b1794e409f22604b7b110b90edc2a8e789 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Wed, 27 Sep 2023 10:41:34 +0200 Subject: [PATCH 2/2] Minor improvements --- examples/test_examples.py | 9 +++++++++ pyodide_pack/_utils.py | 1 + pyodide_pack/js/discovery.js | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/test_examples.py b/examples/test_examples.py index 4cdff40..43796fe 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -46,6 +46,15 @@ def test_all(example_dir, tmp_path): stdout_str = stdout.getvalue() assert "Bundle validation successful" in stdout_str + using_micropip = ( + "Failed to load packages with loadPackage, re-trying with micropip." + in stdout_str + ) + + if example_dir.name == ["micropip_deps"]: + assert using_micropip + else: + assert not using_micropip assert (tmp_path / "python_stdlib_stripped.zip").exists() # Better than 20% size reduction on the stdlib assert (tmp_path / "python_stdlib_stripped.zip").stat().st_size < ( diff --git a/pyodide_pack/_utils.py b/pyodide_pack/_utils.py index fe289d3..05d6977 100644 --- a/pyodide_pack/_utils.py +++ b/pyodide_pack/_utils.py @@ -121,6 +121,7 @@ def _get_packages_from_lockfile( elif val == "pypi": url = pyodide_lock.packages[key].file_name file_name = os.path.basename(url) + # Cache wheels in node_modules/pyodide # Will raise an exception if the URL is not valid with urllib.request.urlopen(url) as response: (package_dir / file_name).write_bytes(response.read()) diff --git a/pyodide_pack/js/discovery.js b/pyodide_pack/js/discovery.js index ae700bb..ffac6a8 100644 --- a/pyodide_pack/js/discovery.js +++ b/pyodide_pack/js/discovery.js @@ -22,7 +22,7 @@ async function main() { try { await pyodide.loadPackage({{packages}}); } catch (e) { - console.log("Failed to load packages, trying to load micropip"); + console.log("Failed to load packages with loadPackage, re-trying with micropip."); await pyodide.loadPackage("micropip"); let micropip = pyodide.pyimport("micropip"); await micropip.install({{packages}});