From 8ed44edb625cab708483f7dfd9b3916f761d6fc4 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 2 Oct 2023 13:35:35 +0100 Subject: [PATCH] moved anything that modifies spec into utils --- pyodide_lock/spec.py | 225 +---------------------------------------- pyodide_lock/utils.py | 226 +++++++++++++++++++++++++++++++++++++++++- tests/test_spec.py | 5 +- tests/test_wheel.py | 33 +++--- 4 files changed, 248 insertions(+), 241 deletions(-) diff --git a/pyodide_lock/spec.py b/pyodide_lock/spec.py index 317694f..b287174 100644 --- a/pyodide_lock/spec.py +++ b/pyodide_lock/spec.py @@ -1,20 +1,11 @@ import json -import re from pathlib import Path from typing import TYPE_CHECKING, Literal from pydantic import BaseModel, Extra, Field if TYPE_CHECKING: - from packaging.requirements import Requirement - -from .utils import ( - _generate_package_hash, - _get_marker_environment, - _wheel_depends, - _wheel_metadata, - parse_top_level_import_name, -) + pass class InfoSpec(BaseModel): @@ -47,80 +38,6 @@ class PackageSpec(BaseModel): class Config: extra = Extra.forbid - @classmethod - def _from_wheel(cls, path: Path, info: InfoSpec) -> "PackageSpec": - """Build a package spec from an on-disk wheel. - - This is internal, because to reliably handle dependencies, we need: - 1) To have access to all the wheels being added at once (to handle extras) - 2) To know whether dependencies are available in the combined lockfile. - 3) To fix up wheel urls and paths consistently - - This is called by PyodideLockSpec.add_wheels below. - """ - from packaging.utils import ( - InvalidWheelFilename, - canonicalize_name, - parse_wheel_filename, - ) - from packaging.version import InvalidVersion - from packaging.version import parse as version_parse - - path = path.absolute() - # throw an error if this is an incompatible wheel - target_python = version_parse(info.python) - target_platform = info.platform + "_" + info.arch - try: - (name, version, build_number, tags) = parse_wheel_filename(str(path.name)) - except (InvalidWheelFilename, InvalidVersion) as e: - raise RuntimeError(f"Wheel filename {path.name} is not valid") from e - python_binary_abi = f"cp{target_python.major}{target_python.minor}" - tags = list(tags) - tag_match = False - for t in tags: - # abi should be - if ( - t.abi == python_binary_abi - and t.interpreter == python_binary_abi - and t.platform == target_platform - ): - tag_match = True - elif t.abi == "none" and t.platform == "any": - match = re.match(rf"py{target_python.major}(\d*)", t.interpreter) - if match: - subver = match.group(1) - if len(subver) == 0 or int(subver) <= target_python.minor: - tag_match = True - if not tag_match: - raise RuntimeError( - f"Package tags {tags} don't match Python version in lockfile:" - f"Lockfile python {target_python.major}.{target_python.minor}" - f"on platform {target_platform} ({python_binary_abi})" - ) - metadata = _wheel_metadata(path) - - if not metadata: - raise RuntimeError(f"Could not parse wheel metadata from {path.name}") - - # returns a draft PackageSpec with: - # 1) absolute path to wheel, - # 2) empty dependency list - return PackageSpec( - name=canonicalize_name(metadata.name), - version=metadata.version, - file_name=str(path), - sha256=_generate_package_hash(path), - package_type="package", - install_dir="site", - imports=parse_top_level_import_name(path), - depends=[], - ) - - def update_sha256(self, path: Path) -> "PackageSpec": - """Update the sha256 hash for a package.""" - self.sha256 = _generate_package_hash(path) - return self - class PyodideLockSpec(BaseModel): """A specification for the pyodide-lock.json file.""" @@ -175,143 +92,3 @@ def check_wheel_filenames(self) -> None: for name, errs in errors.items() ) raise ValueError(error_msg) - - def add_wheels( - self, - wheel_files: list[Path], - base_path: Path | None = None, - base_url: str = "", - ignore_missing_dependencies: bool = False, - ) -> None: - """Add a list of wheel files to this pyodide-lock.json - - Args: - wheel_files (list[Path]): A list of wheel files to import. - base_path (Path | None, optional): - Filenames are stored relative to this base path. By default the - filename is stored relative to the path of the first wheel file - in the list. - - base_url (str, optional): - The base URL stored in the pyodide-lock.json. By default this - is empty which means that wheels must be stored in the same folder - as the core pyodide packages you are using. If you want to store - your custom wheels somewhere else, set this base_url to point to it. - """ - if len(wheel_files) <= 0: - return - wheel_files = [f.resolve() for f in wheel_files] - if base_path is None: - base_path = wheel_files[0].parent - else: - base_path = base_path.resolve() - - new_packages = {} - for f in wheel_files: - spec = PackageSpec._from_wheel(f, info=self.info) - - new_packages[spec.name] = spec - - self._fix_new_package_deps(new_packages, ignore_missing_dependencies) - self._set_package_paths(new_packages, base_path, base_url) - self.packages |= new_packages - - def _fix_new_package_deps( - self, new_packages: dict[str, PackageSpec], ignore_missing_dependencies: bool - ): - # now fix up the dependencies for each of our new packages - # n.b. this assumes existing packages have correct dependencies, - # which is probably a good assumption. - from packaging.utils import canonicalize_name - - requirements_with_extras = [] - marker_environment = _get_marker_environment(**self.info.dict()) - for package in new_packages.values(): - # add any requirements to the list of packages - our_depends = [] - wheel_file = package.file_name - metadata = _wheel_metadata(wheel_file) - requirements = _wheel_depends(metadata) - for r in requirements: - req_marker = r.marker - req_name = canonicalize_name(r.name) - if req_marker is not None: - if not req_marker.evaluate(marker_environment): - # not used in pyodide / emscripten - # or optional requirement - continue - if r.extras: - # this requirement has some extras, we need to check - # that the required package depends on these extras also. - requirements_with_extras.append(r) - if req_name in new_packages or req_name in self.packages: - our_depends.append(req_name) - elif ignore_missing_dependencies: - our_depends.append(req_name) - else: - raise RuntimeError( - f"Requirement {req_name} from {r} is not in this distribution." - ) - package.depends = our_depends - while len(requirements_with_extras) != 0: - extra_req = requirements_with_extras.pop() - requirements_with_extras.extend( - self._fix_extra_dep( - extra_req, new_packages, ignore_missing_dependencies - ) - ) - - # When requirements have extras, we need to make sure that the - # required package includes the dependencies for that extra. - # This is because extras aren't supported in pyodide-lock - def _fix_extra_dep( - self, - extra_req: "Requirement", - new_packages: dict[str, PackageSpec], - ignore_missing_dependencies: bool, - ): - from packaging.utils import canonicalize_name - - requirements_with_extras = [] - - marker_environment = _get_marker_environment(**self.info.dict()) - extra_package_name = canonicalize_name(extra_req.name) - if extra_package_name not in new_packages: - return [] - package = new_packages[extra_package_name] - our_depends = package.depends - wheel_file = package.file_name - metadata = _wheel_metadata(wheel_file) - requirements = _wheel_depends(metadata) - for extra in extra_req.extras: - this_marker_env = marker_environment.copy() - this_marker_env["extra"] = extra - - for r in requirements: - req_marker = r.marker - req_name = canonicalize_name(r.name) - if req_name not in our_depends: - if req_marker is None: - # no marker - this will have been processed above - continue - if req_marker.evaluate(this_marker_env): - if req_name in new_packages or req_name in self.packages: - our_depends.append(req_name) - if r.extras: - requirements_with_extras.append(r) - elif ignore_missing_dependencies: - our_depends.append(req_name) - else: - raise RuntimeError( - f"Requirement {req_name} is not in this distribution." - ) - package.depends = our_depends - return requirements_with_extras - - def _set_package_paths( - self, new_packages: dict[str, PackageSpec], base_path: Path, base_url: str - ): - for p in new_packages.values(): - current_path = Path(p.file_name) - relative_path = current_path.relative_to(base_path) - p.file_name = base_url + str(relative_path) diff --git a/pyodide_lock/utils.py b/pyodide_lock/utils.py index 5de83e8..d4a7db0 100644 --- a/pyodide_lock/utils.py +++ b/pyodide_lock/utils.py @@ -5,7 +5,9 @@ from collections import deque from functools import cache from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING # + +from .spec import InfoSpec, PackageSpec, PyodideLockSpec if TYPE_CHECKING: from packaging.requirements import Requirement @@ -153,3 +155,225 @@ def _wheel_depends(metadata: "Distribution") -> list["Requirement"]: depends.append(req) return depends + + +def add_wheels_to_spec( + lock_spec: PyodideLockSpec, + wheel_files: list[Path], + base_path: Path | None = None, + base_url: str = "", + ignore_missing_dependencies: bool = False, +) -> None: + """Add a list of wheel files to this pyodide-lock.json + + Args: + wheel_files (list[Path]): A list of wheel files to import. + base_path (Path | None, optional): + Filenames are stored relative to this base path. By default the + filename is stored relative to the path of the first wheel file + in the list. + + base_url (str, optional): + The base URL stored in the pyodide-lock.json. By default this + is empty which means that wheels must be stored in the same folder + as the core pyodide packages you are using. If you want to store + your custom wheels somewhere else, set this base_url to point to it. + """ + if len(wheel_files) <= 0: + return + wheel_files = [f.resolve() for f in wheel_files] + if base_path is None: + base_path = wheel_files[0].parent + else: + base_path = base_path.resolve() + + new_packages = {} + for f in wheel_files: + spec = package_spec_from_wheel(f, info=lock_spec.info) + + new_packages[spec.name] = spec + + _fix_new_package_deps(lock_spec, new_packages, ignore_missing_dependencies) + _set_package_paths(new_packages, base_path, base_url) + lock_spec.packages |= new_packages + + +def _fix_new_package_deps( + lock_spec: PyodideLockSpec, + new_packages: dict[str, PackageSpec], + ignore_missing_dependencies: bool, +): + # now fix up the dependencies for each of our new packages + # n.b. this assumes existing packages have correct dependencies, + # which is probably a good assumption. + from packaging.utils import canonicalize_name + + requirements_with_extras = [] + marker_environment = _get_marker_environment(**lock_spec.info.dict()) + for package in new_packages.values(): + # add any requirements to the list of packages + our_depends = [] + wheel_file = package.file_name + metadata = _wheel_metadata(wheel_file) + requirements = _wheel_depends(metadata) + for r in requirements: + req_marker = r.marker + req_name = canonicalize_name(r.name) + if req_marker is not None: + if not req_marker.evaluate(marker_environment): + # not used in pyodide / emscripten + # or optional requirement + continue + if r.extras: + # this requirement has some extras, we need to check + # that the required package depends on these extras also. + requirements_with_extras.append(r) + if req_name in new_packages or req_name in lock_spec.packages: + our_depends.append(req_name) + elif ignore_missing_dependencies: + our_depends.append(req_name) + else: + raise RuntimeError( + f"Requirement {req_name} from {r} is not in this distribution." + ) + package.depends = our_depends + while len(requirements_with_extras) != 0: + extra_req = requirements_with_extras.pop() + requirements_with_extras.extend( + _fix_extra_dep( + lock_spec, extra_req, new_packages, ignore_missing_dependencies + ) + ) + + +# When requirements have extras, we need to make sure that the +# required package includes the dependencies for that extra. +# This is because extras aren't supported in pyodide-lock +def _fix_extra_dep( + lock_spec: PyodideLockSpec, + extra_req: "Requirement", + new_packages: dict[str, PackageSpec], + ignore_missing_dependencies: bool, +): + from packaging.utils import canonicalize_name + + requirements_with_extras = [] + + marker_environment = _get_marker_environment(**lock_spec.info.dict()) + extra_package_name = canonicalize_name(extra_req.name) + if extra_package_name not in new_packages: + return [] + package = new_packages[extra_package_name] + our_depends = package.depends + wheel_file = package.file_name + metadata = _wheel_metadata(wheel_file) + requirements = _wheel_depends(metadata) + for extra in extra_req.extras: + this_marker_env = marker_environment.copy() + this_marker_env["extra"] = extra + + for r in requirements: + req_marker = r.marker + req_name = canonicalize_name(r.name) + if req_name not in our_depends: + if req_marker is None: + # no marker - this will have been processed above + continue + if req_marker.evaluate(this_marker_env): + if req_name in new_packages or req_name in lock_spec.packages: + our_depends.append(req_name) + if r.extras: + requirements_with_extras.append(r) + elif ignore_missing_dependencies: + our_depends.append(req_name) + else: + raise RuntimeError( + f"Requirement {req_name} is not in this distribution." + ) + package.depends = our_depends + return requirements_with_extras + + +def _set_package_paths( + new_packages: dict[str, PackageSpec], base_path: Path, base_url: str +): + for p in new_packages.values(): + current_path = Path(p.file_name) + relative_path = current_path.relative_to(base_path) + p.file_name = base_url + str(relative_path) + + +def package_spec_from_wheel(path: Path, info: InfoSpec) -> "PackageSpec": + """Build a package spec from an on-disk wheel. + + Warning - to reliably handle dependencies, we need: + 1) To have access to all the wheels being added at once (to handle extras) + 2) To know whether dependencies are available in the combined lockfile. + 3) To fix up wheel urls and paths consistently + + This is called by add_wheels_to_spec + """ + from packaging.utils import ( + InvalidWheelFilename, + canonicalize_name, + parse_wheel_filename, + ) + from packaging.version import InvalidVersion + from packaging.version import parse as version_parse + + path = path.absolute() + # throw an error if this is an incompatible wheel + target_python = version_parse(info.python) + target_platform = info.platform + "_" + info.arch + try: + (name, version, build_number, tags) = parse_wheel_filename(str(path.name)) + except (InvalidWheelFilename, InvalidVersion) as e: + raise RuntimeError(f"Wheel filename {path.name} is not valid") from e + python_binary_abi = f"cp{target_python.major}{target_python.minor}" + tags = list(tags) + + tag_match = False + for t in tags: + # abi should be + if ( + t.abi == python_binary_abi + and t.interpreter == python_binary_abi + and t.platform == target_platform + ): + tag_match = True + elif t.abi == "none" and t.platform == "any": + match = re.match(rf"py{target_python.major}(\d*)", t.interpreter) + if match: + subver = match.group(1) + if len(subver) == 0 or int(subver) <= target_python.minor: + tag_match = True + if not tag_match: + raise RuntimeError( + f"Package tags {tags} don't match Python version in lockfile:" + f"Lockfile python {target_python.major}.{target_python.minor}" + f"on platform {target_platform} ({python_binary_abi})" + ) + metadata = _wheel_metadata(path) + + if not metadata: + raise RuntimeError(f"Could not parse wheel metadata from {path.name}") + + # returns a draft PackageSpec with: + # 1) absolute path to wheel, + # 2) empty dependency list + return PackageSpec( + name=canonicalize_name(metadata.name), + version=metadata.version, + file_name=str(path), + sha256=_generate_package_hash(path), + package_type="package", + install_dir="site", + imports=parse_top_level_import_name(path), + depends=[], + ) + + +def update_package_sha256(spec: PackageSpec, path: Path) -> "PackageSpec": + """Update the sha256 hash for a package.""" + spec.sha256 = _generate_package_hash(path) + return spec diff --git a/tests/test_spec.py b/tests/test_spec.py index 7d29ae2..5613e6a 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -7,6 +7,7 @@ from pyodide_lock import PyodideLockSpec from pyodide_lock.spec import InfoSpec, PackageSpec +from pyodide_lock.utils import update_package_sha256 DATA_DIR = Path(__file__).parent / "data" @@ -98,13 +99,13 @@ def test_to_json_indent(tmp_path): def test_update_sha256(monkeypatch): - monkeypatch.setattr("pyodide_lock.spec._generate_package_hash", lambda x: "abcd") + monkeypatch.setattr("pyodide_lock.utils._generate_package_hash", lambda x: "abcd") lock_data = deepcopy(LOCK_EXAMPLE) lock_data["packages"]["numpy"]["sha256"] = "0" # type: ignore[index] spec = PyodideLockSpec(**lock_data) assert spec.packages["numpy"].sha256 == "0" - spec.packages["numpy"].update_sha256(Path("/some/path")) + update_package_sha256(spec.packages["numpy"], Path("/some/path")) assert spec.packages["numpy"].sha256 == "abcd" diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 7395874..08464c6 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -7,11 +7,14 @@ import build import pytest +from test_spec import LOCK_EXAMPLE from pyodide_lock import PackageSpec, PyodideLockSpec -from pyodide_lock.utils import _generate_package_hash, _get_marker_environment - -from .test_spec import LOCK_EXAMPLE +from pyodide_lock.utils import ( + _generate_package_hash, + _get_marker_environment, + add_wheels_to_spec, +) # we test if our own wheel imports nicely # so check if it is built in /dist, or else skip that test @@ -130,7 +133,8 @@ class TestWheel: def test_add_one(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[0:1]) + + add_wheels_to_spec(spec, test_wheel_list[0:1]) # py_one only should get added assert spec.packages["py-one"].imports == ["one"] @@ -138,7 +142,7 @@ def test_add_one(test_wheel_list): def test_add_simple_deps(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[0:3]) + add_wheels_to_spec(spec, test_wheel_list[0:3]) # py_one, needs_one and needs_one_opt should get added assert "py-one" in spec.packages assert "needs-one" in spec.packages @@ -152,7 +156,7 @@ def test_add_simple_deps(test_wheel_list): def test_add_deps_with_extras(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[0:4]) + add_wheels_to_spec(spec, test_wheel_list[0:4]) # py_one, needs_one, needs_one_opt and test_extra_dependencies should get added # because of the extra dependency in test_extra_dependencies, # needs_one_opt should now depend on one @@ -165,13 +169,13 @@ def test_missing_dep(test_wheel_list): spec = PyodideLockSpec(**lock_data) # this has a package with a missing dependency so should fail with pytest.raises(RuntimeError): - spec.add_wheels(test_wheel_list[0:5]) + add_wheels_to_spec(spec, test_wheel_list[0:5]) def test_path_rewriting(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[0:3], base_url="http://www.nowhere.org/") + add_wheels_to_spec(spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/") # py_one, needs_one and needs_one_opt should get added assert "py-one" in spec.packages assert "needs-one" in spec.packages @@ -181,7 +185,8 @@ def test_path_rewriting(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) # this should add the base path "dist" to the file name - spec.add_wheels( + add_wheels_to_spec( + spec, test_wheel_list[0:3], base_url="http://www.nowhere.org/", base_path=test_wheel_list[0].parent.parent, @@ -199,7 +204,7 @@ def test_path_rewriting(test_wheel_list): def test_markers_not_needed(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[5:6]) + add_wheels_to_spec(spec, test_wheel_list[5:6]) assert spec.packages["markers-not-needed-test"].depends == [] @@ -208,7 +213,7 @@ def test_markers_not_needed(test_wheel_list): def test_markers_needed(test_wheel_list): lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels(test_wheel_list[6:7], ignore_missing_dependencies=True) + add_wheels_to_spec(spec, test_wheel_list[6:7], ignore_missing_dependencies=True) assert len(spec.packages["markers-needed-test"].depends) == len( MARKER_EXAMPLES_NEEDED ) @@ -219,7 +224,7 @@ def test_self_wheel(): assert WHEEL is not None lock_data = deepcopy(LOCK_EXAMPLE) spec = PyodideLockSpec(**lock_data) - spec.add_wheels([WHEEL], ignore_missing_dependencies=True) + add_wheels_to_spec(spec, [WHEEL], ignore_missing_dependencies=True) expected = PackageSpec( name="pyodide-lock", @@ -245,7 +250,7 @@ def test_not_wheel(tmp_path): whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="metadata"): - spec.add_wheels([wheel]) + add_wheels_to_spec(spec, [wheel]) @pytest.mark.parametrize( @@ -262,4 +267,4 @@ def test_bad_names(tmp_path, bad_name): with zipfile.ZipFile(wheel, "w") as whlzip: whlzip.writestr("README.md", data="Not a wheel") with pytest.raises(RuntimeError, match="Wheel filename"): - spec.add_wheels([wheel]) + add_wheels_to_spec(spec, [wheel])