From 0adc6d050ac2513396cdd0eade2bd4b01849370d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 7 Nov 2024 07:54:12 +0100 Subject: [PATCH 01/15] bump_version.py: `ast.Str` will be removed in Py314 so use `ast.Constant` (#5164) --- tools/bump_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bump_version.py b/tools/bump_version.py index 1e5c81cab41..e5cec786793 100755 --- a/tools/bump_version.py +++ b/tools/bump_version.py @@ -6,7 +6,7 @@ import itertools import pathlib import re -from ast import Str +from ast import Constant from collections import namedtuple from collections.abc import Callable @@ -81,7 +81,7 @@ def build_version_pattern(pattern): @functools.lru_cache -def python_version_to_js_version(version: str) -> Str: +def python_version_to_js_version(version: str) -> Constant: """ Convert Python version name to JS version name These two are different in prerelease or dev versions. From ce29b59902dd776aa62584d0c3de345370ee8e9c Mon Sep 17 00:00:00 2001 From: Joel Ostblom Date: Thu, 7 Nov 2024 02:04:54 -0800 Subject: [PATCH 02/15] Update altair to 5.4.1 (#5124) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/project/changelog.md | 1 + packages/altair/meta.yaml | 10 ++++------ packages/altair/test_altair.py | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 1983c0829a9..971f898a208 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -86,6 +86,7 @@ myst: - Upgraded `duckdb` to 1.1.0 {pr}`5078` - Upgraded `sympy` to 1.13.3 {pr}`5098` - Upgraded `tree-sitter` to 0.23.1 {pr}`5110` +- Upgraded `altair` to 5.4.1 {pr}`5124` - Upgraded `PyYAML` to 6.0.2 {pr}`5137` - Upgraded `duckdb` to 1.1.2 {pr}`5142` - Added `soxr` 0.5.0.post1 {pr}`5150` diff --git a/packages/altair/meta.yaml b/packages/altair/meta.yaml index ff8b2ef53c0..6cf5d16dae5 100644 --- a/packages/altair/meta.yaml +++ b/packages/altair/meta.yaml @@ -1,11 +1,11 @@ package: name: altair - version: 5.3.0 + version: 5.4.1 top-level: - altair source: - url: https://files.pythonhosted.org/packages/46/30/2118537233fa72c1d91a81f5908a7e843a6601ccc68b76838ebc4951505f/altair-5.3.0-py3-none-any.whl - sha256: 7084a1dab4d83c5e7e5246b92dc1b4451a6c68fd057f3716ee9d315c8980e59a + url: https://files.pythonhosted.org/packages/9b/52/4a86a4fa1cc2aae79137cc9510b7080c3e5aede2310d14fae5486feec7f7/altair-5.4.1-py3-none-any.whl + sha256: 0fb130b8297a569d08991fb6fe763582e7569f8a04643bbd9212436e3be04aef about: home: https://altair-viz.github.io/ PyPI: https://pypi.org/project/altair @@ -19,7 +19,5 @@ requirements: - typing-extensions - Jinja2 - jsonschema - - numpy - - pandas - - toolz - packaging + - narwhals diff --git a/packages/altair/test_altair.py b/packages/altair/test_altair.py index 676603bb207..5159fa91c9e 100644 --- a/packages/altair/test_altair.py +++ b/packages/altair/test_altair.py @@ -6,17 +6,19 @@ @run_in_pyodide(packages=["altair"]) def test_altair(selenium): import altair as alt - import pandas as pd - data = pd.DataFrame( - { - "a": ["A", "B", "C", "D", "E", "F", "G", "H", "I"], - "b": [28, 55, 43, 91, 81, 53, 19, 87, 52], - } + data = alt.Data( + values=[ + {"a": "A", "b": 5}, + {"a": "B", "b": 3}, + {"a": "C", "b": 6}, + {"a": "D", "b": 7}, + {"a": "E", "b": 2}, + ] ) - c = alt.Chart(data).mark_bar().encode(x="a", y="b").to_dict() + c = alt.Chart(data).mark_bar().encode(x="a:N", y="b:Q").to_dict() assert c["mark"]["type"] == "bar" assert c["encoding"]["x"]["field"] == "a" assert c["encoding"]["y"]["type"] == "quantitative" - assert "name" in c["data"] + assert "values" in c["data"] From 782e8c6586433bff5ba03fd3b9563256716337fb Mon Sep 17 00:00:00 2001 From: Yan Wong Date: Thu, 7 Nov 2024 10:05:08 +0000 Subject: [PATCH 03/15] Update msprime to 1.3.3 (#5159) --- docs/project/changelog.md | 1 + packages/msprime/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 971f898a208..b66eb9a9f5e 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,7 @@ myst: ### Packages +- Upgraded `msprime` to 1.3.3 {pr}`5159` - Upgraded `tskit` to 0.6.0 {pr}`5157` - Upgraded `pydantic_core` to 2.25.1 {pr}`5151` - Upgraded `pydantic` to 2.9.2 {pr}`5151` diff --git a/packages/msprime/meta.yaml b/packages/msprime/meta.yaml index ae16db491d0..35940636089 100644 --- a/packages/msprime/meta.yaml +++ b/packages/msprime/meta.yaml @@ -1,11 +1,11 @@ package: name: msprime - version: 1.3.1 + version: 1.3.3 top-level: - msprime source: - url: https://files.pythonhosted.org/packages/9c/c6/d2c24086da4492fe2c9d281d237b5fd09e5bf5c92a8956efad7522cf96b1/msprime-1.3.1.tar.gz - sha256: b3f62cd516ad2e43c8412ea1f242acad1bc939391cfe9caa19625874e2e34835 + url: https://files.pythonhosted.org/packages/70/84/72a914d91c3e9cea91b8089ddc2a32cc9bdee50eda7e8900be6be70d1fdb/msprime-1.3.3.tar.gz + sha256: d8ae798076167f632b8fd7eccd5089faaa84c9034ca69422f886425f94a5b01b build: script: | export LIBGSL_INCLUDE_PATH=$(pkg-config --cflags-only-I --dont-define-prefix gsl) From 1c82138a272e77d5aae420adbc522f11e1fe4608 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 7 Nov 2024 21:04:14 +0900 Subject: [PATCH 04/15] Use pyodide-build submodule (#5152) Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- .circleci/config.yml | 10 ++++ .github/workflows/main.yml | 11 +++- .../workflows/update_cross_build_releases.yml | 2 + .gitmodules | 3 + Makefile | 4 +- Makefile.envs | 9 --- docs/development/contributing.md | 2 +- docs/development/maintainers.md | 23 +++++++- docs/development/new-packages.md | 2 +- pyodide-build | 1 + src/tests/test_cmdline_runner.py | 2 +- tools/check_and_install_pyodide_build.py | 59 ------------------- 12 files changed, 50 insertions(+), 78 deletions(-) create mode 100644 .gitmodules create mode 160000 pyodide-build delete mode 100755 tools/check_and_install_pyodide_build.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d98476b2a0..6a72acdc310 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,8 @@ jobs: - DISABLE_DYLINK: << parameters.disable_dylink >> steps: - checkout + - run: git submodule sync + - run: git submodule update --init - restore_cache: keys: @@ -132,6 +134,8 @@ jobs: resource_class: large steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -190,6 +194,8 @@ jobs: resource_class: large steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -219,6 +225,8 @@ jobs: <<: *defaults steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -246,6 +254,8 @@ jobs: <<: *defaults steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2327f9dc549..fcbb88f8d9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,6 +27,8 @@ jobs: needs: get_python_version steps: - uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ needs.get_python_version.outputs.PYVERSION }} @@ -34,8 +36,8 @@ jobs: shell: bash -l {0} run: | mkdir test-results - python3 -m pip install -r requirements.txt -r requirements-deploy.txt make pyodide_build + python3 -m pip install -r requirements.txt -r requirements-deploy.txt - name: Run tests shell: bash -l {0} run: | @@ -43,7 +45,7 @@ jobs: --junitxml=test-results/junit.xml \ --verbose \ --runtime=host \ - --cov=pyodide_build --cov=pyodide \ + --cov=pyodide \ src packages/_tests tools/ - uses: codecov/codecov-action@v4 with: @@ -68,7 +70,8 @@ jobs: steps: - uses: actions/checkout@v4 - + with: + submodules: recursive - name: Cache ccache output uses: actions/cache@v4 with: @@ -169,6 +172,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Download build artifact uses: actions/download-artifact@v4 diff --git a/.github/workflows/update_cross_build_releases.yml b/.github/workflows/update_cross_build_releases.yml index bcfbda5f1e3..4d75d8a03d7 100644 --- a/.github/workflows/update_cross_build_releases.yml +++ b/.github/workflows/update_cross_build_releases.yml @@ -15,6 +15,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..b32d197f231 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pyodide-build"] + path = pyodide-build + url = https://github.com/pyodide/pyodide-build diff --git a/Makefile b/Makefile index 218619f2cc1..f4f10870b40 100644 --- a/Makefile +++ b/Makefile @@ -225,8 +225,8 @@ $(eval $(call preprocess-js,js2python.js)) .PHONY: pyodide_build pyodide_build: - @echo "Ensuring required pyodide-build version is installed" - ./tools/check_and_install_pyodide_build.py "$(PYODIDE_BUILD_COMMIT)" --repo "$(PYODIDE_BUILD_REPO)" + @echo "Ensuring pyodide-build is installed" + pip install -e ./pyodide-build @which pyodide >/dev/null diff --git a/Makefile.envs b/Makefile.envs index 2b626695d60..b48c6b3529b 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -5,15 +5,6 @@ export PYODIDE_ABI_VERSION ?= 2024_0 export PYTHON_ARCHIVE_SHA256=d01ec6a33bc10009b09c17da95cc2759af5a580a7316b3a446eb4190e13f97b2 -# Update the following variables when you want to update the version of the -# pyodide-build version used in the build process. If you want to make breaking -# changes in pyodide-build, it is also useful to change the repository URL -# to your fork to test the changes are working as expected. - -# v0.27.3 -export PYODIDE_BUILD_COMMIT=fac0109aa2acf14469320b049d710dd42639bf94 -export PYODIDE_BUILD_REPO=https://github.com/pyodide/pyodide-build - ifdef CPYTHON_DEBUG export CPYTHON_ABI_FLAGS=d endif diff --git a/docs/development/contributing.md b/docs/development/contributing.md index 92283fc807a..b1c7529304c 100644 --- a/docs/development/contributing.md +++ b/docs/development/contributing.md @@ -29,7 +29,7 @@ To contribute code, see the following steps, 3. Clone your fork of Pyodide ``` - git clone https://github.com//pyodide.git + git clone --recursive https://github.com//pyodide.git ``` and add the upstream remote, ``` diff --git a/docs/development/maintainers.md b/docs/development/maintainers.md index f4caa870bf2..0d0e2033d27 100644 --- a/docs/development/maintainers.md +++ b/docs/development/maintainers.md @@ -193,8 +193,27 @@ note to update them independently. ## Updating pyodide-build -to change the version of pyodide-build, update the PYODIDE_BUILD_REPO and PYODIDE_BUILD_COMMIT -variables in `Makefile.envs`. +to change the version of pyodide-build, change the commit of the pyodide-build submodule. + +```bash +cd pyodide-build +git checkout "" +``` + +to test with the fork of pyodide-build, change the `.gitmodules` file to point to your fork and update the commit hash + +```ini +# .gitmodules +[submodule "pyodide-build"] + path = pyodide-build + url = https://github.com//pyodide-build +``` + +```bash +git submodule sync +cd pyodide-build +git checkout " str | None: - """ - Return the version of the pyodide-build package or the URL to the repository. - """ - freeze_result = sp.check_output( - [ - sys.executable, - "-m", - "pip", - "freeze", - ] - ) - - for line in freeze_result.decode().split("\n"): - if line.startswith("pyodide-build"): - try: - return line.split(" @ ")[1] - except IndexError: - print("pyodide-build is not installed from a VCS: ", line) - return None - - return None - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("commit", type=str) - parser.add_argument( - "--repo", type=str, default="https://github.com/pyodide/pyodide-build" - ) - - return parser.parse_args() - - -def main(): - args = parse_args() - install_url = f"git+{args.repo}@{args.commit}" - installed_url = get_pyodide_build_install_url() - - if not installed_url or installed_url != install_url: - sp.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - install_url, - ] - ) - - -if __name__ == "__main__": - main() From 24dfc07a4ecdcc702ea60d0bd3136e7e6b5ce06e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:32:39 +0800 Subject: [PATCH 05/15] Add missing Pyodide v0.26.3 cross-build environment metadata (#5168) --- pyodide-cross-build-environments.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyodide-cross-build-environments.json b/pyodide-cross-build-environments.json index f60588ccdbb..d69c3d5437e 100644 --- a/pyodide-cross-build-environments.json +++ b/pyodide-cross-build-environments.json @@ -8,6 +8,14 @@ "emscripten_version": "3.1.58", "min_pyodide_build_version": "0.26.0" }, + "0.26.3": { + "version": "0.26.3", + "url": "https://github.com/pyodide/pyodide/releases/download/0.26.3/xbuildenv-0.26.3.tar.bz2", + "sha256": "538b00bb7b329f96e6f981464bc7875109b9779f8d121e2140235c375f530675", + "python_version": "3.12.1", + "emscripten_version": "3.1.58", + "min_pyodide_build_version": "0.26.0" + }, "0.26.2": { "version": "0.26.2", "url": "https://github.com/pyodide/pyodide/releases/download/0.26.2/xbuildenv-0.26.2.tar.bz2", From 651b690a6c42184d8f910784dc35afcad34ab7c8 Mon Sep 17 00:00:00 2001 From: Hanno Rein Date: Thu, 7 Nov 2024 13:13:08 -0500 Subject: [PATCH 06/15] Bump rebound/reboundx to version 4.4.3/4.3.0 (#5163) --- docs/project/changelog.md | 2 + packages/rebound/meta.yaml | 8 ++- .../patches/0001-fix-install_name.patch | 38 ++++++++----- .../rebound/patches/0002-fix-output.patch | 57 +++++++++++++++++++ .../rebound/patches/0003-no-emscripten.patch | 49 ++++++++++++++++ packages/reboundx/meta.yaml | 6 +- 6 files changed, 141 insertions(+), 19 deletions(-) create mode 100644 packages/rebound/patches/0002-fix-output.patch create mode 100644 packages/rebound/patches/0003-no-emscripten.patch diff --git a/docs/project/changelog.md b/docs/project/changelog.md index b66eb9a9f5e..7f1e704f9bb 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,8 @@ myst: ### Packages +- Upgraded `rebound` to 4.4.3 {pr}`5163` +- Upgraded `reboundx` to 4.3.0 {pr}`5163` - Upgraded `msprime` to 1.3.3 {pr}`5159` - Upgraded `tskit` to 0.6.0 {pr}`5157` - Upgraded `pydantic_core` to 2.25.1 {pr}`5151` diff --git a/packages/rebound/meta.yaml b/packages/rebound/meta.yaml index a850257d793..717fad1f151 100644 --- a/packages/rebound/meta.yaml +++ b/packages/rebound/meta.yaml @@ -1,6 +1,6 @@ package: name: rebound - version: 3.24.2 + version: 4.4.3 top-level: - rebound build: @@ -12,10 +12,12 @@ build: cp librebound.cpython-*.so $WASM_LIBRARY_DIR/lib/ source: - url: https://files.pythonhosted.org/packages/5c/66/7564ac591bb088d7f35a59d560b5c26426bb8ef5415522232819fe11c45a/rebound-3.24.2.tar.gz - sha256: a3c5d4c6a10b9c1538e1051edcd6341ef3d13142b698ee9c4a1f112d9684d803 + url: https://files.pythonhosted.org/packages/9d/f5/93f5abaf2c796e35b775e6fd395c2fd3729c17701fcb90fb4ed052fdd465/rebound-4.4.3.tar.gz + sha256: 8dd2fc69f4db4609abeb50048583d0428c716cd3eab11df868358f5384d826cf patches: - patches/0001-fix-install_name.patch + - patches/0002-fix-output.patch + - patches/0003-no-emscripten.patch requirements: run: diff --git a/packages/rebound/patches/0001-fix-install_name.patch b/packages/rebound/patches/0001-fix-install_name.patch index 5cedb0b5d69..0e88a8e6bab 100644 --- a/packages/rebound/patches/0001-fix-install_name.patch +++ b/packages/rebound/patches/0001-fix-install_name.patch @@ -1,15 +1,27 @@ -Index: rebound-3.24.2/setup.py -=================================================================== ---- rebound-3.24.2.orig/setup.py -+++ rebound-3.24.2/setup.py -@@ -23,8 +23,8 @@ extra_link_args=[] +From 0a6cc79e6814d7e2dbc637541cad86cebc47f855 Mon Sep 17 00:00:00 2001 +From: Hanno Rein +Date: Wed, 6 Nov 2024 17:37:22 -0500 +Subject: [PATCH 3/3] install_name + +--- + setup.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/setup.py b/setup.py +index 6443cb27..965b2fc9 100644 +--- a/setup.py ++++ b/setup.py +@@ -23,8 +23,8 @@ except: + extra_link_args=[] if sys.platform == 'darwin': - from distutils import sysconfig - vars = sysconfig.get_config_vars() -- vars['LDSHARED'] = vars['LDSHARED'].replace('-bundle', '-shared') + config_vars = sysconfig.get_config_vars() +- config_vars['LDSHARED'] = config_vars['LDSHARED'].replace('-bundle', '-shared') - extra_link_args=['-Wl,-install_name,@rpath/librebound'+suffix] -+ # vars['LDSHARED'] = vars['LDSHARED'].replace('-bundle', '-shared') -+ # extra_link_args=['-Wl,-install_name,@rpath/librebound'+suffix] - - libreboundmodule = Extension('librebound', - sources = [ 'src/rebound.c', ++ #config_vars['LDSHARED'] = config_vars['LDSHARED'].replace('-bundle', '-shared') ++ #extra_link_args=['-Wl,-install_name,@rpath/librebound'+suffix] + if sys.platform == 'win32': + extra_compile_args=[ghash_arg, '-DLIBREBOUND', '-D_GNU_SOURCE', '-DSERVER'] + else: +-- +2.39.5 (Apple Git-154) + diff --git a/packages/rebound/patches/0002-fix-output.patch b/packages/rebound/patches/0002-fix-output.patch new file mode 100644 index 00000000000..1b7bc3296de --- /dev/null +++ b/packages/rebound/patches/0002-fix-output.patch @@ -0,0 +1,57 @@ +From f875c41a58ae4b3d59d97a5787ec3e546c737860 Mon Sep 17 00:00:00 2001 +From: Hanno Rein +Date: Wed, 6 Nov 2024 17:21:47 -0500 +Subject: [PATCH] output + +--- + src/output.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/output.c b/src/output.c +index e897743a..fdd4ea45 100644 +--- a/src/output.c ++++ b/src/output.c +@@ -349,7 +349,7 @@ void reb_simulation_output_timing(struct reb_simulation* r, const double tmax){ + r->output_timing_last = temp; + }else{ + #ifdef __EMSCRIPTEN__ +- reb_remove_last_line(); ++ // reb_remove_last_line(); + #else + printf("\r"); + #endif +--- a/src/output.c ++++ b/src/output.c +@@ -257,17 +257,17 @@ void profiling_stop(int cat){ + + #ifdef __EMSCRIPTEN__ + // fflush does not work in emscripten. Workaround. +-EM_JS(void, reb_remove_last_line, (), { +- var output = document.getElementById("output"); +- if (output){ +- const lastIndex1 = output.value.lastIndexOf("\n"); +- const lastIndex2 = output.value.lastIndexOf("\n",lastIndex1-1); +- const lastIndexNtot = output.value.lastIndexOf("N_tot="); +- if(lastIndex1>0 && lastIndex20 && lastIndex2 +Date: Thu, 7 Nov 2024 09:40:09 -0500 +Subject: [PATCH 4/4] no emscripten + +--- + src/rebound.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/rebound.c b/src/rebound.c +index 82f6ac60..accea978 100644 +--- a/src/rebound.c ++++ b/src/rebound.c +@@ -665,7 +665,7 @@ int reb_check_exit(struct reb_simulation* const r, const double tmax, double* la + while(r->status == REB_STATUS_PAUSED || r->status == REB_STATUS_SCREENSHOT){ + // Wait for user to disable paused simulation + #ifdef __EMSCRIPTEN__ +- emscripten_sleep(100); ++ // emscripten_sleep(100); + #else + usleep(1000); + #endif +@@ -820,16 +820,16 @@ static void* reb_simulation_integrate_raw(void* args){ + } + reb_run_heartbeat(r); + #ifdef __EMSCRIPTEN__ +- double t0 = emscripten_performance_now(); ++// double t0 = emscripten_performance_now(); + #endif + while(reb_check_exit(r,thread_info->tmax,&last_full_dt)<0){ + #ifdef __EMSCRIPTEN__ +- double t1 = emscripten_performance_now(); +- if (t1-t0>1000./120.){ // max framerate 120Hz +- t0 = t1; +- emscripten_sleep(0); // allow drawing and event handling +- } +- ++// double t1 = emscripten_performance_now(); ++// if (t1-t0>1000./120.){ // max framerate 120Hz ++// t0 = t1; ++// emscripten_sleep(0); // allow drawing and event handling ++// } ++// + #endif + #ifdef OPENGL + if (r->display_data){ +-- +2.39.5 (Apple Git-154) + diff --git a/packages/reboundx/meta.yaml b/packages/reboundx/meta.yaml index fa52882cb4d..1ff826103a9 100644 --- a/packages/reboundx/meta.yaml +++ b/packages/reboundx/meta.yaml @@ -1,6 +1,6 @@ package: name: reboundx - version: 3.10.1 + version: 4.3.0 top-level: - reboundx build: @@ -10,8 +10,8 @@ build: ldflags: | -L$(WASM_LIBRARY_DIR)/lib/ source: - url: https://files.pythonhosted.org/packages/14/08/c1c0b83c90ea43ea8e218beadb13332259e930a2ca62f85024be89d811f3/reboundx-3.10.1.tar.gz - sha256: 65b2fcecf296661aad15a81a8d7d7b7b96cc2ba834c89e49dba24445f146d642 + url: https://files.pythonhosted.org/packages/2c/c2/32bb671c3f6944755b90c75d20f387daf4b9fc1e6919264cb0e9e26af8d7/reboundx-4.3.0.tar.gz + sha256: 94fc5b0993b623faa9964800f87fbc97236f7f12cb639c659d25702f1e0ac0ec requirements: run: - rebound From 2a19f2ae44741c046e88525305fe60227252ccfc Mon Sep 17 00:00:00 2001 From: Nick Altmann Date: Thu, 7 Nov 2024 20:30:43 +0000 Subject: [PATCH 07/15] Add `rateslib` to packages (#5146) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- packages/rateslib/meta.yaml | 23 +++++++++++++++++++++++ packages/rateslib/test_rateslib.py | 17 +++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 packages/rateslib/meta.yaml create mode 100644 packages/rateslib/test_rateslib.py diff --git a/packages/rateslib/meta.yaml b/packages/rateslib/meta.yaml new file mode 100644 index 00000000000..0df2e18ebf8 --- /dev/null +++ b/packages/rateslib/meta.yaml @@ -0,0 +1,23 @@ +package: + name: rateslib + version: 1.5.0 + top-level: + - rateslib +source: + url: https://files.pythonhosted.org/packages/02/d7/7695bb1b6a240ecfae77804d3fc242e30582fd83973f76dda03b23d921a6/rateslib-1.5.0.tar.gz + sha256: 7c7af2707f20974a70403689f84905757a4ed084e4c92229388d2fc30b331b04 +requirements: + run: + - numpy + - pandas + - matplotlib + executable: + - rustup +about: + home: https://github.com/attack68/rateslib + PyPI: https://pypi.org/project/rateslib + summary: A fixed income library for trading interest rates + license: CC-BY-NC-ND-4.0 +extra: + recipe-maintainers: + - NickAltmann diff --git a/packages/rateslib/test_rateslib.py b/packages/rateslib/test_rateslib.py new file mode 100644 index 00000000000..d352dd28177 --- /dev/null +++ b/packages/rateslib/test_rateslib.py @@ -0,0 +1,17 @@ +import pytest +from pytest_pyodide import run_in_pyodide + + +@pytest.mark.driver_timeout(60) +@run_in_pyodide(packages=["rateslib"]) +def test_add_tenor(selenium): + import datetime + + import rateslib as rl + + start_date = datetime.datetime(2024, 10, 29) + end_date = datetime.datetime(2024, 10, 31) + + test_date = rl.add_tenor(start_date, "2b", "F", "nyc") + + assert end_date == test_date From 912ccca6f0fc9722e2e69ec2ad1868d62c27c10a Mon Sep 17 00:00:00 2001 From: airen1986 <45193110+airen1986@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:13:10 +0530 Subject: [PATCH 08/15] Set MAXIMUM_MEMORY to 4GB and rename TOTAL_MEMORY to INITIAL_MEMORY (#5167) MAXIMUM_MEMORY defaults to 2GB but it can be 4GB. Also the "TOTAL_MEMORY" setting has changed name to "INITIAL_MEMORY". --- Makefile.envs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile.envs b/Makefile.envs index b48c6b3529b..54fea4eb4a7 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -142,8 +142,9 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \ -s USE_ZLIB \ -s USE_BZIP2 \ -s FORCE_FILESYSTEM=1 \ - -s TOTAL_MEMORY=20971520 \ + -s INITIAL_MEMORY=20971520 \ -s ALLOW_MEMORY_GROWTH=1 \ + -s MAXIMUM_MEMORY=4GB \ -s EXPORT_ALL=1 \ -s FS_DEBUG=1 \ -s STACK_SIZE=5MB \ @@ -230,8 +231,9 @@ ifeq ($(DISABLE_DYLINK), 1) -s USE_ZLIB \ -s USE_BZIP2 \ -s FORCE_FILESYSTEM=1 \ - -s TOTAL_MEMORY=20971520 \ + -s INITIAL_MEMORY=20971520 \ -s ALLOW_MEMORY_GROWTH=1 \ + -s MAXIMUM_MEMORY=4GB \ -s STACK_SIZE=5MB \ -s ALLOW_TABLE_GROWTH \ -s FS_DEBUG=1 \ From ecf3f163a83776b6e7ec2122ed155977b24b5c26 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 8 Nov 2024 21:06:21 +0100 Subject: [PATCH 09/15] Drop duplicated array tests (#5174) --- src/tests/test_typeconversions.py | 228 ------------------------------ 1 file changed, 228 deletions(-) diff --git a/src/tests/test_typeconversions.py b/src/tests/test_typeconversions.py index 36151a80d18..6e9c9afdfe9 100644 --- a/src/tests/test_typeconversions.py +++ b/src/tests/test_typeconversions.py @@ -1657,234 +1657,6 @@ class A: assert res.a == 2 -@pytest.mark.parametrize("n", [1 << 31, 1 << 32, 1 << 33, 1 << 63, 1 << 64, 1 << 65]) -@run_in_pyodide -def test_very_large_length(selenium, n): - from unittest import TestCase - - from pyodide.code import run_js - - raises = TestCase().assertRaises( - OverflowError, msg=f"length {n} of object is larger than INT_MAX (2147483647)" - ) - - o = run_js(f"({{length : {n}}})") - with raises: - len(o) - - # 1. Set toStringTag to NodeList to force JsProxy to feature detect this object - # as an array - # 2. Return a very large length - # 3. JsProxy_subscript_array should successfully handle this and propagate the error. - a = run_js(f"({{[Symbol.toStringTag] : 'NodeList', length: {n}}})") - with raises: - a[-1] - - -@pytest.mark.parametrize( - "n", [-1, -2, -3, -100, -1 << 31, -1 << 32, -1 << 33, -1 << 63, -1 << 64, -1 << 65] -) -@run_in_pyodide -def test_negative_length(selenium, n): - from unittest import TestCase - - from pyodide.code import run_js - - raises = TestCase().assertRaises( - ValueError, msg=f"length {n} of object is negative" - ) - - o = run_js(f"({{length : {n}}})") - with raises: - len(o) - - # 1. Set toStringTag to NodeList to force JsProxy to feature detect this object - # as an array - # 2. Return a negative length - # 3. JsProxy_subscript_array should successfully handle this and propagate the error. - a = run_js(f"({{[Symbol.toStringTag] : 'NodeList', length: {n}}})") - with raises: - a[-1] - - -@std_hypothesis_settings -@given(l=st.lists(st.integers()), slice=st.slices(50)) -@example(l=[0, 1], slice=slice(None, None, -1)) -@example(l=list(range(4)), slice=slice(None, None, -2)) -@example(l=list(range(10)), slice=slice(-1, 12)) -@example(l=list(range(10)), slice=slice(12, -1)) -@example(l=list(range(10)), slice=slice(12, -1, -1)) -@example(l=list(range(10)), slice=slice(-1, 12, 2)) -@example(l=list(range(10)), slice=slice(12, -1, -1)) -@example(l=list(range(10)), slice=slice(12, -1, -2)) -@run_in_pyodide -def test_array_slices(selenium, l, slice): - expected = l[slice] - from pyodide.ffi import JsArray, to_js - - jsl = to_js(l) - assert isinstance(jsl, JsArray) - result = jsl[slice] - assert result.to_py() == expected - - -@std_hypothesis_settings -@given(l=st.lists(st.integers()), slice=st.slices(50)) -@example(l=[0, 1], slice=slice(None, None, -1)) -@example(l=list(range(4)), slice=slice(None, None, -2)) -@example(l=list(range(10)), slice=slice(-1, 12)) -@example(l=list(range(10)), slice=slice(12, -1)) -@example(l=list(range(10)), slice=slice(12, -1, -1)) -@example(l=list(range(10)), slice=slice(-1, 12, 2)) -@example(l=list(range(10)), slice=slice(12, -1, -1)) -@example(l=list(range(10)), slice=slice(12, -1, -2)) -@run_in_pyodide -def test_array_slice_del(selenium, l, slice): - from pyodide.ffi import JsArray, to_js - - jsl = to_js(l) - assert isinstance(jsl, JsArray) - del l[slice] - del jsl[slice] - assert jsl.to_py() == l - - -@st.composite -def list_slice_and_value(draw): - l = draw(st.lists(st.integers())) - step_one = draw(st.booleans()) - if step_one: - start = draw(st.integers(0, max(len(l) - 1, 0)) | st.none()) - stop = draw(st.integers(start, len(l)) | st.none()) - if draw(st.booleans()) and start is not None: - start -= len(l) - if draw(st.booleans()) and stop is not None: - stop -= len(l) - s = slice(start, stop) - vals = draw(st.lists(st.integers())) - else: - s = draw(st.slices(50)) - vals_len = len(l[s]) - vals = draw(st.lists(st.integers(), min_size=vals_len, max_size=vals_len)) - return (l, s, vals) - - -@std_hypothesis_settings -@given(lsv=list_slice_and_value()) -@example(lsv=(list(range(5)), slice(5, 2), [])) -@example(lsv=(list(range(5)), slice(2, 5, -1), [])) -@example(lsv=(list(range(5)), slice(5, 2), [-1, -2, -3])) -@run_in_pyodide -def test_array_slice_assign_1(selenium, lsv): - from pyodide.ffi import JsArray, to_js - - [l, s, v] = lsv - jsl = to_js(l) - assert isinstance(jsl, JsArray) - l[s] = v - jsl[s] = v - assert jsl.to_py() == l - - -@run_in_pyodide -def test_array_slice_assign_2(selenium): - import pytest - - from pyodide.ffi import JsArray, to_js - - l = list(range(10)) - with pytest.raises(ValueError) as exc_info_1a: - l[0:4:2] = [1, 2, 3, 4] - - jsl = to_js(l) - assert isinstance(jsl, JsArray) - with pytest.raises(ValueError) as exc_info_1b: - jsl[0:4:2] = [1, 2, 3, 4] - - l = list(range(10)) - with pytest.raises(ValueError) as exc_info_2a: - l[0:4:2] = [] - - with pytest.raises(ValueError) as exc_info_2b: - jsl[0:4:2] = [] - - with pytest.raises(TypeError) as exc_info_3a: - l[:] = 1 # type: ignore[call-overload] - - with pytest.raises(TypeError) as exc_info_3b: - jsl[:] = 1 # type: ignore[call-overload] - - assert exc_info_1a.value.args == exc_info_1b.value.args - assert exc_info_2a.value.args == exc_info_2b.value.args - assert exc_info_3a.value.args == exc_info_3b.value.args - - -@std_hypothesis_settings -@given(l1=st.lists(st.integers()), l2=st.lists(st.integers())) -@example(l1=[], l2=[]) -@example(l1=[], l2=[1]) -@run_in_pyodide -def test_array_extend(selenium_module_scope, l1, l2): - from pyodide.ffi import to_js - - l1js1 = to_js(l1) - l1js1.extend(l2) - - l1js2 = to_js(l1) - l1js2 += l2 - - l1.extend(l2) - - assert l1 == l1js1.to_py() - assert l1 == l1js2.to_py() - - -@run_in_pyodide -def test_typed_array(selenium): - from pyodide.code import run_js - - a = run_js("self.a = new Uint8Array([1,2,3,4]); a") - assert a[0] == 1 - assert a[-1] == 4 - a[-2] = 7 - assert run_js("self.a[2]") == 7 - - import pytest - - with pytest.raises(TypeError, match="does ?n[o']t support item deletion"): - del a[0] - - msg = "Slice subscripting isn't implemented for typed arrays" - with pytest.raises(NotImplementedError, match=msg): - a[:] - - msg = "Slice assignment isn't implemented for typed arrays" - with pytest.raises(NotImplementedError, match=msg): - a[:] = [-1, -2, -3, -4] - - assert not hasattr(a, "extend") - with pytest.raises(TypeError): - a += [1, 2, 3] - - -@pytest.mark.xfail_browsers(node="No document in node") -@run_in_pyodide -def test_html_array(selenium): - from pyodide.code import run_js - - x = run_js("document.querySelectorAll('*')") - assert run_js("(a, b) => a === b[0]")(x[0], x) - assert run_js("(a, b) => a === Array.from(b).pop()")(x[-1], x) - - import pytest - - with pytest.raises(TypeError, match="does ?n[o']t support item assignment"): - x[0] = 0 - - with pytest.raises(TypeError, match="does ?n[o']t support item deletion"): - del x[0] - - @run_in_pyodide def test_bind_attrs(selenium): from typing import Annotated From 870a67c9b15b27b4b4afe060eabe6e08110c221c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 9 Nov 2024 18:50:02 +0800 Subject: [PATCH 10/15] Bump `crc32c` to version 2.7.1 (#5169) --- docs/project/changelog.md | 1 + packages/crc32c/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 7f1e704f9bb..8e806b154a5 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,7 @@ myst: ### Packages +- Upgraded `crc32c` to 2.7.1 {pr}`5169` - Upgraded `rebound` to 4.4.3 {pr}`5163` - Upgraded `reboundx` to 4.3.0 {pr}`5163` - Upgraded `msprime` to 1.3.3 {pr}`5159` diff --git a/packages/crc32c/meta.yaml b/packages/crc32c/meta.yaml index 02a1483f756..988633c0a03 100644 --- a/packages/crc32c/meta.yaml +++ b/packages/crc32c/meta.yaml @@ -1,11 +1,11 @@ package: name: crc32c - version: "2.4" + version: 2.7.1 top-level: - crc32c source: - url: https://files.pythonhosted.org/packages/6f/19/47ac8d1d5b81a83272fe56d4cf425274437fd619ed0af9a5f9805484748c/crc32c-2.4.tar.gz - sha256: d985c4d9b1a1fd16c593d83f8735a8e4e156790a95338a1e0b199aac51ca1e5e + url: https://files.pythonhosted.org/packages/7f/4c/4e40cc26347ac8254d3f25b9f94710b8e8df24ee4dddc1ba41907a88a94d/crc32c-2.7.1.tar.gz + sha256: f91b144a21eef834d64178e01982bb9179c354b3e9e5f4c803b0e5096384968c about: home: https://github.com/ICRAR/crc32c PyPI: https://pypi.org/project/crc32c From 4c2878f01ec3053444ee187b1370b8c08f6dd798 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 9 Nov 2024 22:30:01 +0800 Subject: [PATCH 11/15] Add `zfp` and `zfpy` (#5172) Co-authored-by: Gyeongjae Choi --- .pre-commit-config.yaml | 2 +- docs/project/changelog.md | 1 + packages/libzfp/meta.yaml | 34 +++++++++ packages/zfpy/meta.yaml | 37 ++++++++++ .../zfpy/patches/Modernise-packaging.patch | 69 +++++++++++++++++ packages/zfpy/test_zfpy.py | 74 +++++++++++++++++++ 6 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 packages/libzfp/meta.yaml create mode 100644 packages/zfpy/meta.yaml create mode 100644 packages/zfpy/patches/Modernise-packaging.patch create mode 100644 packages/zfpy/test_zfpy.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b44923b4573..d27f0871a9c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,7 +82,7 @@ repos: name: mypy-tests args: [--ignore-missing-imports] files: ^(packages/|docs|/conftest.py|src/tests) - exclude: (^packages/.*/setup.py|/src|^packages/aiohttp/aiohttp_patch.py$) + exclude: (^packages/.*/setup.py|/src|^packages/aiohttp/aiohttp_patch.py$|^packages/zfpy/test_zfpy.py$) additional_dependencies: *mypy-deps - repo: https://github.com/pre-commit/mirrors-prettier diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 8e806b154a5..04a3cb4ea8d 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -105,6 +105,7 @@ myst: - Added `tree-sitter-java` 0.23.2 {pr}`5102` - Added `tree-sitter-python` 0.23.2 {pr}`5102` - Added `Narwhals` 1.9.4 {pr}`5121` +- Added `libzfp` and `zfpy` 1.0.1 {pr}`5172` ## Version 0.26.3 diff --git a/packages/libzfp/meta.yaml b/packages/libzfp/meta.yaml new file mode 100644 index 00000000000..7f66a86166d --- /dev/null +++ b/packages/libzfp/meta.yaml @@ -0,0 +1,34 @@ +package: + name: libzfp + version: 1.0.1 + tag: + - library + - static_library +source: + url: https://github.com/LLNL/zfp/archive/refs/tags/1.0.1.tar.gz + sha256: 4984db6a55bc919831966dd17ba5e47ca7ac58668f4fd278ebd98cd2200da66f + +build: + type: static_library + exports: requested + script: | + mkdir -p build + cd build + emcmake cmake ${CMAKE_ARGS} \ + -DBUILD_CFP=ON \ + -DBUILD_UTILITIES=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DZFP_ENABLE_PIC=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-s WASM=1" \ + -DZFP_WITH_OPENMP=OFF \ + -DCMAKE_INSTALL_PREFIX=${WASM_LIBRARY_DIR} \ + .. + + emmake make -j ${PYODIDE_JOBS:-3} install + +about: + home: https://zfp.llnl.gov/ + summary: Open-source software for compressed floating-point arrays + license: BSD-3-Clause diff --git a/packages/zfpy/meta.yaml b/packages/zfpy/meta.yaml new file mode 100644 index 00000000000..b883a3a76df --- /dev/null +++ b/packages/zfpy/meta.yaml @@ -0,0 +1,37 @@ +package: + name: zfpy + version: 1.0.1 + top-level: + - zfpy + +source: + # There's no sdist for zfpy on PyPI, so we get the tarball from the GitHub release instead + url: https://github.com/LLNL/zfp/archive/refs/tags/1.0.1.tar.gz + sha256: 4984db6a55bc919831966dd17ba5e47ca7ac58668f4fd278ebd98cd2200da66f + patches: + - patches/Modernise-packaging.patch + +build: + exports: requested + cflags: | + -I$(PYTHONINCLUDE) + -I$(WASM_LIBRARY_DIR)/include + ldflags: | + -L$(WASM_LIBRARY_DIR)/lib + +requirements: + host: + - libzfp + - numpy + run: + - numpy + +test: + imports: + - zfpy + +about: + home: https://zfp.llnl.gov/ + PyPI: https://pypi.org/project/zfpy/ + summary: zfp compression in Python + license: BSD-3-Clause diff --git a/packages/zfpy/patches/Modernise-packaging.patch b/packages/zfpy/patches/Modernise-packaging.patch new file mode 100644 index 00000000000..411f7835bf2 --- /dev/null +++ b/packages/zfpy/patches/Modernise-packaging.patch @@ -0,0 +1,69 @@ +From f36ef86e67185b16b65445092c40a3d75d0be45c Mon Sep 17 00:00:00 2001 +From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> +Date: Fri, 8 Nov 2024 02:46:00 +0530 +Subject: [PATCH] Modernise packaging + +This patch introduces a PEP 517/518 compliant build system using setuptools and +declarative metadata in pyproject.toml, and fixes syntax errors in setup.py. The +sources for the extension module are correctly specified as Cython source files. + +Some part of the credits go to the upstream +PR at https://github.com/LLNL/zfp/pull/234 + +Co-Authored-By: David M. Rogers <2243447+frobnitzem@users.noreply.github.com> +--- + pyproject.toml | 13 +++++++++++++ + setup.py | 18 +++++++++--------- + 2 files changed, 22 insertions(+), 9 deletions(-) + create mode 100644 pyproject.toml + +diff --git a/pyproject.toml b/pyproject.toml +new file mode 100644 +index 0000000..88adcc8 +--- /dev/null ++++ b/pyproject.toml +@@ -0,0 +1,13 @@ ++[build-system] ++requires = ["setuptools", "cython", "numpy"] ++build-backend = "setuptools.build_meta" ++ ++[project] ++name = "zfpy" ++version = "1.0.1" ++authors = [ ++ { name = "Peter Lindstrom", email = "zfp@llnl.gov" }, ++ { name = "Danielle Asher", email = "zfp@llnl.gov" }, ++] ++description = "zfp compression in Python" ++requires-python = ">=3.0" +diff --git a/setup.py b/setup.py +index fa2da6e..919fa7c 100644 +--- a/setup.py ++++ b/setup.py +@@ -2,14 +2,14 @@ from setuptools import setup, Extension + import numpy as np + + setup( +- name="zfpy", +- version="1.0.1", +- author="Peter Lindstrom, Danielle Asher", +- author_email="zfp@llnl.gov", + url="https://zfp.llnl.gov", +- description="zfp compression in Python", +- long_description="zfp is a compressed format for representing multidimensional floating-point and integer arrays. zfp provides compressed-array classes that support high throughput read and write random access to individual array elements. zfp also supports serial and parallel compression of whole arrays using both lossless and lossy compression with error tolerances. zfp is primarily written in C and C++ but also includes Python and Fortran bindings.", +- ext_modules=[Extension("zfpy", ["build/python/zfpy.c"], +- include_dirs=["include", np.get_include()], +- libraries=["zfp"], library_dirs=["build/lib64", "build/lib/Release"]), language_level = "3"] ++ ext_modules= ++ [ ++ Extension( ++ name="zfpy", ++ sources=["python/zfpy.pyx"], ++ include_dirs=["include", np.get_include()], ++ libraries=["zfp"], ++ library_dirs=["build/lib64", "build/lib"]) ++ ], + ) +-- +2.39.3 (Apple Git-146) + diff --git a/packages/zfpy/test_zfpy.py b/packages/zfpy/test_zfpy.py new file mode 100644 index 00000000000..0896f323d32 --- /dev/null +++ b/packages/zfpy/test_zfpy.py @@ -0,0 +1,74 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["zfpy", "numpy"]) +def test_compression(selenium): + import numpy as np + import zfpy + + my_array = np.arange(1, 20) + compressed_data = zfpy.compress_numpy(my_array) + decompressed_array = zfpy.decompress_numpy(compressed_data) + np.testing.assert_array_equal(my_array, decompressed_array) + + +@run_in_pyodide(packages=["zfpy", "numpy"]) +def test_compression_with_tolerance(selenium): + import numpy as np + import zfpy + + my_array = np.linspace(0, 1, 1000) + compressed_data = zfpy.compress_numpy(my_array, tolerance=1e-3) + decompressed_array = zfpy.decompress_numpy(compressed_data) + np.testing.assert_allclose(my_array, decompressed_array, atol=1e-3) + + +@run_in_pyodide(packages=["zfpy", "numpy"]) +def test_different_dimensions(selenium) -> None: + import numpy as np + import zfpy + + np.random.seed(42) + + # Test arrays; from 1D to 4D + for dimensions in range(1, 5): + # 1. test with uniform dimensions + shape1 = tuple([5] * dimensions) + array = np.random.rand(*shape1).astype(np.float64) + compressed1 = zfpy.compress_numpy(array, write_header=True) + decompressed1 = zfpy.decompress_numpy(compressed1) + np.testing.assert_array_equal(decompressed1, array) + + # 2. test with increasing dimensions + shape2 = tuple(range(2, 2 + dimensions)) + array = np.random.rand(*shape2).astype(np.float64) + compressed2 = zfpy.compress_numpy(array, write_header=True) + decompressed2 = zfpy.decompress_numpy(compressed2) + np.testing.assert_array_equal(decompressed2, array) + + +@run_in_pyodide(packages=["zfpy", "numpy"]) +def test_different_dtypes(selenium) -> None: + """Test ZFP compression/decompression with different numeric dtypes.""" + import numpy as np + import zfpy + + np.random.seed(42) + + shape = (5, 5) + num_elements = np.prod(shape) + + # Test floating-point types + for dtype in [np.float32, np.float64]: + elements = np.random.random_sample(num_elements) + array = np.reshape(elements, shape).astype(dtype) + compressed1 = zfpy.compress_numpy(array, write_header=True) + decompressed1 = zfpy.decompress_numpy(compressed1) + np.testing.assert_array_equal(decompressed1, array) + + # Test integer types + for dtype in [np.int32, np.int64]: + array = np.random.randint(low=-(2**30), high=2**30, size=shape, dtype=dtype) + compressed2 = zfpy.compress_numpy(array, write_header=True) + decompressed2 = zfpy.decompress_numpy(compressed2) + np.testing.assert_array_equal(decompressed2, array) From 993a8ee02f2bc3c2b5e1fca4c454d2eb2a0a968b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 10 Nov 2024 12:43:04 +0100 Subject: [PATCH 12/15] requirements.txt: Pin selenium==4.25.0 to fix ci failures (#5182) --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index a520fb49b29..55363655789 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,6 @@ pytest-cov pytest-httpserver pytest-benchmark pytest-pyodide==0.58.3 +setuptools; python_version >= '3.12' +# TODO(cclauss): Remove the Selenium line. +selenium==4.25.0 From 39e62fcb35d79c4340ce0e12062ea0cf402f1300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Gyarmati?= Date: Mon, 11 Nov 2024 10:34:50 +0100 Subject: [PATCH 13/15] Add `clingo` package (#5184) Adds pypi.org/project/clingo, a powerful Answer Set Programming (ASP) solver. --- docs/project/changelog.md | 1 + packages/clingo/meta.yaml | 21 +++++++++++++++++++++ packages/clingo/test_clingo.py | 25 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 packages/clingo/meta.yaml create mode 100644 packages/clingo/test_clingo.py diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 04a3cb4ea8d..24a00742aa4 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -106,6 +106,7 @@ myst: - Added `tree-sitter-python` 0.23.2 {pr}`5102` - Added `Narwhals` 1.9.4 {pr}`5121` - Added `libzfp` and `zfpy` 1.0.1 {pr}`5172` +- Added `clingo` 5.7.1 {pr}`5184` ## Version 0.26.3 diff --git a/packages/clingo/meta.yaml b/packages/clingo/meta.yaml new file mode 100644 index 00000000000..8adeda01d0e --- /dev/null +++ b/packages/clingo/meta.yaml @@ -0,0 +1,21 @@ +package: + name: clingo + version: 5.7.1 + top-level: + - clingo +source: + url: https://files.pythonhosted.org/packages/3b/b3/c7b464426f70fe100cb4f3e5b45623e3eb82ea50f1dffdc2f820f1418fe0/clingo-5.7.1.tar.gz + sha256: 17400a1894da46b2d4941a4a85012c98fa8b3c67a5c6b4c73dcd8c8facbc059f +about: + home: https://github.com/potassco/clingo + PyPI: https://pypi.org/project/clingo + summary: CFFI-based bindings to the clingo solver. + license: MIT +requirements: + host: + - cffi + run: + - cffi +extra: + recipe-maintainers: + - peter-gy diff --git a/packages/clingo/test_clingo.py b/packages/clingo/test_clingo.py new file mode 100644 index 00000000000..5ce43335304 --- /dev/null +++ b/packages/clingo/test_clingo.py @@ -0,0 +1,25 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["clingo"]) +def test_clingo(selenium): + from clingo.control import Control + from clingo.symbol import Number + + class Context: + def inc(self, x): + return Number(x.number + 1) + + def seq(self, x, y): + return [x, y] + + def on_model(m): + print(m) + + ctl = Control() + program = "\n".join(["p(@inc(10)).", "q(@seq(1,2))."]) + ctl.add("base", [], program) + + ctl.ground([("base", [])], context=Context()) + solution = ctl.solve(on_model=on_model) + assert solution.satisfiable From ea444c11bb841916aa4c2d8c782a34bc52d4f033 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 11 Nov 2024 11:38:38 +0100 Subject: [PATCH 14/15] Fix support for JSPI when there is no type reflection (#5129) The new JSPI does not require wasm type reflection, so it's possible that JS runtimes will have JSPI support but no type reflection, for instance if they are enrolled in the JSPI origin trial or have the JSPI flag on but not the type reflection flag. If type reflection is missing, we fall back to attempting to instantiate a dummy module that imports a function with a given signature. This will raise a LinkerError if the signature is wrong. By trying all four possibilities, we learn the actual signature of the function. --- .circleci/config.yml | 2 +- .devcontainer/devcontainer.json | 2 +- Dockerfile | 38 ++++---- conftest.py | 7 +- .../0001-Public-pymain_run_python.patch | 2 +- ...latform-support-to-ctypes.util.find_.patch | 2 +- ...ocessing.connection-top-level-import.patch | 2 +- ...mscripten-trampolines-work-with-JSPI.patch | 2 +- ...mpoline-for-JSPI-with-no-type-reflec.patch | 34 +++++++ ...ONG_BIT-constant-to-be-always-32bit.patch} | 4 +- ...ZoneInfo-is-imported-without-tzdata.patch} | 4 +- ...xy_GetMethod-to-help-remove-tempora.patch} | 7 +- docs/project/changelog.md | 4 + run_docker | 2 +- .../calculate_wasm_func_nargs_fallback.mjs | 90 +++++++++++++++++++ src/core/stack_switching/stack_switching.mjs | 20 +++++ 16 files changed, 187 insertions(+), 35 deletions(-) create mode 100644 cpython/patches/0005-Add-fallback-trampoline-for-JSPI-with-no-type-reflec.patch rename cpython/patches/{0005-Fix-LONG_BIT-constant-to-be-always-32bit.patch => 0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch} (88%) rename cpython/patches/{0006-Warn-if-ZoneInfo-is-imported-without-tzdata.patch => 0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch} (88%) rename cpython/patches/{0007-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch => 0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch} (89%) create mode 100644 src/core/stack_switching/calculate_wasm_func_nargs_fallback.mjs diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a72acdc310..1a4dc6d8ff7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ defaults: &defaults # Note: when updating the docker image version, # make sure there are no extra old versions lying around. # (e.g. `rg -F --hidden `) - - image: pyodide/pyodide-env:20240928-chrome127-firefox128 + - image: pyodide/pyodide-env:20241028-chrome130-firefox131 environment: - EMSDK_NUM_CORES: 3 EMCC_CORES: 3 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 528cd57938c..9ecfffcad00 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "name": "Docker", // keep in sync with "run_docker" - "image": "pyodide/pyodide-env:20240928-chrome127-firefox128", + "image": "pyodide/pyodide-env:20241028-chrome130-firefox131", "remoteUser": "root", "onCreateCommand": ".devcontainer/onCreate-docker.sh" } diff --git a/Dockerfile b/Dockerfile index f3d47bfe141..ae5eb07b064 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,25 @@ RUN pip3 --no-cache-dir install -r requirements.txt \ && pip3 --no-cache-dir install -r requirements-doc.txt \ && rm -rf requirements.txt requirements-doc.txt +RUN cd / \ + && git clone --recursive https://github.com/WebAssembly/wabt \ + && cd wabt \ + && git submodule update --init \ + && make install-gcc-release-no-tests \ + && cd ~ \ + && rm -rf /wabt + +COPY --from=node-image /usr/local/bin/node /usr/local/bin/ +COPY --from=node-image /usr/local/lib/node_modules /usr/local/lib/node_modules +RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \ + && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx + +RUN npm install -g \ + jsdoc \ + prettier \ + rollup \ + rollup-plugin-terser + # Get Chrome and Firefox (borrowed from https://github.com/SeleniumHQ/docker-selenium) ARG CHROME_VERSION="latest" @@ -99,24 +118,5 @@ RUN if [ $CHROME_VERSION = "latest" ]; \ && echo "Using Chrome version: $(google-chrome --version)" \ && echo "Using Chrome Driver version: $(chromedriver --version)" -COPY --from=node-image /usr/local/bin/node /usr/local/bin/ -COPY --from=node-image /usr/local/lib/node_modules /usr/local/lib/node_modules -RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \ - && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx - -RUN npm install -g \ - jsdoc \ - prettier \ - rollup \ - rollup-plugin-terser - -RUN cd / \ - && git clone --recursive https://github.com/WebAssembly/wabt \ - && cd wabt \ - && git submodule update --init \ - && make install-gcc-release-no-tests \ - && cd ~ \ - && rm -rf /wabt - CMD ["/bin/sh"] WORKDIR /src diff --git a/conftest.py b/conftest.py index 5765ed1328a..f3d88104fdd 100644 --- a/conftest.py +++ b/conftest.py @@ -31,8 +31,13 @@ def set_configs(): "chrome", pytest_pyodide_config.get_flags("chrome") + [ + # Note: in Chrome > 128 (or so?) WebAssemblyExperimentalJSPI no + # longer implies type reflection. If we passed + # `--enable-experimental-webassembly-features` + # here it would enable type reflection. We'd like to make sure that + # everything works in the configuration where JSPI exists but type + # reflection does not, so we don't pass it. "--enable-features=WebAssemblyExperimentalJSPI", - "--enable-experimental-webassembly-features", ], ) diff --git a/cpython/patches/0001-Public-pymain_run_python.patch b/cpython/patches/0001-Public-pymain_run_python.patch index 9fe9a0cc16f..eb89f2eadfb 100644 --- a/cpython/patches/0001-Public-pymain_run_python.patch +++ b/cpython/patches/0001-Public-pymain_run_python.patch @@ -1,7 +1,7 @@ From 3fe0bfb78022df996fb88da1791d1c8410c07610 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sun, 17 Jul 2022 14:40:39 +0100 -Subject: [PATCH 1/7] Public pymain_run_python +Subject: [PATCH 1/8] Public pymain_run_python Discussion here: https://discuss.python.org/t/unstable-api-for-pymain-run-python-run-python-cli-but-dont-finalize-interpreter/44675 diff --git a/cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch b/cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch index 907039c5eae..d66055bdb25 100644 --- a/cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch +++ b/cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch @@ -1,7 +1,7 @@ From a68778f7d8d2fa121ff4cab402f2a862e5cb1afe Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Fri, 2 Dec 2022 11:36:44 +0000 -Subject: [PATCH 2/7] Add emscripten platform support to +Subject: [PATCH 2/8] Add emscripten platform support to ctypes.util.find_library --- diff --git a/cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch b/cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch index 21ba1ee2f03..4c2b54e89c9 100644 --- a/cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch +++ b/cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch @@ -1,7 +1,7 @@ From cfda6ef57f6c49a3bd8a14ebb989c9e2222f10a6 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 19 Dec 2022 09:09:14 -0800 -Subject: [PATCH 3/7] Allow multiprocessing.connection top level import +Subject: [PATCH 3/8] Allow multiprocessing.connection top level import Upstream PR: https://github.com/python/cpython/pull/114808 diff --git a/cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch b/cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch index 91c55942917..85483f094f6 100644 --- a/cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch +++ b/cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch @@ -1,7 +1,7 @@ From a3f1217f7527907c237085c5e900c950c7d2603a Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 28 Jun 2023 10:46:19 -0700 -Subject: [PATCH 4/7] Make Emscripten trampolines work with JSPI +Subject: [PATCH 4/8] Make Emscripten trampolines work with JSPI There is a WIP proposal to enable webassembly stack switching which have been implemented in v8: diff --git a/cpython/patches/0005-Add-fallback-trampoline-for-JSPI-with-no-type-reflec.patch b/cpython/patches/0005-Add-fallback-trampoline-for-JSPI-with-no-type-reflec.patch new file mode 100644 index 00000000000..b5d4ec02925 --- /dev/null +++ b/cpython/patches/0005-Add-fallback-trampoline-for-JSPI-with-no-type-reflec.patch @@ -0,0 +1,34 @@ +From 2119fa65d7af744fd0050214d3452312bbb8d063 Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Tue, 22 Oct 2024 15:16:03 +0200 +Subject: [PATCH 5/8] Add fallback trampoline for JSPI with no type reflection + +--- + Python/emscripten_trampoline.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c +index 8d29393bd87..812cc10167f 100644 +--- a/Python/emscripten_trampoline.c ++++ b/Python/emscripten_trampoline.c +@@ -10,7 +10,7 @@ + * https://github.com/GoogleChromeLabs/wasm-feature-detect/blob/main/src/detectors/type-reflection/index.js + */ + EM_JS(int, _PyEM_detect_type_reflection, (), { +- return "Function" in WebAssembly; ++ return !!Module.PyEM_CountArgs; + }); + + void +@@ -36,7 +36,7 @@ EM_JS(int, _PyEM_CountFuncParams, (PyCFunctionWithKeywords func), { + if (n !== undefined) { + return n; + } +- n = wasmFunctionType(wasmTable.get(func)).parameters.length; ++ n = Module.PyEM_CountArgs(func); + _PyEM_CountFuncParams.cache.set(func, n); + return n; + } +-- +2.34.1 + diff --git a/cpython/patches/0005-Fix-LONG_BIT-constant-to-be-always-32bit.patch b/cpython/patches/0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch similarity index 88% rename from cpython/patches/0005-Fix-LONG_BIT-constant-to-be-always-32bit.patch rename to cpython/patches/0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch index 6f9c1014b05..ec64d1186c8 100644 --- a/cpython/patches/0005-Fix-LONG_BIT-constant-to-be-always-32bit.patch +++ b/cpython/patches/0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch @@ -1,7 +1,7 @@ -From 986bb199865ea5ec81aaec40e02a51d90f339da4 Mon Sep 17 00:00:00 2001 +From 0539585c0d6ed3df3199ce248f5b4d59e89bb6f5 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Fri, 12 Jan 2024 00:52:57 +0900 -Subject: [PATCH 5/7] Fix LONG_BIT constant to be always 32bit +Subject: [PATCH 6/8] Fix LONG_BIT constant to be always 32bit Starting from Emscripten 3.1.50, there is an issue where LONG_BIT is calculated to 64 for some reason. This is very strange because LONG_MAX diff --git a/cpython/patches/0006-Warn-if-ZoneInfo-is-imported-without-tzdata.patch b/cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch similarity index 88% rename from cpython/patches/0006-Warn-if-ZoneInfo-is-imported-without-tzdata.patch rename to cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch index 0204bff9693..48798244982 100644 --- a/cpython/patches/0006-Warn-if-ZoneInfo-is-imported-without-tzdata.patch +++ b/cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch @@ -1,7 +1,7 @@ -From 2643d68535a5ab6c8b4d9105257e53f3a2498b46 Mon Sep 17 00:00:00 2001 +From 0ccdd69344894759ca9de6d74c32b9843f099c90 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 25 Jul 2024 14:28:57 +0200 -Subject: [PATCH 6/7] Warn if ZoneInfo is imported without tzdata +Subject: [PATCH 7/8] Warn if ZoneInfo is imported without tzdata --- Lib/zoneinfo/_common.py | 6 ++++++ diff --git a/cpython/patches/0007-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch b/cpython/patches/0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch similarity index 89% rename from cpython/patches/0007-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch rename to cpython/patches/0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch index 4fc8ac90c11..2e805b73e56 100644 --- a/cpython/patches/0007-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch +++ b/cpython/patches/0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch @@ -1,9 +1,9 @@ -From 0f4f4f02bda0f6ee4efd24e8c2670e8bc80194a8 Mon Sep 17 00:00:00 2001 +From f9b7a02aba3bf050189f3347dce5fcd39bf7dc8d Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 25 Jul 2024 14:41:37 +0200 -Subject: [PATCH 7/7] Add call to `JsProxy_GetMethod` to help remove temporary +Subject: [PATCH 8/8] Add call to `JsProxy_GetMethod` to help remove temporary -_PyObject_GetMethod is a special attribute lookup function that won't call the +`_PyObject_GetMethod` is a special attribute lookup function that won't call the `__get__` descriptor on a method to avoid creating a temporary PyMethodObject. We also want to optimize away a temporary JsProxy in a special way. In order to do this, we patch the behavior of `_PyObject_GetMethod` to use @@ -16,7 +16,6 @@ function. Otherwise, this patch does nothing. See the definition of `JsProxy_GetMethod` in `jsproxy.c` and particularly `JsMethodCallSingleton` for how this is used. - --- Objects/object.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 24a00742aa4..ad62e2a3856 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -61,6 +61,10 @@ myst: file system operations. See [Emscripten docs](https://emscripten.org/docs/api_reference/Filesystem-API.html#FS.trackingDelegate[callback%20name]) for more information. + {pr}`5100` + +- {{ Fix }} JSPI support now works with support for new JSPI and no WebAssembly type reflection. + {pr}`5129` ### Packages diff --git a/run_docker b/run_docker index 33907da1002..0632be6739c 100755 --- a/run_docker +++ b/run_docker @@ -1,7 +1,7 @@ #!/usr/bin/env bash PYODIDE_IMAGE_REPO="pyodide" -PYODIDE_IMAGE_TAG="20240928-chrome127-firefox128" +PYODIDE_IMAGE_TAG="20241028-chrome130-firefox131" DEFAULT_PYODIDE_DOCKER_IMAGE="${PYODIDE_IMAGE_REPO}/pyodide-env:${PYODIDE_IMAGE_TAG}" DEFAULT_PYODIDE_SYSTEM_PORT="none" DOCKER_COMMAND="/bin/bash" diff --git a/src/core/stack_switching/calculate_wasm_func_nargs_fallback.mjs b/src/core/stack_switching/calculate_wasm_func_nargs_fallback.mjs new file mode 100644 index 00000000000..0fef878e5ab --- /dev/null +++ b/src/core/stack_switching/calculate_wasm_func_nargs_fallback.mjs @@ -0,0 +1,90 @@ +// Various function pointer mismatch bugs occur because people declare a +// `METH_NOARGS` function which should take 2 arguments: +// ``` +// my_no_arg_meth(PyObject* module, PyObject* always_null); +// ``` +// but leave off the `always_null` second argument or both arguments. Similar +// errors occur less frequently with `METH_VARARGS | METH_KWDS` functions. When +// the interpreter tries to use a call_indirect to invoke these methods, we hit +// an indirect call signature mismatch fatal error. +// +// Traditionally we used a JS trampoline to deal with this, because calls from +// JS to Wasm don't care if the wrong number of arguments are passed. However, +// these trampolines do not work with JSPI because we cannot stack switch +// through JavaScript frames. +// +// Originally JSPI implied wasm type reflection, so we could ask JS what the +// type of the function was and then select a `call_indirect` with the right +// number of arguments based on this result. +// +// The new JSPI does not imply that wasm type reflection exists, so we need a +// way to handle the case when JSPI is available but wasm type reflection is +// not. What we do here is make a tiny WebAssembly module that attempts to +// import a single function. If the signature of the function matches the +// signature of the import, it will succeed. Otherwise, we will raise a +// LinkError. +// +// We only need to handle functions with four different possible signatures: +// (n i32's) => i32 where n is between 0 and 3. So we try to link at most 4 +// different Wasm modules and find out the signature. + +const checkerModules = [undefined, undefined, undefined, undefined]; + +function makeCheckerModule(n) { + // Make a webassembly module of the form: + // + // (module + // (import "e" "f" (func (param i32 ... i32) (result i32))) + // ) + // + // with n i32's. We'll try to import the function to this module to see if it + // has this signature. If not it rasises a link error. + return new WebAssembly.Module( + // prettier-ignore + new Uint8Array([ + 0, 97, 115, 109, // magic number + 1, 0, 0, 0, // wasm version + // Type section code and byte length + 1, 5 + n, + 1, // One type + // Type is (i32 i32 ... i32) => i32 where there are n i32's + 96, n, ...Array.from({ length: n }, () => 127), + 1, 127, + // Import section code and byte length + 2, 7, + // One import + 1, + // "e" "f" + 1, 101, 1, 102, + // A function of type 0 + 0, 0, + ]), + ); +} + +function getCheckerModule(n) { + if (checkerModules[n] === undefined) { + checkerModules[n] = makeCheckerModule(n); + } + return checkerModules[n]; +} + +export function calculateWasmFuncNargsFallback(functionPtr) { + for (let n = 0; n < 4; n++) { + const mod = getCheckerModule(n); + const imports = { + e: { f: wasmTable.get(functionPtr) }, + }; + try { + new WebAssembly.Instance(mod, imports); + return n; + } catch (e) { + // Should be a LinkError, if not we have a logic error. Raise fatal error + // so it's noisy. + if (!(e instanceof WebAssembly.LinkError)) { + throw e; + } + } + } + return -1; +} diff --git a/src/core/stack_switching/stack_switching.mjs b/src/core/stack_switching/stack_switching.mjs index d1ef8091c9b..9b36be2a222 100644 --- a/src/core/stack_switching/stack_switching.mjs +++ b/src/core/stack_switching/stack_switching.mjs @@ -10,6 +10,7 @@ import { createInvoke, } from "./create_invokes.mjs"; import { initSuspenders } from "./suspenders.mjs"; +import { calculateWasmFuncNargsFallback } from "./calculate_wasm_func_nargs_fallback.mjs"; export { promisingApply, @@ -48,3 +49,22 @@ if (jspiSupported) { Module.wrapException = wrapException; Module.createInvoke = createInvoke; } + +function getPyEmCountArgs() { + if ("Function" in WebAssembly) { + if (WebAssembly.Function.type) { + // Node v20 + return (func) => + WebAssembly.Function.type(wasmTable.get(func)).parameters.length; + } else { + // Node >= 22, v8-based browsers + return (func) => wasmTable.get(func).type().parameters.length; + } + } + if ("Suspending" in WebAssembly) { + return calculateWasmFuncNargsFallback; + } + return undefined; +} + +Module.PyEM_CountArgs = getPyEmCountArgs(); From b09893480ed3b670d7ed0c74c4317d387274c104 Mon Sep 17 00:00:00 2001 From: josephrocca <1167575+josephrocca@users.noreply.github.com> Date: Mon, 11 Nov 2024 20:47:36 +0800 Subject: [PATCH 15/15] Fix sessionStorage-related crash when running in sandboxed iframe (#5186) Fixes this problem when Pyodide is running within a sandboxed iframe: Uncaught SecurityError: Failed to read the 'sessionStorage' property from 'Window': The document is sandboxed and lacks the 'allow-same-origin' flag. --- docs/project/changelog.md | 3 +++ src/js/environments.ts | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index ad62e2a3856..578948a9a2b 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -66,6 +66,9 @@ myst: - {{ Fix }} JSPI support now works with support for new JSPI and no WebAssembly type reflection. {pr}`5129` +- {{ Fix }} Fix sessionStorage-related crash when running in sandboxed iframe. + {pr}`5186` + ### Packages - Upgraded `crc32c` to 2.7.1 {pr}`5169` diff --git a/src/js/environments.ts b/src/js/environments.ts index bc42780b922..f9ce46d9af7 100644 --- a/src/js/environments.ts +++ b/src/js/environments.ts @@ -33,7 +33,7 @@ export const IN_BROWSER_MAIN_THREAD = typeof window === "object" && typeof document === "object" && typeof document.createElement === "function" && - typeof sessionStorage === "object" && + "sessionStorage" in window && typeof importScripts !== "function"; /** @private */