Merge branch 'main' into importhook-key-error

.circleci/config.yml

@@ -6,7 +6,7 @@ defaults: &defaults
     # Note: when updating the docker image version,
     #       make sure there are no extra old versions lying around.
     #       (e.g. `rg -F --hidden <old_tag>`)
-    - image: pyodide/pyodide-env:20240928-chrome127-firefox128
+    - image: pyodide/pyodide-env:20241028-chrome130-firefox131
   environment:
     - EMSDK_NUM_CORES: 3
       EMCC_CORES: 3
@@ -29,6 +29,8 @@ jobs:
       - DISABLE_DYLINK: << parameters.disable_dylink >>
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - restore_cache:
           keys:
@@ -132,6 +134,8 @@ jobs:
     resource_class: large
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -190,6 +194,8 @@ jobs:
     resource_class: large
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -219,6 +225,8 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -246,6 +254,8 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 {
   "name": "Docker",
   // keep in sync with "run_docker"
-  "image": "pyodide/pyodide-env:20240928-chrome127-firefox128",
+  "image": "pyodide/pyodide-env:20241028-chrome130-firefox131",
   "remoteUser": "root",
   "onCreateCommand": ".devcontainer/onCreate-docker.sh"
 }

.github/workflows/main.yml

@@ -27,23 +27,25 @@ jobs:
     needs: get_python_version
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ needs.get_python_version.outputs.PYVERSION }}
       - name: Install requirements
         shell: bash -l {0}
         run: |
           mkdir test-results
-          python3 -m pip install -r requirements.txt -r requirements-deploy.txt
           make pyodide_build
+          python3 -m pip install -r requirements.txt -r requirements-deploy.txt
       - name: Run tests
         shell: bash -l {0}
         run: |
           PYODIDE_ROOT=. pytest \
               --junitxml=test-results/junit.xml \
               --verbose \
               --runtime=host \
-              --cov=pyodide_build --cov=pyodide \
+              --cov=pyodide \
               src packages/_tests tools/
       - uses: codecov/codecov-action@v4
         with:
@@ -68,7 +70,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-
+        with:
+          submodules: recursive
       - name: Cache ccache output
         uses: actions/cache@v4
         with:
@@ -169,6 +172,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Download build artifact
         uses: actions/download-artifact@v4

.github/workflows/update_cross_build_releases.yml

@@ -15,6 +15,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "pyodide-build"]
+	path = pyodide-build
+	url = https://github.com/pyodide/pyodide-build

.pre-commit-config.yaml

@@ -82,7 +82,7 @@ repos:
         name: mypy-tests
         args: [--ignore-missing-imports]
         files: ^(packages/|docs|/conftest.py|src/tests)
-        exclude: (^packages/.*/setup.py|/src|^packages/aiohttp/aiohttp_patch.py$)
+        exclude: (^packages/.*/setup.py|/src|^packages/aiohttp/aiohttp_patch.py$|^packages/zfpy/test_zfpy.py$)
         additional_dependencies: *mypy-deps
 
   - repo: https://github.com/pre-commit/mirrors-prettier

Dockerfile

@@ -34,6 +34,25 @@ RUN pip3 --no-cache-dir install -r requirements.txt \
     && pip3 --no-cache-dir install -r requirements-doc.txt \
     && rm -rf requirements.txt requirements-doc.txt
 
+RUN cd / \
+    && git clone --recursive https://github.com/WebAssembly/wabt \
+    && cd wabt \
+    && git submodule update --init \
+    && make install-gcc-release-no-tests \
+    && cd ~  \
+    && rm -rf /wabt
+
+COPY --from=node-image /usr/local/bin/node /usr/local/bin/
+COPY --from=node-image /usr/local/lib/node_modules /usr/local/lib/node_modules
+RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
+    && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx
+
+RUN npm install -g \
+  jsdoc \
+  prettier \
+  rollup \
+  rollup-plugin-terser
+
 # Get Chrome and Firefox (borrowed from https://github.com/SeleniumHQ/docker-selenium)
 
 ARG CHROME_VERSION="latest"
@@ -99,24 +118,5 @@ RUN if [ $CHROME_VERSION = "latest" ]; \
   && echo "Using Chrome version: $(google-chrome --version)" \
   && echo "Using Chrome Driver version: $(chromedriver --version)"
 
-COPY --from=node-image /usr/local/bin/node /usr/local/bin/
-COPY --from=node-image /usr/local/lib/node_modules /usr/local/lib/node_modules
-RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
-    && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx
-
-RUN npm install -g \
-  jsdoc \
-  prettier \
-  rollup \
-  rollup-plugin-terser
-
-RUN cd / \
-    && git clone --recursive https://github.com/WebAssembly/wabt \
-    && cd wabt \
-    && git submodule update --init \
-    && make install-gcc-release-no-tests \
-    && cd ~  \
-    && rm -rf /wabt
-
 CMD ["/bin/sh"]
 WORKDIR /src

Makefile

@@ -225,8 +225,8 @@ $(eval $(call preprocess-js,js2python.js))
 
 .PHONY: pyodide_build
 pyodide_build:
-	@echo "Ensuring required pyodide-build version is installed"
-	./tools/check_and_install_pyodide_build.py "$(PYODIDE_BUILD_COMMIT)" --repo "$(PYODIDE_BUILD_REPO)"
+	@echo "Ensuring pyodide-build is installed"
+	pip install -e ./pyodide-build
 	@which pyodide >/dev/null
 
 

Makefile.envs

@@ -5,15 +5,6 @@ export PYODIDE_ABI_VERSION ?= 2024_0
 
 export PYTHON_ARCHIVE_SHA256=d01ec6a33bc10009b09c17da95cc2759af5a580a7316b3a446eb4190e13f97b2
 
-# Update the following variables when you want to update the version of the
-# pyodide-build version used in the build process. If you want to make breaking
-# changes in pyodide-build, it is also useful to change the repository URL
-# to your fork to test the changes are working as expected.
-
-# v0.27.3
-export PYODIDE_BUILD_COMMIT=fac0109aa2acf14469320b049d710dd42639bf94
-export PYODIDE_BUILD_REPO=https://github.com/pyodide/pyodide-build
-
 ifdef CPYTHON_DEBUG
 	export CPYTHON_ABI_FLAGS=d
 endif
@@ -151,8 +142,9 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \
 	-s USE_ZLIB \
 	-s USE_BZIP2 \
 	-s FORCE_FILESYSTEM=1 \
-	-s TOTAL_MEMORY=20971520 \
+	-s INITIAL_MEMORY=20971520 \
 	-s ALLOW_MEMORY_GROWTH=1 \
+	-s MAXIMUM_MEMORY=4GB \
 	-s EXPORT_ALL=1 \
 	-s FS_DEBUG=1 \
 	-s STACK_SIZE=5MB \
@@ -239,8 +231,9 @@ ifeq ($(DISABLE_DYLINK), 1)
 		-s USE_ZLIB \
 		-s USE_BZIP2 \
 		-s FORCE_FILESYSTEM=1 \
-		-s TOTAL_MEMORY=20971520 \
+		-s INITIAL_MEMORY=20971520 \
 		-s ALLOW_MEMORY_GROWTH=1 \
+		-s MAXIMUM_MEMORY=4GB \
 		-s STACK_SIZE=5MB \
 		-s ALLOW_TABLE_GROWTH \
 		-s FS_DEBUG=1 \

conftest.py

@@ -31,8 +31,13 @@ def set_configs():
         "chrome",
         pytest_pyodide_config.get_flags("chrome")
         + [
+            # Note: in Chrome > 128 (or so?) WebAssemblyExperimentalJSPI no
+            # longer implies type reflection. If we passed
+            # `--enable-experimental-webassembly-features`
+            # here it would enable type reflection. We'd like to make sure that
+            # everything works in the configuration where JSPI exists but type
+            # reflection does not, so we don't pass it.
             "--enable-features=WebAssemblyExperimentalJSPI",
-            "--enable-experimental-webassembly-features",
         ],
     )
 

cpython/patches/0001-Public-pymain_run_python.patch

@@ -1,7 +1,7 @@
 From 3fe0bfb78022df996fb88da1791d1c8410c07610 Mon Sep 17 00:00:00 2001
 From: Hood Chatham <[email protected]>
 Date: Sun, 17 Jul 2022 14:40:39 +0100
-Subject: [PATCH 1/7] Public pymain_run_python
+Subject: [PATCH 1/8] Public pymain_run_python
 
 Discussion here:
 https://discuss.python.org/t/unstable-api-for-pymain-run-python-run-python-cli-but-dont-finalize-interpreter/44675

cpython/patches/0002-Add-emscripten-platform-support-to-ctypes.util.find_.patch

@@ -1,7 +1,7 @@
 From a68778f7d8d2fa121ff4cab402f2a862e5cb1afe Mon Sep 17 00:00:00 2001
 From: ryanking13 <[email protected]>
 Date: Fri, 2 Dec 2022 11:36:44 +0000
-Subject: [PATCH 2/7] Add emscripten platform support to
+Subject: [PATCH 2/8] Add emscripten platform support to
  ctypes.util.find_library
 
 ---

cpython/patches/0003-Allow-multiprocessing.connection-top-level-import.patch

@@ -1,7 +1,7 @@
 From cfda6ef57f6c49a3bd8a14ebb989c9e2222f10a6 Mon Sep 17 00:00:00 2001
 From: Hood Chatham <[email protected]>
 Date: Mon, 19 Dec 2022 09:09:14 -0800
-Subject: [PATCH 3/7] Allow multiprocessing.connection top level import
+Subject: [PATCH 3/8] Allow multiprocessing.connection top level import
 
 Upstream PR:
 https://github.com/python/cpython/pull/114808

cpython/patches/0004-Make-Emscripten-trampolines-work-with-JSPI.patch

@@ -1,7 +1,7 @@
 From a3f1217f7527907c237085c5e900c950c7d2603a Mon Sep 17 00:00:00 2001
 From: Hood Chatham <[email protected]>
 Date: Wed, 28 Jun 2023 10:46:19 -0700
-Subject: [PATCH 4/7] Make Emscripten trampolines work with JSPI
+Subject: [PATCH 4/8] Make Emscripten trampolines work with JSPI
 
 There is a WIP proposal to enable webassembly stack switching which have been
 implemented in v8:

cpython/patches/0005-Add-fallback-trampoline-for-JSPI-with-no-type-reflec.patch

@@ -0,0 +1,34 @@
+From 2119fa65d7af744fd0050214d3452312bbb8d063 Mon Sep 17 00:00:00 2001
+From: Hood Chatham <[email protected]>
+Date: Tue, 22 Oct 2024 15:16:03 +0200
+Subject: [PATCH 5/8] Add fallback trampoline for JSPI with no type reflection
+
+---
+ Python/emscripten_trampoline.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c
+index 8d29393bd87..812cc10167f 100644
+--- a/Python/emscripten_trampoline.c
++++ b/Python/emscripten_trampoline.c
+@@ -10,7 +10,7 @@
+  * https://github.com/GoogleChromeLabs/wasm-feature-detect/blob/main/src/detectors/type-reflection/index.js
+  */
+ EM_JS(int, _PyEM_detect_type_reflection, (), {
+-  return "Function" in WebAssembly;
++    return !!Module.PyEM_CountArgs;
+ });
+
+ void
+@@ -36,7 +36,7 @@ EM_JS(int, _PyEM_CountFuncParams, (PyCFunctionWithKeywords func), {
+   if (n !== undefined) {
+     return n;
+   }
+-  n = wasmFunctionType(wasmTable.get(func)).parameters.length;
++  n = Module.PyEM_CountArgs(func);
+   _PyEM_CountFuncParams.cache.set(func, n);
+   return n;
+ }
+-- 
+2.34.1
+

cpython/patches/0005-Fix-LONG_BIT-constant-to-be-always-32bit.patch → cpython/patches/0006-Fix-LONG_BIT-constant-to-be-always-32bit.patch

@@ -1,7 +1,7 @@
-From 986bb199865ea5ec81aaec40e02a51d90f339da4 Mon Sep 17 00:00:00 2001
+From 0539585c0d6ed3df3199ce248f5b4d59e89bb6f5 Mon Sep 17 00:00:00 2001
 From: ryanking13 <[email protected]>
 Date: Fri, 12 Jan 2024 00:52:57 +0900
-Subject: [PATCH 5/7] Fix LONG_BIT constant to be always 32bit
+Subject: [PATCH 6/8] Fix LONG_BIT constant to be always 32bit
 
 Starting from Emscripten 3.1.50, there is an issue where LONG_BIT is
 calculated to 64 for some reason. This is very strange because LONG_MAX

cpython/patches/0006-Warn-if-ZoneInfo-is-imported-without-tzdata.patch → cpython/patches/0007-Warn-if-ZoneInfo-is-imported-without-tzdata.patch

@@ -1,7 +1,7 @@
-From 2643d68535a5ab6c8b4d9105257e53f3a2498b46 Mon Sep 17 00:00:00 2001
+From 0ccdd69344894759ca9de6d74c32b9843f099c90 Mon Sep 17 00:00:00 2001
 From: Hood Chatham <[email protected]>
 Date: Thu, 25 Jul 2024 14:28:57 +0200
-Subject: [PATCH 6/7] Warn if ZoneInfo is imported without tzdata
+Subject: [PATCH 7/8] Warn if ZoneInfo is imported without tzdata
 
 ---
  Lib/zoneinfo/_common.py | 6 ++++++

cpython/patches/0007-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch → cpython/patches/0008-Add-call-to-JsProxy_GetMethod-to-help-remove-tempora.patch

@@ -1,9 +1,9 @@
-From 0f4f4f02bda0f6ee4efd24e8c2670e8bc80194a8 Mon Sep 17 00:00:00 2001
+From f9b7a02aba3bf050189f3347dce5fcd39bf7dc8d Mon Sep 17 00:00:00 2001
 From: Hood Chatham <[email protected]>
 Date: Thu, 25 Jul 2024 14:41:37 +0200
-Subject: [PATCH 7/7] Add call to `JsProxy_GetMethod` to help remove temporary
+Subject: [PATCH 8/8] Add call to `JsProxy_GetMethod` to help remove temporary
 
-_PyObject_GetMethod is a special attribute lookup function that won't call the
+`_PyObject_GetMethod` is a special attribute lookup function that won't call the
 `__get__` descriptor on a method to avoid creating a temporary PyMethodObject.
 We also want to optimize away a temporary JsProxy in a special way. In order to
 do this, we patch the behavior of `_PyObject_GetMethod` to use
@@ -16,7 +16,6 @@ function. Otherwise, this patch does nothing.
 
 See the definition of `JsProxy_GetMethod` in `jsproxy.c` and particularly
 `JsMethodCallSingleton` for how this is used.
-
 ---
  Objects/object.c | 17 +++++++++++++++++
  1 file changed, 17 insertions(+)

docs/development/contributing.md

@@ -29,7 +29,7 @@ To contribute code, see the following steps,
 
 3. Clone your fork of Pyodide
    ```
-   git clone https://github.com/<your-username>/pyodide.git
+   git clone --recursive https://github.com/<your-username>/pyodide.git
    ```
    and add the upstream remote,
    ```

docs/development/maintainers.md

@@ -193,8 +193,27 @@ note to update them independently.
 
 ## Updating pyodide-build
 
-to change the version of pyodide-build, update the PYODIDE_BUILD_REPO and PYODIDE_BUILD_COMMIT
-variables in `Makefile.envs`.
+to change the version of pyodide-build, change the commit of the pyodide-build submodule.
+
+```bash
+cd pyodide-build
+git checkout "<COMMIT HASH>"
+```
+
+to test with the fork of pyodide-build, change the `.gitmodules` file to point to your fork and update the commit hash
+
+```ini
+# .gitmodules
+[submodule "pyodide-build"]
+	path = pyodide-build
+	url = https://github.com/<yourfork>/pyodide-build
+```
+
+```bash
+git submodule sync
+cd pyodide-build
+git checkout "<COMMIT HASH"
+```
 
 ## Upgrading pyodide to a new version of CPython