Merge branch 'main' into jspi-no-type-reflection

hoodmane · Nov 11, 2024 · ed3d723 · ed3d723
2 parents 2d5ba57 + 39e62fc
commit ed3d723
Show file tree

Hide file tree

Showing 101 changed files with 1,491 additions and 741 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -29,6 +29,8 @@ jobs:
       - DISABLE_DYLINK: << parameters.disable_dylink >>
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - restore_cache:
           keys:
@@ -132,6 +134,8 @@ jobs:
     resource_class: large
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -190,6 +194,8 @@ jobs:
     resource_class: large
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -219,6 +225,8 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -246,6 +254,8 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -578,11 +588,27 @@ workflows:
             tags:
               only: /.*/
 
+      - build-packages:
+          name: build-static-libraries
+          packages: "tag:static_library"
+          requires:
+            - build-core
+          filters:
+            tags:
+              only: /.*/
+          post-steps:
+            - persist_to_workspace:
+                root: .
+                paths:
+                  - ./packages
+                  - ./dist
+
       - build-packages:
           name: build-libraries
           packages: "tag:library"
           requires:
             - build-core
+            - build-static-libraries
           filters:
             tags:
               only: /.*/

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -27,23 +27,25 @@ jobs:
     needs: get_python_version
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ needs.get_python_version.outputs.PYVERSION }}
       - name: Install requirements
         shell: bash -l {0}
         run: |
           mkdir test-results
-          python3 -m pip install -r requirements.txt -r requirements-deploy.txt
           make pyodide_build
+          python3 -m pip install -r requirements.txt -r requirements-deploy.txt
       - name: Run tests
         shell: bash -l {0}
         run: |
           PYODIDE_ROOT=. pytest \
               --junitxml=test-results/junit.xml \
               --verbose \
               --runtime=host \
-              --cov=pyodide_build --cov=pyodide \
+              --cov=pyodide \
               src packages/_tests tools/
       - uses: codecov/codecov-action@v4
         with:
@@ -68,7 +70,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-
+        with:
+          submodules: recursive
       - name: Cache ccache output
         uses: actions/cache@v4
         with:
@@ -169,6 +172,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Download build artifact
         uses: actions/download-artifact@v4

diff --git a/.github/workflows/update_cross_build_releases.yml b/.github/workflows/update_cross_build_releases.yml
@@ -15,6 +15,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:

diff --git a/.gitignore b/.gitignore
@@ -41,3 +41,4 @@ pytest-pyodide
 tools/symlinks
 xbuildenv/
 .pyodide-xbuildenv*
+DS_Store
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "pyodide-build"]
+	path = pyodide-build
+	url = https://github.com/pyodide/pyodide-build
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -82,7 +82,7 @@ repos:
         name: mypy-tests
         args: [--ignore-missing-imports]
         files: ^(packages/|docs|/conftest.py|src/tests)
-        exclude: (^packages/.*/setup.py|/src|^packages/aiohttp/aiohttp_patch.py$)
+        exclude: (^packages/.*/setup.py|/src|^packages/aiohttp/aiohttp_patch.py$|^packages/zfpy/test_zfpy.py$)
         additional_dependencies: *mypy-deps
 
   - repo: https://github.com/pre-commit/mirrors-prettier

diff --git a/Makefile b/Makefile
@@ -28,9 +28,6 @@ all-but-packages: \
 	dist/python_stdlib.zip \
 	dist/test.html \
 	dist/module_test.html \
-	dist/webworker.js \
-	dist/webworker_dev.js \
-	dist/module_webworker_dev.js \
 
 
 src/core/pyodide_pre.o: src/js/generated/_pyodide.out.js src/core/pre.js src/core/stack_switching/stack_switching.out.js
@@ -228,8 +225,8 @@ $(eval $(call preprocess-js,js2python.js))
 
 .PHONY: pyodide_build
 pyodide_build:
-	@echo "Ensuring required pyodide-build version is installed"
-	./tools/check_and_install_pyodide_build.py "$(PYODIDE_BUILD_COMMIT)" --repo "$(PYODIDE_BUILD_REPO)"
+	@echo "Ensuring pyodide-build is installed"
+	pip install -e ./pyodide-build
 	@which pyodide >/dev/null
 
 
@@ -261,15 +258,6 @@ dist/console.html: src/templates/console.html
 	cp $< $@
 	sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@
 
-dist/webworker.js: src/templates/webworker.js
-	cp $< $@
-
-dist/module_webworker_dev.js: src/templates/module_webworker.js
-	cp $< $@
-
-dist/webworker_dev.js: src/templates/webworker.js
-	cp $< $@
-
 
 # Prepare the dist directory for the release by removing unneeded files
 .PHONY: clean-dist-dir
@@ -278,8 +266,6 @@ clean-dist-dir:
 	rm dist/makesnap.mjs
 	rm dist/snapshot.bin
 	rm dist/module_test.html dist/test.html
-	# TODO: Remove webworker.js too? Would require updating the docs I think.
-	rm dist/module_webworker_dev.js  dist/webworker_dev.js
 
 	# TODO: Source maps aren't useful outside of debug builds I don't think. But
 	# removing them adds "missing sourcemap" warnings to JS console. We should

diff --git a/Makefile.envs b/Makefile.envs
@@ -5,15 +5,6 @@ export PYODIDE_ABI_VERSION ?= 2024_0
 
 export PYTHON_ARCHIVE_SHA256=d01ec6a33bc10009b09c17da95cc2759af5a580a7316b3a446eb4190e13f97b2
 
-# Update the following variables when you want to update the version of the
-# pyodide-build version used in the build process. If you want to make breaking
-# changes in pyodide-build, it is also useful to change the repository URL
-# to your fork to test the changes are working as expected.
-
-# v0.27.3
-export PYODIDE_BUILD_COMMIT=fac0109aa2acf14469320b049d710dd42639bf94
-export PYODIDE_BUILD_REPO=https://github.com/pyodide/pyodide-build
-
 ifdef CPYTHON_DEBUG
 	export CPYTHON_ABI_FLAGS=d
 endif
@@ -151,8 +142,9 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \
 	-s USE_ZLIB \
 	-s USE_BZIP2 \
 	-s FORCE_FILESYSTEM=1 \
-	-s TOTAL_MEMORY=20971520 \
+	-s INITIAL_MEMORY=20971520 \
 	-s ALLOW_MEMORY_GROWTH=1 \
+	-s MAXIMUM_MEMORY=4GB \
 	-s EXPORT_ALL=1 \
 	-s FS_DEBUG=1 \
 	-s STACK_SIZE=5MB \
@@ -239,8 +231,9 @@ ifeq ($(DISABLE_DYLINK), 1)
 		-s USE_ZLIB \
 		-s USE_BZIP2 \
 		-s FORCE_FILESYSTEM=1 \
-		-s TOTAL_MEMORY=20971520 \
+		-s INITIAL_MEMORY=20971520 \
 		-s ALLOW_MEMORY_GROWTH=1 \
+		-s MAXIMUM_MEMORY=4GB \
 		-s STACK_SIZE=5MB \
 		-s ALLOW_TABLE_GROWTH \
 		-s FS_DEBUG=1 \

diff --git a/docs/development/contributing.md b/docs/development/contributing.md
@@ -29,7 +29,7 @@ To contribute code, see the following steps,
 
 3. Clone your fork of Pyodide
    ```
-   git clone https://github.com/<your-username>/pyodide.git
+   git clone --recursive https://github.com/<your-username>/pyodide.git
    ```
    and add the upstream remote,
    ```

diff --git a/docs/development/maintainers.md b/docs/development/maintainers.md
@@ -193,8 +193,27 @@ note to update them independently.
 
 ## Updating pyodide-build
 
-to change the version of pyodide-build, update the PYODIDE_BUILD_REPO and PYODIDE_BUILD_COMMIT
-variables in `Makefile.envs`.
+to change the version of pyodide-build, change the commit of the pyodide-build submodule.
+
+```bash
+cd pyodide-build
+git checkout "<COMMIT HASH>"
+```
+
+to test with the fork of pyodide-build, change the `.gitmodules` file to point to your fork and update the commit hash
+
+```ini
+# .gitmodules
+[submodule "pyodide-build"]
+	path = pyodide-build
+	url = https://github.com/<yourfork>/pyodide-build
+```
+
+```bash
+git submodule sync
+cd pyodide-build
+git checkout "<COMMIT HASH"
+```
 
 ## Upgrading pyodide to a new version of CPython
 

diff --git a/docs/development/new-packages.md b/docs/development/new-packages.md
@@ -60,7 +60,7 @@ folder](https://github.com/pyodide/pyodide/tree/main/packages).
 First clone the Pyodide git repository:
 
 ```bash
-git clone https://github.com/pyodide/pyodide
+git clone --recursive https://github.com/pyodide/pyodide
 cd pyodide
 ```
 
@@ -196,7 +196,7 @@ from pytest_pyodide import run_in_pyodide
 @run_in_pyodide(packages=["<package-name>-tests", "pytest"])
 def test_mytestname(selenium):
   import pytest
-  pytest.main(["--pyargs", "<package-name>", "-k", "some_filter", ...])
+  assert pytest.main(["--pyargs", "<package-name>", "-k", "some_filter", ...]) == 0
 ```
 
 you can put whatever command line arguments you would pass to `pytest` as

diff --git a/docs/project/changelog.md b/docs/project/changelog.md
@@ -36,24 +36,10 @@ myst:
 - {{ Enhancement }} Unvendored stdlibs are now packaged in a wheel format
   {pr}`4902`
 
-- {{ Enhancement }} Updated stack switching support to handle new JSPI.
-  {pr}`4982`
-
-- {{ Performance }} Attribute lookup on a `JsProxy` is now about 40% faster.
-  {pr}`4961`
-
-- {{ Performance }} Method calls on a `JsProxy` are now much faster. If the
-  method has no arguments and no return value, it is about 80% faster. The
-  speedup for methods with arguments is less drastic but still quite a lot.
-  {pr}`4961`
-
 - {{ Breaking }} Prebuilt third-party libraries like `openblas`, `openssl`, `zlib` are
   not included in the cross-build env anymore.
   {pr}`4995`
 
-- {{ Fix }} `pyimport("a.b")` won't fail when `a` is removed by `del sys.modules["a"]`
-  {pr}`4993`
-
 - {{ Enhancement }} Added `JsProxy.as_py_json` method to adapt from JavaScript
   JSON (Arrays and Objects). to Python JSON (lists and dicts).
   {pr}`4666`
@@ -66,19 +52,27 @@ myst:
   for namespace packages.
   {pr}`5039`
 
+- {{ Breaking }} Removed `webworker.js` from the distribution files. It was a
+  pretty small file, if someone is using it for something they can just copy it into
+  their own code.
+  {pr}`5114`
+
 - {{ Enhancement }} Enabled `pyodide.FS.trackingDelegate` which can be used to track
   file system operations. See
   [Emscripten docs](https://emscripten.org/docs/api_reference/Filesystem-API.html#FS.trackingDelegate[callback%20name])
   for more information.
 
-- {{ Fix }} It now works to convert a 0d Python buffer to JavaScript.
-  {pr}`5092`
-
-- {{ Fix }} It now works to convert buffers of 64 bit signed or unsigned integers to JavaScript.
-  {pr}`5092`
-
 ### Packages
 
+- Upgraded `crc32c` to 2.7.1 {pr}`5169`
+- Upgraded `rebound` to 4.4.3 {pr}`5163`
+- Upgraded `reboundx` to 4.3.0 {pr}`5163`
+- Upgraded `msprime` to 1.3.3 {pr}`5159`
+- Upgraded `tskit` to 0.6.0 {pr}`5157`
+- Upgraded `pydantic_core` to 2.25.1 {pr}`5151`
+- Upgraded `pydantic` to 2.9.2 {pr}`5151`
+- Upgraded `msgpack` to 1.1.0 {pr}`5144`
+- Upgraded `protobuf` to 5.28.3 {pr}`5136`
 - Upgraded `scikit-learn` to 1.5.2 {pr}`4823`, {pr}`5016`, {pr}`5072`
 - Upgraded `libcst` to 1.4.0 {pr}`4856`
 - Upgraded `lakers` to 0.3.3 {pr}`4885`
@@ -96,12 +90,47 @@ myst:
 - Upgraded `duckdb` to 1.1.0 {pr}`5078`
 - Upgraded `sympy` to 1.13.3 {pr}`5098`
 - Upgraded `tree-sitter` to 0.23.1 {pr}`5110`
+- Upgraded `altair` to 5.4.1 {pr}`5124`
+- Upgraded `PyYAML` to 6.0.2 {pr}`5137`
+- Upgraded `duckdb` to 1.1.2 {pr}`5142`
+- Added `soxr` 0.5.0.post1 {pr}`5150`
+- Added `tiktoken` v0.8.0 in {pr}`5147`
 - Added `casadi` 3.6.6 {pr}`4936`, {pr}`5057`
 - Added `pyarrow` 17.0.0 {pr}`4950`
 - Added `rasterio` 1.13.10, `affine` 2.4.0 {pr}`4983`
 - Added `iminuit` 2.29.1 {pr}`4767`, {pr}`5072`
 - Added `arro3-core`, `arro3-io`, and `arro3-compute` 0.3.0, 0.4.0, 0.4.1 {pr}`5020`, {pr}`5095`, {pr}`5104`
 - Added `tree-sitter` 0.23.0 {pr}`5099`
+- Added `tree-sitter-go` 0.23.1 {pr}`5102`
+- Added `tree-sitter-java` 0.23.2 {pr}`5102`
+- Added `tree-sitter-python` 0.23.2 {pr}`5102`
+- Added `Narwhals` 1.9.4 {pr}`5121`
+- Added `libzfp` and `zfpy` 1.0.1 {pr}`5172`
+- Added `clingo` 5.7.1 {pr}`5184`
+
+## Version 0.26.3
+
+_October 19, 2024_
+
+- {{ Performance }} Attribute lookup on a `JsProxy` is now about 40% faster.
+  {pr}`4961`
+
+- {{ Performance }} Method calls on a `JsProxy` are now much faster. If the
+  method has no arguments and no return value, it is about 80% faster. The
+  speedup for methods with arguments is less drastic but still quite a lot.
+  {pr}`4963`
+
+- {{ Enhancement }} Updated stack switching support to handle new JSPI.
+  {pr}`4982`
+
+- {{ Fix }} `pyimport("a.b")` won't fail when `a` is removed by `del sys.modules["a"]`
+  {pr}`4993`
+
+- {{ Fix }} It now works to convert a 0d Python buffer to JavaScript.
+  {pr}`5092`
+
+- {{ Fix }} It now works to convert buffers of 64 bit signed or unsigned integers to JavaScript.
+  {pr}`5092`
 
 ## Version 0.26.2
 

diff --git a/docs/usage/downloading-and-deploying.md b/docs/usage/downloading-and-deploying.md
@@ -87,4 +87,3 @@ deployments](https://emscripten.org/docs/compiling/Deploying-Pages.html).
 | pyodide.mjs.map | Source maps to improve tracebacks. Not really that useful to people outside of the project, probably should be only included in debug builds.                                                        |
 | \*-tests.tar    | Unvendored tests from wheels. If a wheel includes a test folder, we take them out and put them here.                                                                                                 |
 | console.html    | The Pyodide repl.                                                                                                                                                                                    |
-| webworker.js    | Mentioned in the docs. Developers should probably implement their own Web Worker.                                                                                                                    |