Merge branch 'main' into fix-setjmp-matplotlib

.github/workflows/deploy_release.yml

@@ -9,7 +9,7 @@ jobs:
     environment: PyPi
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -40,7 +40,7 @@ jobs:
     environment: PyPi
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v4

.github/workflows/docker_image.yml

@@ -23,9 +23,9 @@ jobs:
     environment: Docker
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Log into Docker Hub registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -60,9 +60,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Log into registry ${{ env.GHCR_REGISTRY }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.GHCR_REGISTRY }}
           username: ${{ github.repository_owner }}

.github/workflows/main.yml

@@ -14,7 +14,7 @@ jobs:
     outputs:
       PYVERSION: ${{ steps.get_python_version.outputs.PYVERSION }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Read the Python version from Makefile.envs
         id: get_python_version
         run: |
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: get_python_version
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Python ${{ needs.get_python_version.outputs.PYVERSION }}
         uses: actions/setup-python@v4
         with:
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: get_python_version
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Python ${{ needs.get_python_version.outputs.PYVERSION }}
         uses: actions/setup-python@v4
         with:
@@ -85,7 +85,7 @@ jobs:
       CCACHE_DIR: /tmp/ccache
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Cache ccache output
         uses: actions/cache@v3
@@ -183,7 +183,7 @@ jobs:
         browser: [chrome]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Download build artifact
         uses: actions/download-artifact@v3
@@ -251,7 +251,7 @@ jobs:
         os: [ubuntu-latest]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Download build artifact
         uses: actions/download-artifact@v3

.prettierignore

@@ -9,5 +9,5 @@ cpython
 .vscode
 .pytest_cache
 .clang-format
-packages/libf2c/make.inc
+packages/libf2c/extras/make.inc
 src/js/generated_struct_info32.json

docs/development/building-and-testing-packages.md

@@ -152,13 +152,13 @@ runs-on: ubuntu-22.04
   steps:
   - uses: actions/checkout@v3
   - uses: actions/setup-python@v4
-     with:
+    with:
        python-version: 3.11.2
   - run: |
       pip install pyodide-build>=0.23.0
       echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV
   - uses: mymindstorm/setup-emsdk@v12
-     with:
+    with:
        version: ${{ env.EMSCRIPTEN_VERSION }}
   - run: pyodide build
 ```

docs/project/changelog.md

@@ -19,6 +19,21 @@ myst:
 - {{ Fix }} Fixed an Emscripten bug that broke some matplotlib functionality.
   {pr}`4163`
 
+- {{ Fix }} Fixed `LONG_BIT definition appears wrong for platform` error happened in out-of-tree build.
+  {pr}`4136`
+
+### Load time & size optimizations
+
+- {{ Performance }} Do not use `importlib.metadata` when identifying installed packages,
+  which reduces the time to load Pyodide.
+  {pr}`4147`
+
+### Packages
+
+- Upgraded scipy to 1.11.2 {pr}`4156`
+
+- Upgraded scikit-learn to 1.3.1 {pr}`4161`
+
 ## Version 0.24.0
 
 _September 13, 2023_

packages/coolprop/meta.yaml

@@ -10,9 +10,9 @@ source:
   extract_dir: source
 
   extras:
-    - - setup.py
+    - - extras/setup.py
       - ./setup.py
-    - - pyproject.toml
+    - - extras/pyproject.toml
       - ./pyproject.toml
 
 requirements:

packages/libf2c/meta.yaml

@@ -24,7 +24,7 @@ source:
     - patches/0006-adjust-ld-ar-ranlib.patch
 
   extras:
-    - [make.inc, make.inc]
+    - [extras/make.inc, make.inc]
 
 build:
   type: static_library

packages/matplotlib/meta.yaml

@@ -17,7 +17,7 @@ source:
     - patches/fix-threading.patch
 
   extras:
-    - - src/setup.cfg
+    - - extras/setup.cfg
       - ./mplsetup.cfg
 
 build:
@@ -40,8 +40,8 @@ build:
     rm -rf matplotlib/backends/qt_editor
     rm -rf matplotlib/backends/web_backend
     rm -rf sphinxext
-    cp $PKGDIR/src/fontlist.json matplotlib
-    cp $PKGDIR/src/Humor-Sans-1.0.ttf matplotlib/mpl-data/fonts/ttf/Humor-Sans.ttf
+    cp $PKGDIR/extras/fontlist.json matplotlib
+    cp $PKGDIR/extras/Humor-Sans-1.0.ttf matplotlib/mpl-data/fonts/ttf/Humor-Sans.ttf
     mkdir -p $PKGDIR/../../dist/fonts
     cp matplotlib/mpl-data/fonts/ttf/* $PKGDIR/../../dist/fonts/
 

packages/nlopt/meta.yaml

@@ -9,7 +9,7 @@ source:
   sha256: b881cc2a5face5139f1c5a30caf26b7d3cb43d69d5e423c9d78392f99844499f
 
   extras:
-    - - src/setup.py
+    - - extras/setup.py
       - ./setup.py
 
 requirements:

packages/opencv-python/meta.yaml

@@ -12,9 +12,9 @@ source:
   url: https://files.pythonhosted.org/packages/32/72/03747a6820bc970aeb0b89e653d1084068ac1ed606a83d8b5ac6fc237c14/opencv-python-4.8.0.76.tar.gz
   sha256: 56d84c43ce800938b9b1ec74b33942b2edbcef3f70c2754eb9bfe5dff1ee3ace
   extras:
-    - [cmake/OpenCVFindLibsGrfmt.cmake, opencv/cmake/OpenCVFindLibsGrfmt.cmake]
+    - [extras/OpenCVFindLibsGrfmt.cmake, opencv/cmake/OpenCVFindLibsGrfmt.cmake]
     - [
-        cmake/detect_ffmpeg.cmake,
+        extras/detect_ffmpeg.cmake,
         opencv/modules/videoio/cmake/detect_ffmpeg.cmake,
       ]
   patches:
@@ -52,4 +52,4 @@ build:
     export NUMPY_INCLUDE_DIR="$HOSTINSTALLDIR/lib/python$PYMAJOR.$PYMINOR/site-packages/numpy/core/include/"
     export FFMPEG_ROOT="$WASM_LIBRARY_DIR"
 
-    source $PKGDIR/cmake/build_args.sh
+    source $PKGDIR/extras/build_args.sh

packages/scikit-learn/meta.yaml

@@ -1,13 +1,13 @@
 package:
   name: scikit-learn
-  version: 1.3.0
+  version: 1.3.1
   tag:
     - min-scipy-stack
   top-level:
     - sklearn
 source:
-  url: https://files.pythonhosted.org/packages/72/cd/4761675df1b3dd93072c89697278f3ed3dc004a60c034cd2603c43ff64b5/scikit-learn-1.3.0.tar.gz
-  sha256: 8be549886f5eda46436b6e555b0e4873b4f10aa21c07df45c4bc1735afbccd7a
+  url: https://files.pythonhosted.org/packages/4b/49/4b3e90399f49e875a1a6a0e72bb99d7e8fe808fcfe0a6a12b43a77e7d64d/scikit-learn-1.3.1.tar.gz
+  sha256: 1a231cced3ee3fa04756b4a7ab532dc9417acd581a330adff5f2c01ac2831fcf
 
 build:
   cflags: -Wno-implicit-function-declaration

packages/scipy/meta.yaml

@@ -1,6 +1,6 @@
 package:
   name: scipy
-  version: 1.11.1
+  version: 1.11.2
   tag:
     - min-scipy-stack
   top-level:
@@ -17,8 +17,8 @@ package:
 # subroutine. Try deleting it.
 
 source:
-  url: https://files.pythonhosted.org/packages/a6/98/fceb84466a74b8fe74ce2dcc3a0a89cb7b4a689d4775e0fb4c95f335ef6a/scipy-1.11.1.tar.gz
-  sha256: fb5b492fa035334fd249f0973cc79ecad8b09c604b42a127a677b45a9a3d4289
+  url: https://files.pythonhosted.org/packages/9c/ef/87a5565907645998d7c62e76b84b0ca9f0b7c25cd433f5617a968051cec3/scipy-1.11.2.tar.gz
+  sha256: b29318a5e39bd200ca4381d80b065cdf3076c7d7281c5e36569e99273867f61d
 
   patches:
     - patches/0001-Fix-dstevr-in-special-lapack_defs.h.patch
@@ -53,6 +53,10 @@ build:
     export NPY_BLAS_LIBS="-I$WASM_LIBRARY_DIR/include $WASM_LIBRARY_DIR/lib/libopenblas.so"
     export NPY_LAPACK_LIBS="-I$WASM_LIBRARY_DIR/include $WASM_LIBRARY_DIR/lib/libopenblas.so"
 
+    # setup.py has been renamed to _setup.py in scipy 1.11.2 to discourage its
+    # usage but we still need it until we manage to build with meson
+    mv _setup.py setup.py
+
     sed -i 's/recursive //g' scipy/integrate/quadpack/* scipy/interpolate/fitpack/*
     sed -i 's/void DQA/int DQA/g' scipy/integrate/__quadpack.h
 

pyodide-build/pyodide_build/out_of_tree/build.py

@@ -15,6 +15,9 @@ def run(
     cxxflags += f" {os.environ.get('CXXFLAGS', '')}"
     ldflags = build_env.get_build_flag("SIDE_MODULE_LDFLAGS")
     ldflags += f" {os.environ.get('LDFLAGS', '')}"
+    target_install_dir = os.environ.get(
+        "TARGETINSTALLDIR", build_env.get_build_flag("TARGETINSTALLDIR")
+    )
     env = os.environ.copy()
     env.update(build_env.get_build_environment_vars())
 
@@ -24,7 +27,7 @@ def run(
         cflags=cflags,
         cxxflags=cxxflags,
         ldflags=ldflags,
-        target_install_dir="",
+        target_install_dir=target_install_dir,
         exports=exports,
     )
 

pyodide-build/pyodide_build/pywasmcross.py

@@ -226,14 +226,19 @@ def replay_genargs_handle_dashI(arg: str, target_install_dir: str) -> str | None
         The new argument, or None to delete the argument.
     """
     assert arg.startswith("-I")
-    if (
-        str(Path(arg[2:]).resolve()).startswith(sys.prefix + "/include/python")
-        and "site-packages" not in arg
-    ):
-        return arg.replace("-I" + sys.prefix, "-I" + target_install_dir)
+
     # Don't include any system directories
     if arg[2:].startswith("/usr"):
         return None
+
+    # Replace local Python include paths with the cross compiled ones
+    include_path = str(Path(arg[2:]).resolve())
+    if include_path.startswith(sys.prefix + "/include/python"):
+        return arg.replace("-I" + sys.prefix, "-I" + target_install_dir)
+
+    if include_path.startswith(sys.base_prefix + "/include/python"):
+        return arg.replace("-I" + sys.base_prefix, "-I" + target_install_dir)
+
     return arg
 
 

pyodide-build/pyodide_build/tests/test_pywasmcross.py

@@ -10,6 +10,7 @@
     get_library_output,
     handle_command_generate_args,
     replay_f2c,
+    replay_genargs_handle_dashI,
 )
 
 
@@ -145,6 +146,30 @@ def test_handle_command_ldflags(build_args):
     )
 
 
+def test_replay_genargs_handle_dashI(monkeypatch):
+    import sys
+
+    mock_prefix = "/mock_prefix"
+    mock_base_prefix = "/mock_base_prefix"
+    monkeypatch.setattr(sys, "prefix", mock_prefix)
+    monkeypatch.setattr(sys, "base_prefix", mock_base_prefix)
+
+    target_dir = "/target"
+    target_cpython_include = "/target/include/python3.11"
+
+    assert replay_genargs_handle_dashI("-I/usr/include", target_dir) is None
+    assert (
+        replay_genargs_handle_dashI(f"-I{mock_prefix}/include/python3.11", target_dir)
+        == f"-I{target_cpython_include}"
+    )
+    assert (
+        replay_genargs_handle_dashI(
+            f"-I{mock_base_prefix}/include/python3.11", target_dir
+        )
+        == f"-I{target_cpython_include}"
+    )
+
+
 def test_f2c():
     assert f2c_wrap("gfortran test.f") == "gcc test.c"
     assert f2c_wrap("gcc test.c") is None

pyproject.toml

@@ -108,8 +108,8 @@ norecursedirs = [
 ]
 addopts = '''
 --doctest-modules
---ignore="packages/matplotlib/src"
 --ignore-glob="**/dist/"
+--ignore-glob="packages/**/extras/"
 --tb=short
 --dist-dir=dist'''
 testpaths = [

src/core/pyproxy.ts

@@ -140,6 +140,7 @@ type PyProxyShared = {
   flags: number;
   promise: Promise<any> | undefined;
   destroyed_msg: string | undefined;
+  gcRegistered: boolean;
 };
 type PyProxyProps = {
   /**
@@ -261,6 +262,7 @@ function pyproxy_new(
       flags,
       promise: undefined,
       destroyed_msg: undefined,
+      gcRegistered: false,
     };
     Module._Py_IncRef(ptr);
   }
@@ -293,6 +295,7 @@ Module.pyproxy_new = pyproxy_new;
 
 function gc_register_proxy(shared: PyProxyShared) {
   const shared_copy = Object.assign({}, shared);
+  shared.gcRegistered = true;
   Module.finalizationRegistry.register(shared, shared_copy, shared);
 }
 Module.gc_register_proxy = gc_register_proxy;
@@ -462,7 +465,9 @@ Module.pyproxy_destroy = function (
   // just in case!
   const ptr = shared.ptr;
   shared.ptr = 0;
-  Module.finalizationRegistry.unregister(shared);
+  if (shared.gcRegistered) {
+    Module.finalizationRegistry.unregister(shared);
+  }
   pyproxy_decref_cache(shared.cache);
 
   try {