From b0162c139e4ea2e4783402de26617c912eef1e19 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 18 Oct 2024 14:02:25 +0200 Subject: [PATCH 01/29] Remove webworker.js template and update webworker docs (#5114) --- Makefile | 14 -- docs/project/changelog.md | 5 + docs/usage/downloading-and-deploying.md | 1 - docs/usage/webworker.md | 215 ++++++++++-------------- src/templates/module_webworker.js | 26 --- src/templates/webworker.js | 26 --- 6 files changed, 94 insertions(+), 193 deletions(-) delete mode 100644 src/templates/module_webworker.js delete mode 100644 src/templates/webworker.js diff --git a/Makefile b/Makefile index 071b488138f..218619f2cc1 100644 --- a/Makefile +++ b/Makefile @@ -28,9 +28,6 @@ all-but-packages: \ dist/python_stdlib.zip \ dist/test.html \ dist/module_test.html \ - dist/webworker.js \ - dist/webworker_dev.js \ - dist/module_webworker_dev.js \ src/core/pyodide_pre.o: src/js/generated/_pyodide.out.js src/core/pre.js src/core/stack_switching/stack_switching.out.js @@ -261,15 +258,6 @@ dist/console.html: src/templates/console.html cp $< $@ sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@ -dist/webworker.js: src/templates/webworker.js - cp $< $@ - -dist/module_webworker_dev.js: src/templates/module_webworker.js - cp $< $@ - -dist/webworker_dev.js: src/templates/webworker.js - cp $< $@ - # Prepare the dist directory for the release by removing unneeded files .PHONY: clean-dist-dir @@ -278,8 +266,6 @@ clean-dist-dir: rm dist/makesnap.mjs rm dist/snapshot.bin rm dist/module_test.html dist/test.html - # TODO: Remove webworker.js too? Would require updating the docs I think. - rm dist/module_webworker_dev.js dist/webworker_dev.js # TODO: Source maps aren't useful outside of debug builds I don't think. But # removing them adds "missing sourcemap" warnings to JS console. We should diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 4c9d06d779e..260354540b4 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -66,6 +66,11 @@ myst: for namespace packages. {pr}`5039` +- {{ Breaking }} Removed `webworker.js` from the distribution files. It was a + pretty small file, if someone is using it for something they can just copy it into + their own code. + {pr}`5114` + - {{ Enhancement }} Enabled `pyodide.FS.trackingDelegate` which can be used to track file system operations. See [Emscripten docs](https://emscripten.org/docs/api_reference/Filesystem-API.html#FS.trackingDelegate[callback%20name]) diff --git a/docs/usage/downloading-and-deploying.md b/docs/usage/downloading-and-deploying.md index 1bf05a08478..616866d4e07 100644 --- a/docs/usage/downloading-and-deploying.md +++ b/docs/usage/downloading-and-deploying.md @@ -87,4 +87,3 @@ deployments](https://emscripten.org/docs/compiling/Deploying-Pages.html). | pyodide.mjs.map | Source maps to improve tracebacks. Not really that useful to people outside of the project, probably should be only included in debug builds. | | \*-tests.tar | Unvendored tests from wheels. If a wheel includes a test folder, we take them out and put them here. | | console.html | The Pyodide repl. | -| webworker.js | Mentioned in the docs. Developers should probably implement their own Web Worker. | diff --git a/docs/usage/webworker.md b/docs/usage/webworker.md index 2abcc1666bb..3ddb6bf691c 100644 --- a/docs/usage/webworker.md +++ b/docs/usage/webworker.md @@ -2,31 +2,24 @@ # Using Pyodide in a web worker -This document describes how to use Pyodide to execute Python scripts -asynchronously in a web worker. +This document includes an example demonstrating how to use Pyodide to execute +Python scripts asynchronously in a web worker. -## Setup +Let's start with [the definition of a worker][worker api]. -Setup your project to serve `webworker.js`. You should also serve -`pyodide.js`, and all its associated `.asm.js`, `.json`, and `.wasm` -files as well, though this is not strictly required if `pyodide.js` is pointing -to a site serving current versions of these files. -The simplest way to serve the required files is to use a CDN, -such as `https://cdn.jsdelivr.net/pyodide`. This is the solution -presented here. +> A worker is an object created using a constructor (e.g. [Worker()][worker constructor]) +> that runs a named JavaScript file — this file contains the code +> that will run in the worker thread; workers run in another global context that +> is different from the current window. -Update the `webworker.js` sample so that it has as valid URL for `pyodide.js`, and sets -{js:func}`indexURL ` to the location of the supporting files. +A lot of Python programs do long-running synchronous computations. Running them +in the main thread blocks the UI. Using a web worker is advantageous because the +Python code runs in a separate thread from your UI and does not impact your +application's responsiveness. -In your application code create a web worker `new Worker(...)`, -and attach listeners to it using its `.onerror` and `.onmessage` -methods (listeners). - -Communication from the worker to the main thread is done via the `Worker.postMessage()` -method (and vice versa). - -[worker onmessage]: https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers#Sending_messages_to_and_from_a_dedicated_worker -[worker onerror]: https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers#Handling_errors +On the other hand, since workers run in a separate global context, you cannot +directly share globals between a worker and the main thread. In particular, a +worker cannot directly manipulate the DOM. ## Detailed example @@ -34,25 +27,26 @@ In this example process we will have three parties involved: - The **web worker** is responsible for running scripts in its own separate thread. - The **worker API** exposes a consumer-to-provider communication interface. -- The **consumer**s want to run some scripts outside the main thread, so they don't block the main thread. +- The **consumer**s want to run some scripts outside the main thread, so they + don't block the main thread. ### Consumers -Our goal is to run some Python code in another thread, this other thread will -not have access to the main thread objects. Therefore, we will need an API that takes -as input not only the Python `script` we want to run, but also the `context` on which -it relies (some JavaScript variables that we would normally get access to if we -were running the Python script in the main thread). Let's first describe what API -we would like to have. +Our goal is to run some Python code in another thread. This other thread will +not have access to the main thread objects. Therefore, we will need an API that +takes as input both the Python `script` we want to run and the `context` on +which it relies (some JavaScript variables that our code needs access to). Let's +first describe what API we would like to have. -Here is an example of consumer that will exchange with the web worker, via the worker interface/API `py-worker.js`. It runs the following Python `script` using the provided `context` and a function called `asyncRun()`. +Here is an example of consumer that will exchange with the web worker, via +`workerApi.mjs`. It runs the following Python `script` using the provided +`context` and a function called `asyncRun()`. ```js -import { asyncRun } from "./py-worker"; +import { asyncRun } from "./workerApi.js"; const script = ` import statistics - from js import A_rank statistics.stdev(A_rank) `; @@ -61,72 +55,43 @@ const context = { }; async function main() { - try { - const { results, error } = await asyncRun(script, context); - if (results) { - console.log("pyodideWorker return results: ", results); - } else if (error) { - console.log("pyodideWorker error: ", error); - } - } catch (e) { - console.log( - `Error in pyodideWorker at ${e.filename}, Line: ${e.lineno}, ${e.message}`, - ); + const { result, error } = await asyncRun(script, context); + if (result) { + console.log("pyodideWorker result:", result); + } else if (error) { + console.log("pyodideWorker error:", error); } } main(); ``` -Before writing the API, let's first have a look at how the worker operates. -How does our web worker run the `script` using a given `context`. +Before writing the API, let's first have a look at how the worker runs the +`script` using a given `context`. ### Web worker -Let's start with the definition. [A worker][worker api] is: - -> A worker is an object created using a constructor (e.g. [Worker()][worker constructor]) that runs a named JavaScript file — this file contains the code that will run in the worker thread; workers run in another global context that is different from the current window. This context is represented by either a DedicatedWorkerGlobalScope object (in the case of dedicated workers - workers that are utilized by a single script), or a SharedWorkerGlobalScope (in the case of shared workers - workers that are shared between multiple scripts). - -In our case we will use a single worker to execute Python code without interfering with -client side rendering (which is done by the main JavaScript thread). The worker does -two things: - -1. Listen on new messages from the main thread -2. Respond back once it finished executing the Python script - -These are the required tasks it should fulfill, but it can do other things. For -example, to always load packages `numpy` and `pytz`, you would insert the line -{js:func}`await pyodide.loadPackage(['numpy', 'pytz']); ` as -shown below: +Here is the worker code: ```js -// webworker.js +// webworker.mjs +import { loadPyodide } from "{{PYODIDE_CDN_URL}}pyodide.mjs"; -// Setup your project to serve `py-worker.js`. You should also serve -// `pyodide.js`, and all its associated `.asm.js`, `.json`, -// and `.wasm` files as well: -importScripts("{{PYODIDE_CDN_URL}}pyodide.js"); - -async function loadPyodideAndPackages() { - self.pyodide = await loadPyodide(); - await self.pyodide.loadPackage(["numpy", "pytz"]); -} -let pyodideReadyPromise = loadPyodideAndPackages(); +let pyodideReadyPromise = loadPyodide(); self.onmessage = async (event) => { // make sure loading is done - await pyodideReadyPromise; - // Don't bother yet with this line, suppose our API is built in such a way: - const { id, python, ...context } = event.data; - // The worker copies the context in its own "memory" (an object mapping name to values) - for (const key of Object.keys(context)) { - self[key] = context[key]; - } - // Now is the easy part, the one that is similar to working in the main thread: + const pyodide = await pyodideReadyPromise; + const { id, python, context } = event.data; + // Now load any packages we need, run the code, and send the result back. + await pyodide.loadPackagesFromImports(python); + // make a Python dictionary with the data from `context` + const dict = pyodide.globals.get("dict"); + const globals = dict(Object.entries(context)); try { - await self.pyodide.loadPackagesFromImports(python); - let results = await self.pyodide.runPythonAsync(python); - self.postMessage({ results, id }); + // Execute the python code in this context + const result = await pyodide.runPythonAsync(python, { globals }); + self.postMessage({ result, id }); } catch (error) { self.postMessage({ error: error.message, id }); } @@ -135,57 +100,55 @@ self.onmessage = async (event) => { ### The worker API -Now that we established what the two sides need and how they operate, -let's connect them using this simple API (`py-worker.js`). This part is -optional and only a design choice, you could achieve similar results -by exchanging message directly between your main thread and the webworker. -You would just need to call `.postMessages()` with the right arguments as -this API does. +Now that we established what the two sides need and how they operate, let's +connect them using an API that wraps the message passing code into an +asynchronous function. ```js -const pyodideWorker = new Worker("./dist/webworker.js"); +// workerApi.mjs +function getPromiseAndResolve() { + let resolve; + let promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +} -const callbacks = {}; +// Each message needs a unique id to identify the response. In a real example, +// we might use a real uuid package +let lastId = 1; +function getId() { + return lastId++; +} -pyodideWorker.onmessage = (event) => { - const { id, ...data } = event.data; - const onSuccess = callbacks[id]; - delete callbacks[id]; - onSuccess(data); -}; +// Add an id to msg, send it to worker, then wait for a response with the same id. +// When we get such a response, use it to resolve the promise. +function requestResponse(worker, msg) { + const { promise, resolve } = getPromiseAndResolve(); + const id = getId(); + worker.addEventListener("message", function listener(event) { + if (event.data?.id !== id) { + return; + } + // This listener is done so remove it. + worker.removeEventListener("message", listener); + // Filter the id out of the result + const { id, ...rest } = data; + resolve(rest); + }); + worker.postMessage({ id, ...msg }); + return promise; +} + +const pyodideWorker = new Worker("./webworker.mjs", { type: "module" }); -const asyncRun = (() => { - let id = 0; // identify a Promise - return (script, context) => { - // the id could be generated more carefully - id = (id + 1) % Number.MAX_SAFE_INTEGER; - return new Promise((onSuccess) => { - callbacks[id] = onSuccess; - pyodideWorker.postMessage({ - ...context, - python: script, - id, - }); - }); - }; -})(); - -export { asyncRun }; +export function asyncRun(script, context) { + return requestResponse(pyodideWorker, { + context, + python: script, + }); +} ``` [worker api]: https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API [worker constructor]: https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker - -## Caveats - -Using a web worker is advantageous because the Python code is run in a separate -thread from your main UI, and hence does not impact your application's -responsiveness. -There are some limitations, however. -At present, Pyodide does not support sharing the Python interpreter and -packages between multiple web workers or with your main thread. -Since web workers are each in their own virtual machine, you also cannot share -globals between a web worker and your main thread. -Finally, although the web worker is separate from your main thread, -the web worker is itself single threaded, so only one Python script will -execute at a time. diff --git a/src/templates/module_webworker.js b/src/templates/module_webworker.js deleted file mode 100644 index 0fd899d6660..00000000000 --- a/src/templates/module_webworker.js +++ /dev/null @@ -1,26 +0,0 @@ -import { loadPyodide } from "./pyodide.mjs"; - -onmessage = async function (e) { - try { - const data = e.data; - for (let key of Object.keys(data)) { - if (key !== "python") { - // Keys other than python must be arguments for the python script. - // Set them on self, so that `from js import key` works. - self[key] = data[key]; - } - } - - if (!loadPyodide.inProgress) { - self.pyodide = await loadPyodide(); - } - await self.pyodide.loadPackagesFromImports(data.python); - let results = await self.pyodide.runPythonAsync(data.python); - self.postMessage({ results }); - } catch (e) { - // if you prefer messages with the error - self.postMessage({ error: e.message + "\n" + e.stack }); - // if you prefer onerror events - // setTimeout(() => { throw err; }); - } -}; diff --git a/src/templates/webworker.js b/src/templates/webworker.js deleted file mode 100644 index 7325a2896c4..00000000000 --- a/src/templates/webworker.js +++ /dev/null @@ -1,26 +0,0 @@ -importScripts("./pyodide.js"); - -onmessage = async function (e) { - try { - const data = e.data; - for (let key of Object.keys(data)) { - if (key !== "python") { - // Keys other than python must be arguments for the python script. - // Set them on self, so that `from js import key` works. - self[key] = data[key]; - } - } - - if (!loadPyodide.inProgress) { - self.pyodide = await loadPyodide(); - } - await self.pyodide.loadPackagesFromImports(data.python); - let results = await self.pyodide.runPythonAsync(data.python); - self.postMessage({ results }); - } catch (e) { - // if you prefer messages with the error - self.postMessage({ error: e.message + "\n" + e.stack }); - // if you prefer onerror events - // setTimeout(() => { throw err; }); - } -}; From 5fdd87b4cf49a30a10c56b1a4d5291b5d9e871e4 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sat, 19 Oct 2024 14:15:03 +0200 Subject: [PATCH 02/29] Update changelog for 0.26.3 (#5127) --- docs/project/changelog.md | 44 +++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 260354540b4..d491c990e07 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -36,24 +36,10 @@ myst: - {{ Enhancement }} Unvendored stdlibs are now packaged in a wheel format {pr}`4902` -- {{ Enhancement }} Updated stack switching support to handle new JSPI. - {pr}`4982` - -- {{ Performance }} Attribute lookup on a `JsProxy` is now about 40% faster. - {pr}`4961` - -- {{ Performance }} Method calls on a `JsProxy` are now much faster. If the - method has no arguments and no return value, it is about 80% faster. The - speedup for methods with arguments is less drastic but still quite a lot. - {pr}`4961` - - {{ Breaking }} Prebuilt third-party libraries like `openblas`, `openssl`, `zlib` are not included in the cross-build env anymore. {pr}`4995` -- {{ Fix }} `pyimport("a.b")` won't fail when `a` is removed by `del sys.modules["a"]` - {pr}`4993` - - {{ Enhancement }} Added `JsProxy.as_py_json` method to adapt from JavaScript JSON (Arrays and Objects). to Python JSON (lists and dicts). {pr}`4666` @@ -76,12 +62,6 @@ myst: [Emscripten docs](https://emscripten.org/docs/api_reference/Filesystem-API.html#FS.trackingDelegate[callback%20name]) for more information. -- {{ Fix }} It now works to convert a 0d Python buffer to JavaScript. - {pr}`5092` - -- {{ Fix }} It now works to convert buffers of 64 bit signed or unsigned integers to JavaScript. - {pr}`5092` - ### Packages - Upgraded `scikit-learn` to 1.5.2 {pr}`4823`, {pr}`5016`, {pr}`5072` @@ -108,6 +88,30 @@ myst: - Added `arro3-core`, `arro3-io`, and `arro3-compute` 0.3.0, 0.4.0, 0.4.1 {pr}`5020`, {pr}`5095`, {pr}`5104` - Added `tree-sitter` 0.23.0 {pr}`5099` +## Version 0.26.3 + +_October 19, 2024_ + +- {{ Performance }} Attribute lookup on a `JsProxy` is now about 40% faster. + {pr}`4961` + +- {{ Performance }} Method calls on a `JsProxy` are now much faster. If the + method has no arguments and no return value, it is about 80% faster. The + speedup for methods with arguments is less drastic but still quite a lot. + {pr}`4963` + +- {{ Enhancement }} Updated stack switching support to handle new JSPI. + {pr}`4982` + +- {{ Fix }} `pyimport("a.b")` won't fail when `a` is removed by `del sys.modules["a"]` + {pr}`4993` + +- {{ Fix }} It now works to convert a 0d Python buffer to JavaScript. + {pr}`5092` + +- {{ Fix }} It now works to convert buffers of 64 bit signed or unsigned integers to JavaScript. + {pr}`5092` + ## Version 0.26.2 _July 26, 2024_ From 858e0e4424a03ce7013d57e8a5013289d08c03fa Mon Sep 17 00:00:00 2001 From: Eric Brown Date: Mon, 21 Oct 2024 07:33:22 -0700 Subject: [PATCH 03/29] Add tree-sitter-java, tree-sitter-go, tree-sitter-python (#5102) This is a follow-on to PR #5099 which added tree-sitter to the packages. This change adds tree-sitter-java, tree-sitter-go, tree-sitter-python packages and respective unit tests. Java, Go, and Python are popular languages, so its useful to add these grammars so a user can parse source code in those languages. --- .gitignore | 1 + docs/project/changelog.md | 3 ++ packages/tree-sitter-go/meta.yaml | 23 ++++++++++ .../tree-sitter-go/test_tree_sitter_go.py | 41 ++++++++++++++++++ packages/tree-sitter-java/meta.yaml | 23 ++++++++++ .../tree-sitter-java/test_tree_sitter_java.py | 42 +++++++++++++++++++ packages/tree-sitter-python/meta.yaml | 23 ++++++++++ .../test_tree_sitter_python.py | 39 +++++++++++++++++ 8 files changed, 195 insertions(+) create mode 100644 packages/tree-sitter-go/meta.yaml create mode 100644 packages/tree-sitter-go/test_tree_sitter_go.py create mode 100644 packages/tree-sitter-java/meta.yaml create mode 100644 packages/tree-sitter-java/test_tree_sitter_java.py create mode 100644 packages/tree-sitter-python/meta.yaml create mode 100644 packages/tree-sitter-python/test_tree_sitter_python.py diff --git a/.gitignore b/.gitignore index 7d8d30f90f0..f59bc6264d5 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ pytest-pyodide tools/symlinks xbuildenv/ .pyodide-xbuildenv* +DS_Store diff --git a/docs/project/changelog.md b/docs/project/changelog.md index d491c990e07..857571dde14 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -87,6 +87,9 @@ myst: - Added `iminuit` 2.29.1 {pr}`4767`, {pr}`5072` - Added `arro3-core`, `arro3-io`, and `arro3-compute` 0.3.0, 0.4.0, 0.4.1 {pr}`5020`, {pr}`5095`, {pr}`5104` - Added `tree-sitter` 0.23.0 {pr}`5099` +- Added `tree-sitter-go` 0.23.1 {pr}`5102` +- Added `tree-sitter-java` 0.23.2 {pr}`5102` +- Added `tree-sitter-python` 0.23.2 {pr}`5102` ## Version 0.26.3 diff --git a/packages/tree-sitter-go/meta.yaml b/packages/tree-sitter-go/meta.yaml new file mode 100644 index 00000000000..303a1bf4f21 --- /dev/null +++ b/packages/tree-sitter-go/meta.yaml @@ -0,0 +1,23 @@ +package: + name: tree-sitter-go + version: 0.23.1 + top-level: + - tree_sitter_go +source: + # Have to use github source to work around broken sdist + # https://github.com/tree-sitter/tree-sitter-go/pull/151 + url: https://github.com/tree-sitter/tree-sitter-go/archive/refs/tags/v0.23.1.tar.gz + sha256: 4173bafc4c59be78642a0faf1bed8cb3854458aa59c506e8f51001a9f28da09b +requirements: + run: + - tree-sitter + host: + - tree-sitter +about: + home: https://github.com/tree-sitter/tree-sitter-go + PyPI: https://pypi.org/project/tree-sitter-go + summary: Go grammar for tree-sitter + license: MIT +extra: + recipe-maintainers: + - ericwb diff --git a/packages/tree-sitter-go/test_tree_sitter_go.py b/packages/tree-sitter-go/test_tree_sitter_go.py new file mode 100644 index 00000000000..a732395fa4a --- /dev/null +++ b/packages/tree-sitter-go/test_tree_sitter_go.py @@ -0,0 +1,41 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["tree-sitter-go"]) +def test_tree_sitter_go(selenium): + import textwrap + + import tree_sitter_go + from tree_sitter import Language, Parser + + GO_LANGUAGE = Language(tree_sitter_go.language()) + parser = Parser(GO_LANGUAGE) + + code = bytes( + textwrap.dedent( + """ + func foo() { + if bar { + baz() + } + } + """ + ), + "utf-8", + ) + tree = parser.parse(code) + root_node = tree.root_node + + assert str(root_node) == ( + "(source_file " + "(function_declaration " + "name: (identifier) " + "parameters: (parameter_list) " + "body: (block " + "(if_statement " + "condition: (identifier) " + "consequence: (block " + "(expression_statement (call_expression " + "function: (identifier) " + "arguments: (argument_list))))))))" + ) diff --git a/packages/tree-sitter-java/meta.yaml b/packages/tree-sitter-java/meta.yaml new file mode 100644 index 00000000000..b8c479dd418 --- /dev/null +++ b/packages/tree-sitter-java/meta.yaml @@ -0,0 +1,23 @@ +package: + name: tree-sitter-java + version: 0.23.2 + top-level: + - tree_sitter_java +source: + # Have to use github url to work around broken sdist: + # https://github.com/tree-sitter/tree-sitter-java/pull/188 + url: https://github.com/tree-sitter/tree-sitter-java/archive/refs/tags/v0.23.2.tar.gz + sha256: 062fe5746deb4e362ccb987228e7b41bfc017eb2b91a0413a7447719abaa07b2 +requirements: + run: + - tree-sitter + host: + - tree-sitter +about: + home: https://github.com/tree-sitter/tree-sitter-java + PyPI: https://pypi.org/project/tree-sitter-java + summary: Java grammar for tree-sitter + license: MIT +extra: + recipe-maintainers: + - ericwb diff --git a/packages/tree-sitter-java/test_tree_sitter_java.py b/packages/tree-sitter-java/test_tree_sitter_java.py new file mode 100644 index 00000000000..49726e0d4fe --- /dev/null +++ b/packages/tree-sitter-java/test_tree_sitter_java.py @@ -0,0 +1,42 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["tree-sitter-java"]) +def test_tree_sitter_java(selenium): + import textwrap + + import tree_sitter_java + from tree_sitter import Language, Parser + + JAV_LANGUAGE = Language(tree_sitter_java.language()) + parser = Parser(JAV_LANGUAGE) + + code = bytes( + textwrap.dedent( + """ + void foo() { + if (bar) { + baz(); + } + } + """ + ), + "utf-8", + ) + tree = parser.parse(code) + root_node = tree.root_node + + assert str(root_node) == ( + "(program " + "(method_declaration " + "type: (void_type) " + "name: (identifier) " + "parameters: (formal_parameters) " + "body: (block " + "(if_statement " + "condition: (parenthesized_expression (identifier)) " + "consequence: (block " + "(expression_statement (method_invocation " + "name: (identifier) " + "arguments: (argument_list))))))))" + ) diff --git a/packages/tree-sitter-python/meta.yaml b/packages/tree-sitter-python/meta.yaml new file mode 100644 index 00000000000..655178f97c7 --- /dev/null +++ b/packages/tree-sitter-python/meta.yaml @@ -0,0 +1,23 @@ +package: + name: tree-sitter-python + version: 0.23.2 + top-level: + - tree_sitter_python +source: + # Have to use github url to work around broken sdist: + # https://github.com/tree-sitter/tree-sitter-python/pull/285 + url: https://github.com/tree-sitter/tree-sitter-python/archive/refs/tags/v0.23.2.tar.gz + sha256: b38e5b1f5237377b506109978af76422975bda5859165835e3c7b5afee987383 +requirements: + run: + - tree-sitter + host: + - tree-sitter +about: + home: https://github.com/tree-sitter/tree-sitter-python + PyPI: https://pypi.org/project/tree-sitter-python + summary: Python grammar for tree-sitter + license: MIT +extra: + recipe-maintainers: + - ericwb diff --git a/packages/tree-sitter-python/test_tree_sitter_python.py b/packages/tree-sitter-python/test_tree_sitter_python.py new file mode 100644 index 00000000000..c34bc165f6d --- /dev/null +++ b/packages/tree-sitter-python/test_tree_sitter_python.py @@ -0,0 +1,39 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["tree-sitter-python"]) +def test_tree_sitter_python(selenium): + import textwrap + + import tree_sitter_python + from tree_sitter import Language, Parser + + PY_LANGUAGE = Language(tree_sitter_python.language()) + parser = Parser(PY_LANGUAGE) + + code = bytes( + textwrap.dedent( + """ + def foo(): + if bar: + baz() + """ + ), + "utf-8", + ) + tree = parser.parse(code) + root_node = tree.root_node + + assert str(root_node) == ( + "(module " + "(function_definition " + "name: (identifier) " + "parameters: (parameters) " + "body: (block " + "(if_statement " + "condition: (identifier) " + "consequence: (block " + "(expression_statement (call " + "function: (identifier) " + "arguments: (argument_list))))))))" + ) From 2cd495b88fdbe8ca226982f88ba50ddd4f6e2d87 Mon Sep 17 00:00:00 2001 From: Marco Edward Gorelli Date: Tue, 22 Oct 2024 08:43:24 +0100 Subject: [PATCH 04/29] Add Narwhals to package list (#5121) Narwhals is a lightweight compatibility layer between dataframe libraries, with zero dependencies. It's now used by Altair, so I'm opening this PR to enable them to update their version here on Pyodide vega/altair#3642 --- docs/project/changelog.md | 1 + packages/narwhals/meta.yaml | 16 ++++++++++++++++ packages/narwhals/test_narwhals.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 packages/narwhals/meta.yaml create mode 100644 packages/narwhals/test_narwhals.py diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 857571dde14..121bf7432aa 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -90,6 +90,7 @@ myst: - Added `tree-sitter-go` 0.23.1 {pr}`5102` - Added `tree-sitter-java` 0.23.2 {pr}`5102` - Added `tree-sitter-python` 0.23.2 {pr}`5102` +- Added `Narwhals` 1.9.4 {pr}`5121` ## Version 0.26.3 diff --git a/packages/narwhals/meta.yaml b/packages/narwhals/meta.yaml new file mode 100644 index 00000000000..455220285cf --- /dev/null +++ b/packages/narwhals/meta.yaml @@ -0,0 +1,16 @@ +package: + name: narwhals + version: 1.10.0 + top-level: + - narwhals +source: + url: https://files.pythonhosted.org/packages/82/45/6bb08a5e5ac2d9a95f4116567d0aafb1e5f86c4bbcf982e209e5ae872b3d/narwhals-1.10.0-py3-none-any.whl + sha256: c83a378960651c391e5f3d68af3a821eda74c9713073518fe0c39aefc5ad8f8e +about: + home: https://github.com/narwhals-dev/narwhals + PyPI: https://pypi.org/project/narwhals + summary: Extremely lightweight compatibility layer between dataframe libraries + license: MIT +extra: + recipe-maintainers: + - MarcoGorelli diff --git a/packages/narwhals/test_narwhals.py b/packages/narwhals/test_narwhals.py new file mode 100644 index 00000000000..a4c36e9b7de --- /dev/null +++ b/packages/narwhals/test_narwhals.py @@ -0,0 +1,21 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["narwhals"]) +def test_narwhals_from_native(selenium): + import narwhals as nw + + class MyDictDataFrame: + def __init__(self, data): + self._data = data + + def __narwhals_dataframe__(self): + return self + + @property + def columns(self): + return list(self._data) + + assert nw.from_native( + MyDictDataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + ).columns == ["a", "b"] From 2aa31871e4c22182c3945df5baa92b12b3d0ca00 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:44:21 +0300 Subject: [PATCH 05/29] Constrain meson < 1.6 to fix the scipy build (#5132) --- tools/constraints.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/constraints.txt b/tools/constraints.txt index e69de29bb2d..b1eb7ca825f 100644 --- a/tools/constraints.txt +++ b/tools/constraints.txt @@ -0,0 +1 @@ +meson < 1.6 From ce9990dccffde094ac04f6cad96b20b102a7d05b Mon Sep 17 00:00:00 2001 From: Bart Broere Date: Mon, 28 Oct 2024 06:12:28 +0100 Subject: [PATCH 06/29] Bump protobuf package to 5.28.3 (#5136) --- docs/project/changelog.md | 1 + packages/protobuf/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 121bf7432aa..66a5bfdd4e5 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,7 @@ myst: ### Packages +- Upgraded `protobuf` to 5.28.3 {pr}`5136` - Upgraded `scikit-learn` to 1.5.2 {pr}`4823`, {pr}`5016`, {pr}`5072` - Upgraded `libcst` to 1.4.0 {pr}`4856` - Upgraded `lakers` to 0.3.3 {pr}`4885` diff --git a/packages/protobuf/meta.yaml b/packages/protobuf/meta.yaml index ab7a5fc2f68..02a478e12de 100644 --- a/packages/protobuf/meta.yaml +++ b/packages/protobuf/meta.yaml @@ -1,12 +1,12 @@ package: name: protobuf - version: 5.27.3 + version: 5.28.3 top-level: - google source: - url: https://files.pythonhosted.org/packages/1b/61/0671db2ab2aee7c92d6c1b617c39b30a4cd973950118da56d77e7f397a9d/protobuf-5.27.3.tar.gz - sha256: 82460903e640f2b7e34ee81a947fdaad89de796d324bcbc38ff5430bcdead82c + url: https://files.pythonhosted.org/packages/74/6e/e69eb906fddcb38f8530a12f4b410699972ab7ced4e21524ece9d546ac27/protobuf-5.28.3.tar.gz + sha256: 64badbc49180a5e401f373f9ce7ab1d18b63f7dd4a9cdc43c92b9f0b481cef7b patches: - patches/0001-Fix-signature-of-PyUpb_MessageMeta_Clear.patch From 4967029933ba120e0a320f21f6c2b0b736b5cdfd Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 28 Oct 2024 01:49:12 -0700 Subject: [PATCH 07/29] Upgrade PyYAML to 6.0.2 (#5137) --- docs/project/changelog.md | 1 + packages/pyyaml/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 66a5bfdd4e5..4cfb9317fb2 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -82,6 +82,7 @@ myst: - Upgraded `duckdb` to 1.1.0 {pr}`5078` - Upgraded `sympy` to 1.13.3 {pr}`5098` - Upgraded `tree-sitter` to 0.23.1 {pr}`5110` +- Upgraded `PyYAML` to 6.0.2 {pr}`5137` - Added `casadi` 3.6.6 {pr}`4936`, {pr}`5057` - Added `pyarrow` 17.0.0 {pr}`4950` - Added `rasterio` 1.13.10, `affine` 2.4.0 {pr}`4983` diff --git a/packages/pyyaml/meta.yaml b/packages/pyyaml/meta.yaml index fb3805d6c40..86473397059 100644 --- a/packages/pyyaml/meta.yaml +++ b/packages/pyyaml/meta.yaml @@ -1,12 +1,12 @@ package: name: pyyaml - version: 6.0.1 + version: 6.0.2 top-level: - _yaml - yaml source: - url: https://files.pythonhosted.org/packages/cd/e5/af35f7ea75cf72f2cd079c95ee16797de7cd71f29ea7c68ae5ce7be1eda0/PyYAML-6.0.1.tar.gz - sha256: bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 + url: https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz + sha256: d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e build: cflags: | -I$(PYTHONINCLUDE) From 376564e72bfeb6cd4b44706d6875876636714560 Mon Sep 17 00:00:00 2001 From: Castedo Ellerman Date: Thu, 31 Oct 2024 05:17:47 -0400 Subject: [PATCH 08/29] use vite-plugin-static-copy in doc example of Vite (#5125) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/usage/working-with-bundlers.md | 57 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/docs/usage/working-with-bundlers.md b/docs/usage/working-with-bundlers.md index 9bfdab7ef3d..5228e61ad1f 100644 --- a/docs/usage/working-with-bundlers.md +++ b/docs/usage/working-with-bundlers.md @@ -10,50 +10,49 @@ project. ## Vite ```{note} -The following instructions have been tested with Pyodide 0.26.0 and Vite 5.2.13. +The following instructions have been tested with Pyodide 0.26.2, Vite 5.4.9, and +vite-plugin-pyodide 2.0.0. ``` -First, install the Pyodide npm package: +First, install the Pyodide and vite-plugin-pyodide npm packages: ``` -$ npm install pyodide +$ npm install pyodide vite-plugin-static-copy ``` -Then, in your `vite.config.js` file, exclude Pyodide from [Vite's dependency +Then, in your `vite.config.mjs` file, exclude Pyodide from [Vite's dependency pre-bundling][optimizedeps] by setting `optimizeDeps.exclude` and ensure that all Pyodide files will be available in `dist/assets` for production builds by using a Vite plugin: ```js import { defineConfig } from "vite"; -import { copyFile, mkdir } from "fs/promises"; +import { viteStaticCopy } from "vite-plugin-static-copy"; import { dirname, join } from "path"; import { fileURLToPath } from "url"; +const PYODIDE_EXCLUDE = [ + "!**/*.{md,html}", + "!**/*.d.ts", + "!**/*.whl", + "!**/node_modules", +]; + +export function viteStaticCopyPyodide() { + const pyodideDir = dirname(fileURLToPath(import.meta.resolve("pyodide"))); + return viteStaticCopy({ + targets: [ + { + src: [join(pyodideDir, "*")].concat(PYODIDE_EXCLUDE), + dest: "assets", + }, + ], + }); +} + export default defineConfig({ optimizeDeps: { exclude: ["pyodide"] }, - plugins: [ - { - name: "vite-plugin-pyodide", - generateBundle: async () => { - const assetsDir = "dist/assets"; - await mkdir(assetsDir, { recursive: true }); - const files = [ - "pyodide-lock.json", - "pyodide.asm.js", - "pyodide.asm.wasm", - "python_stdlib.zip", - ]; - const modulePath = fileURLToPath(import.meta.resolve("pyodide")); - for (const file of files) { - await copyFile( - join(dirname(modulePath), file), - join(assetsDir, file), - ); - } - }, - }, - ], + plugins: [viteStaticCopyPyodide()], }); ``` @@ -64,12 +63,12 @@ You can test your setup with this `index.html` file: Vite + Pyodide - + ``` -And this `src/main.js` file: +And this `src/main.mjs` file: ```js import { loadPyodide } from "pyodide"; From 00b8accc530f4464c261abcd600e1aa8ed42600e Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 31 Oct 2024 23:46:40 +0900 Subject: [PATCH 09/29] MAINT Split large TS functions add add unit tests (#5123) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- src/js/compat.ts | 20 +++ src/js/dynload.ts | 24 ++-- src/js/load-package.ts | 148 ++++++++++++++--------- src/js/test/unit/compat.test.ts | 34 ++++++ src/js/test/unit/package-manager.test.ts | 136 +++++++++++++-------- src/js/test/unit/test-helper.ts | 56 +++++++++ 6 files changed, 294 insertions(+), 124 deletions(-) create mode 100644 src/js/test/unit/compat.test.ts create mode 100644 src/js/test/unit/test-helper.ts diff --git a/src/js/compat.ts b/src/js/compat.ts index 5a08873c942..3bd5eea4c24 100644 --- a/src/js/compat.ts +++ b/src/js/compat.ts @@ -283,3 +283,23 @@ export async function calculateDirname(): Promise { } return fileName.slice(0, indexOfLastSlash); } + +/** + * Ensure that the directory exists before trying to download files into it (Node.js only). + * @param dir The directory to ensure exists + */ +export async function ensureDirNode(dir: string) { + if (!IN_NODE) { + return; + } + + try { + // Check if the `installBaseUrl` directory exists + await nodeFsPromisesMod.stat(dir); // Use `.stat()` which works even on ASAR archives of Electron apps, while `.access` doesn't. + } catch { + // If it doesn't exist, make it. Call mkdir() here only when necessary after checking the existence to avoid an error on read-only file systems. See https://github.com/pyodide/pyodide/issues/4736 + await nodeFsPromisesMod.mkdir(dir, { + recursive: true, + }); + } +} diff --git a/src/js/dynload.ts b/src/js/dynload.ts index 98bee72839c..cb55cad7764 100644 --- a/src/js/dynload.ts +++ b/src/js/dynload.ts @@ -1,7 +1,5 @@ /* Handle dynamic library loading. */ -declare var DEBUG: boolean; - import { PackageManagerAPI, PackageManagerModule } from "./types"; import { createLock } from "./common/lock"; @@ -69,14 +67,11 @@ export class DynlibLoader { // TODO: add rpath to Emscripten dsos and remove this logic const resolvePath = (path: string) => { - if (DEBUG) { - if ( - this.#module.PATH.basename(path) !== this.#module.PATH.basename(lib) - ) { - console.debug( - `Searching a library from ${path}, required by ${lib}.`, - ); - } + if ( + DEBUG && + this.#module.PATH.basename(path) !== this.#module.PATH.basename(lib) + ) { + console.debug(`Searching a library from ${path}, required by ${lib}.`); } // If the path is absolute, we don't need to search for it. @@ -114,10 +109,8 @@ export class DynlibLoader { resolvePath(path), dontResolveLastLink, ); - if (DEBUG) { - if (obj === null) { - console.debug(`Failed to find a library: ${resolvePath(path)}`); - } + if (DEBUG && obj === null) { + console.debug(`Failed to find a library: ${resolvePath(path)}`); } return obj; }, @@ -141,9 +134,8 @@ export class DynlibLoader { public async loadDynlib(lib: string, global: boolean, searchDirs?: string[]) { const releaseDynlibLock = await this._lock(); - if (DEBUG) { + DEBUG && console.debug(`Loading a dynamic library ${lib} (global: ${global})`); - } const fs = this.createDynlibFS(lib, searchDirs); const localScope = global ? null : {}; diff --git a/src/js/load-package.ts b/src/js/load-package.ts index 218eee7a3cb..caad827755e 100644 --- a/src/js/load-package.ts +++ b/src/js/load-package.ts @@ -21,6 +21,7 @@ import { loadBinaryFile, resolvePath, initNodeModules, + ensureDirNode, } from "./compat"; import { DynlibLoader } from "./dynload"; @@ -120,6 +121,14 @@ export class PackageManager { private _lock = createLock(); + /** + * The function to use for stdout and stderr, defaults to console.log and console.error + */ + private stdout: (message: string) => void = console.log; + private stderr: (message: string) => void = console.error; + + private defaultChannel: string = DEFAULT_CHANNEL; + constructor(api: PackageManagerAPI, pyodideModule: PackageManagerModule) { this.#api = api; this.#module = pyodideModule; @@ -173,46 +182,35 @@ export class PackageManager { }, ): Promise> { const loadedPackageData = new Set(); - const messageCallback = options.messageCallback || console.log; - const errorCallback = options.errorCallback || console.error; + const { messageCallback, errorCallback } = options; + const pkgNames = toStringArray(names); - // originally, this condition was "names instanceof PyProxy", - // but it is changed to check names.toJs so that we can use type-only import for PyProxy and remove side effects. - // this change is required to run unit tests against this file, when global API or Module is not available. - // TODO: remove side effects from pyproxy.ts so that we can directly import PyProxy - // @ts-ignore - if (typeof names.toJs === "function") { - // @ts-ignore - names = names.toJs(); - } - if (!Array.isArray(names)) { - names = [names as string]; - } - - const toLoad = this.recursiveDependencies(names, errorCallback); + const toLoad = this.recursiveDependencies(pkgNames, errorCallback); for (const [_, { name, normalizedName, channel }] of toLoad) { - const loaded = loadedPackages[name]; - if (loaded === undefined) { - continue; - } + const loadedChannel = this.getLoadedPackageChannel(name); + if (!loadedChannel) continue; toLoad.delete(normalizedName); - // If uri is from the DEFAULT_CHANNEL, we assume it was added as a + // If uri is from the default channel, we assume it was added as a // dependency, which was previously overridden. - if (loaded === channel || channel === DEFAULT_CHANNEL) { - messageCallback(`${name} already loaded from ${loaded}`); + if (loadedChannel === channel || channel === this.defaultChannel) { + this.logStdout( + `${name} already loaded from ${loadedChannel}`, + messageCallback, + ); } else { - errorCallback( + this.logStderr( `URI mismatch, attempting to load package ${name} from ${channel} ` + - `while it is already loaded from ${loaded}. To override a dependency, ` + + `while it is already loaded from ${loadedChannel}. To override a dependency, ` + `load the custom package first.`, + errorCallback, ); } } if (toLoad.size === 0) { - messageCallback("No new packages to load"); + this.logStdout("No new packages to load", messageCallback); return []; } @@ -222,9 +220,9 @@ export class PackageManager { const failed = new Map(); const releaseLock = await this._lock(); try { - messageCallback(`Loading ${packageNames}`); + this.logStdout(`Loading ${packageNames}`, messageCallback); for (const [_, pkg] of toLoad) { - if (loadedPackages[pkg.name]) { + if (this.getLoadedPackageChannel(pkg.name)) { // Handle the race condition where the package was loaded between when // we did dependency resolution and when we acquired the lock. toLoad.delete(pkg.normalizedName); @@ -253,15 +251,18 @@ export class PackageManager { const successNames = Array.from(loadedPackageData, (pkg) => pkg.name) .sort() .join(", "); - messageCallback(`Loaded ${successNames}`); + this.logStdout(`Loaded ${successNames}`, messageCallback); } if (failed.size > 0) { const failedNames = Array.from(failed.keys()).sort().join(", "); - messageCallback(`Failed to load ${failedNames}`); + this.logStdout(`Failed to load ${failedNames}`, messageCallback); for (const [name, err] of failed) { - errorCallback(`The following error occurred while loading ${name}:`); - errorCallback(err.message); + this.logStderr( + `The following error occurred while loading ${name}:`, + errorCallback, + ); + this.logStderr(err.message, errorCallback); } } @@ -297,7 +298,7 @@ export class PackageManager { toLoad.set(normalizedName, { name: pkgInfo.name, normalizedName, - channel: DEFAULT_CHANNEL, + channel: this.defaultChannel, depends: pkgInfo.depends, installPromise: undefined, done: createResolvable(), @@ -307,7 +308,7 @@ export class PackageManager { // If the package is already loaded, we don't add dependencies, but warn // the user later. This is especially important if the loaded package is // from a custom url, in which case adding dependencies is wrong. - if (loadedPackages[pkgInfo.name] !== undefined) { + if (this.getLoadedPackageChannel(pkgInfo.name)) { return; } @@ -324,7 +325,7 @@ export class PackageManager { */ public recursiveDependencies( names: string[], - errorCallback: (err: string) => void, + errorCallback?: (err: string) => void, ): Map { const toLoad: Map = new Map(); for (let name of names) { @@ -338,10 +339,11 @@ export class PackageManager { const channel = name; if (toLoad.has(pkgname) && toLoad.get(pkgname)!.channel !== channel) { - errorCallback( + this.logStderr( `Loading same package ${pkgname} from ${channel} and ${ toLoad.get(pkgname)!.channel }`, + errorCallback, ); continue; } @@ -383,25 +385,13 @@ export class PackageManager { pkg: PackageLoadMetadata, checkIntegrity: boolean = true, ): Promise { - let installBaseUrl: string; - if (IN_NODE) { - installBaseUrl = this.#api.config.packageCacheDir; - // Ensure that the directory exists before trying to download files into it. - try { - // Check if the `installBaseUrl` directory exists - await nodeFsPromisesMod.stat(installBaseUrl); // Use `.stat()` which works even on ASAR archives of Electron apps, while `.access` doesn't. - } catch { - // If it doesn't exist, make it. Call mkdir() here only when necessary after checking the existence to avoid an error on read-only file systems. See https://github.com/pyodide/pyodide/issues/4736 - await nodeFsPromisesMod.mkdir(installBaseUrl, { - recursive: true, - }); - } - } else { - installBaseUrl = this.#api.config.indexURL; - } + const installBaseUrl = IN_NODE + ? this.#api.config.packageCacheDir + : this.#api.config.indexURL; + await ensureDirNode(installBaseUrl); let fileName, uri, fileSubResourceHash; - if (pkg.channel === DEFAULT_CHANNEL) { + if (pkg.channel === this.defaultChannel) { if (!(pkg.normalizedName in this.#api.lockfile_packages)) { throw new Error(`Internal error: no entry for package named ${name}`); } @@ -421,7 +411,7 @@ export class PackageManager { try { return await loadBinaryFile(uri, fileSubResourceHash); } catch (e) { - if (!IN_NODE || pkg.channel !== DEFAULT_CHANNEL) { + if (!IN_NODE || pkg.channel !== this.defaultChannel) { throw e; } } @@ -466,15 +456,16 @@ export class PackageManager { calculate_dynlibs: true, installer: "pyodide.loadPackage", source: - metadata.channel === DEFAULT_CHANNEL ? "pyodide" : metadata.channel, + metadata.channel === this.defaultChannel + ? "pyodide" + : metadata.channel, }, ); - if (DEBUG) { + DEBUG && console.debug( `Found ${dynlibs.length} dynamic libraries inside ${filename}`, ); - } await this.#dynlibLoader.loadDynlibsFromPackage(pkg, dynlibs); } @@ -530,6 +521,28 @@ export class PackageManager { public setCdnUrl(url: string) { this.cdnURL = url; } + + /** + * getLoadedPackageChannel returns the channel from which a package was loaded. + * if the package is not loaded, it returns null. + * @param pkg package name + */ + public getLoadedPackageChannel(pkg: string): string | null { + const channel = this.loadedPackages[pkg]; + if (channel === undefined) { + return null; + } + + return channel; + } + + public logStdout(message: string, logger?: (message: string) => void) { + logger ? logger(message) : this.stdout(message); + } + + public logStderr(message: string, logger?: (message: string) => void) { + logger ? logger(message) : this.stderr(message); + } } function filterPackageData({ @@ -541,6 +554,27 @@ function filterPackageData({ return { name, version, fileName: file_name, packageType: package_type }; } +/** + * Converts a string or PyProxy to an array of strings. + * @private + */ +export function toStringArray(str: string | PyProxy | string[]): string[] { + // originally, this condition was "names instanceof PyProxy", + // but it is changed to check names.toJs so that we can use type-only import for PyProxy and remove side effects. + // this change is required to run unit tests against this file, when global API or Module is not available. + // TODO: remove side effects from pyproxy.ts so that we can directly import PyProxy + // @ts-ignore + if (typeof str.toJs === "function") { + // @ts-ignore + str = str.toJs(); + } + if (!Array.isArray(str)) { + str = [str as string]; + } + + return str; +} + export let loadPackage: typeof PackageManager.prototype.loadPackage; export let loadedPackages: typeof PackageManager.prototype.loadedPackages; diff --git a/src/js/test/unit/compat.test.ts b/src/js/test/unit/compat.test.ts new file mode 100644 index 00000000000..f0b79ba6511 --- /dev/null +++ b/src/js/test/unit/compat.test.ts @@ -0,0 +1,34 @@ +import * as chai from "chai"; +import * as fs from "fs"; +import * as path from "path"; +import * as os from "os"; + +import { ensureDirNode, initNodeModules } from "../../compat"; + +describe("ensureDirNode", () => { + it("Should create the dir if it does not exist", async () => { + await initNodeModules(); + + const baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "foo-")); + + const notExistDir = path.join(baseDir, "notExistDir"); + + chai.assert.isFalse(fs.existsSync(notExistDir)); + + await ensureDirNode(notExistDir); + + chai.assert.isTrue(fs.existsSync(notExistDir)); + }); + + it("Should not throw if the dir already exists", async () => { + await initNodeModules(); + + const baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "foo-")); + + chai.assert.isTrue(fs.existsSync(baseDir)); + + await ensureDirNode(baseDir); + + chai.assert.isTrue(fs.existsSync(baseDir)); + }); +}); diff --git a/src/js/test/unit/package-manager.test.ts b/src/js/test/unit/package-manager.test.ts index 563a69f766c..8ef36b8dd26 100644 --- a/src/js/test/unit/package-manager.test.ts +++ b/src/js/test/unit/package-manager.test.ts @@ -1,59 +1,93 @@ -import { PackageManager } from "../../load-package.ts"; -import { PackageManagerAPI, PackageManagerModule } from "../../types.ts"; +import * as chai from "chai"; +import sinon from "sinon"; +import { PackageManager, toStringArray } from "../../load-package.ts"; +import { genMockAPI, genMockModule } from "./test-helper.ts"; describe("PackageManager", () => { - // TODO: add more unittests it("should initialize with API and Module", () => { - const mockApi: PackageManagerAPI = { - importlib: { - invalidate_caches: () => {}, - }, - package_loader: { - get_install_dir: () => "", - init_loaded_packages: () => {}, - unpack_buffer: { - callKwargs: () => {}, - }, - }, - config: { - indexURL: "", - packageCacheDir: "", - }, - lockfile_packages: {}, - bootstrapFinalizedPromise: Promise.resolve(), - sitepackages: "", - defaultLdLibraryPath: [], + const mockApi = genMockAPI(); + const mockMod = genMockModule(); + const _ = new PackageManager(mockApi, mockMod); + }); +}); + +describe("logStdout and logStderr", () => { + it("Should use console.log and console.error if no logger is provided", () => { + const mockApi = genMockAPI(); + const mockMod = genMockModule(); + + const pm = new PackageManager(mockApi, mockMod); + + const logSpy = sinon.spy(pm, "stdout"); + const errorSpy = sinon.spy(pm, "stderr"); + + pm.logStdout("stdout"); + pm.logStderr("stderr"); + + chai.assert.isTrue(logSpy.calledOnce); + chai.assert.isTrue(errorSpy.calledOnce); + chai.assert.isTrue(logSpy.calledWith("stdout")); + chai.assert.isTrue(errorSpy.calledWith("stderr")); + + logSpy.restore(); + errorSpy.restore(); + }); + + it("Should use the provided logger if one is provided", () => { + const mockApi = genMockAPI(); + const mockMod = genMockModule(); + + const pm = new PackageManager(mockApi, mockMod); + + const stdoutLogger = sinon.spy(); + const stderrLogger = sinon.spy(); + + pm.logStdout("stdout", stdoutLogger); + pm.logStderr("stderr", stderrLogger); + + chai.assert.isTrue(stdoutLogger.calledOnce); + chai.assert.isTrue(stderrLogger.calledOnce); + chai.assert.isTrue(stdoutLogger.calledWith("stdout")); + chai.assert.isTrue(stderrLogger.calledWith("stderr")); + }); +}); + +describe("toStringArray", () => { + it("Should convert string to array of strings", () => { + chai.assert.deepEqual(toStringArray("hello"), ["hello"]); + }); + + it("Should return the array if it is already an array", () => { + chai.assert.deepEqual(toStringArray(["hello", "world"]), [ + "hello", + "world", + ]); + }); + + it("Should convert PyProxy to array of strings", () => { + // TODO: use real PyProxy + const pyProxyMock = { + toJs: () => ["hello", "world"], }; - const mockMod: PackageManagerModule = { - reportUndefinedSymbols: () => {}, - loadDynamicLibrary: () => {}, - LDSO: { - loadedLibsByName: {}, - }, - PATH: {}, - FS: { - readdir: (path: string) => [], - isDir: (mode: number) => true, - findObject: (path: string, dontResolveLastLink?: boolean) => {}, - readFile: (path: string) => new Uint8Array(), - lookupPath: ( - path: string, - options?: { - follow_mount?: boolean; - }, - ) => { - return { - node: { - timestamp: 1, - rdev: 2, - contents: new Uint8Array(), - mode: 3, - }, - }; - }, - }, + + chai.assert.deepEqual(toStringArray(pyProxyMock), ["hello", "world"]); + }); +}); + +describe("getLoadedPackageChannel", () => { + it("Should return the loaded package from loadedPackages obj", () => { + const mockApi = genMockAPI(); + const mockMod = genMockModule(); + + const pm = new PackageManager(mockApi, mockMod); + pm.loadedPackages = { + package: "channel", }; - const _ = new PackageManager(mockApi, mockMod); + const loadedPackage = pm.getLoadedPackageChannel("package"); + chai.assert.equal(loadedPackage, "channel"); + + const notLoadedPackage = pm.getLoadedPackageChannel("notLoadedPackage"); + chai.assert.equal(notLoadedPackage, null); }); }); diff --git a/src/js/test/unit/test-helper.ts b/src/js/test/unit/test-helper.ts new file mode 100644 index 00000000000..b1abcfaa2b0 --- /dev/null +++ b/src/js/test/unit/test-helper.ts @@ -0,0 +1,56 @@ +import { PackageManagerAPI, PackageManagerModule } from "../../types.ts"; + +export const genMockAPI = (): PackageManagerAPI => { + return { + importlib: { + invalidate_caches: () => {}, + }, + package_loader: { + get_install_dir: () => "", + init_loaded_packages: () => {}, + unpack_buffer: { + callKwargs: () => {}, + }, + }, + config: { + indexURL: "", + packageCacheDir: "", + }, + lockfile_packages: {}, + bootstrapFinalizedPromise: Promise.resolve(), + sitepackages: "", + defaultLdLibraryPath: [], + }; +}; + +export const genMockModule = (): PackageManagerModule => { + return { + reportUndefinedSymbols: () => {}, + loadDynamicLibrary: () => {}, + LDSO: { + loadedLibsByName: {}, + }, + PATH: {}, + FS: { + readdir: (path: string) => [], + isDir: (mode: number) => true, + findObject: (path: string, dontResolveLastLink?: boolean) => {}, + readFile: (path: string) => new Uint8Array(), + lookupPath: ( + path: string, + options?: { + follow_mount?: boolean; + }, + ) => { + return { + node: { + timestamp: 1, + rdev: 2, + contents: new Uint8Array(), + mode: 3, + }, + }; + }, + }, + }; +}; From 6071fa7eb6701065ae3d0e405fa9178ae3b043c3 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Fri, 1 Nov 2024 19:08:59 +0900 Subject: [PATCH 10/29] CI Split library build job (#5148) --- .circleci/config.yml | 16 ++++++++++++++++ packages/boost-cpp/meta.yaml | 1 + packages/cpp-exceptions-test/meta.yaml | 1 + packages/ffmpeg/meta.yaml | 1 + packages/flint/meta.yaml | 1 + packages/gdal/meta.yaml | 1 + packages/geos/meta.yaml | 1 + packages/glpk/meta.yaml | 1 + packages/libde265/meta.yaml | 1 + packages/libf2c/meta.yaml | 1 + packages/libgmp/meta.yaml | 1 + packages/libgsl/meta.yaml | 1 + packages/libhdf5/meta.yaml | 1 + packages/libheif/meta.yaml | 1 + packages/libiconv/meta.yaml | 1 + packages/liblzma/meta.yaml | 1 + packages/libmagic/meta.yaml | 1 + packages/libmpc/meta.yaml | 1 + packages/libmpfr/meta.yaml | 1 + packages/libnetcdf/meta.yaml | 1 + packages/libproj/meta.yaml | 1 + packages/libtiff/meta.yaml | 1 + packages/libwebp/meta.yaml | 1 + packages/libxml/meta.yaml | 1 + packages/libxslt/meta.yaml | 1 + packages/libyaml/meta.yaml | 1 + packages/openblas/meta.yaml | 1 + packages/openssl/meta.yaml | 1 + packages/ppl/meta.yaml | 1 + packages/primecount/meta.yaml | 1 + packages/primesieve/meta.yaml | 1 + packages/sharedlib-test/meta.yaml | 1 + packages/suitesparse/meta.yaml | 1 + packages/zlib/meta.yaml | 1 + 34 files changed, 49 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d26d926412..8d98476b2a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -578,11 +578,27 @@ workflows: tags: only: /.*/ + - build-packages: + name: build-static-libraries + packages: "tag:static_library" + requires: + - build-core + filters: + tags: + only: /.*/ + post-steps: + - persist_to_workspace: + root: . + paths: + - ./packages + - ./dist + - build-packages: name: build-libraries packages: "tag:library" requires: - build-core + - build-static-libraries filters: tags: only: /.*/ diff --git a/packages/boost-cpp/meta.yaml b/packages/boost-cpp/meta.yaml index 407062be8f8..db4f2871755 100644 --- a/packages/boost-cpp/meta.yaml +++ b/packages/boost-cpp/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.84.0 tag: - library + - static_library source: url: https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz sha256: 4d27e9efed0f6f152dc28db6430b9d3dfb40c0345da7342eaa5a987dde57bd95 diff --git a/packages/cpp-exceptions-test/meta.yaml b/packages/cpp-exceptions-test/meta.yaml index 04afb8cd6d3..09e5bd9c071 100644 --- a/packages/cpp-exceptions-test/meta.yaml +++ b/packages/cpp-exceptions-test/meta.yaml @@ -5,6 +5,7 @@ package: - core - pyodide.test - library + - shared_library source: path: src build: diff --git a/packages/ffmpeg/meta.yaml b/packages/ffmpeg/meta.yaml index 5a217b01379..300e10f9364 100644 --- a/packages/ffmpeg/meta.yaml +++ b/packages/ffmpeg/meta.yaml @@ -3,6 +3,7 @@ package: version: "4.4.1" tag: - library + - static_library source: url: https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n4.4.1.tar.gz sha256: 82b43cc67296bcd01a59ae6b327cdb50121d3a9e35f41a30de1edd71bb4a6666 diff --git a/packages/flint/meta.yaml b/packages/flint/meta.yaml index 7f425e35a55..ae42bcd02ad 100644 --- a/packages/flint/meta.yaml +++ b/packages/flint/meta.yaml @@ -3,6 +3,7 @@ package: version: 3.0.1 tag: - library + - static_library source: url: https://github.com/flintlib/flint/releases/download/v3.0.1/flint-3.0.1.tar.gz sha256: 7b311a00503a863881eb8177dbeb84322f29399f3d7d72f3b1a4c9ba1d5794b4 diff --git a/packages/gdal/meta.yaml b/packages/gdal/meta.yaml index 9b8a0c01c55..08a0a87e4f7 100644 --- a/packages/gdal/meta.yaml +++ b/packages/gdal/meta.yaml @@ -3,6 +3,7 @@ package: version: 3.8.3 tag: - library + - shared_library source: url: https://github.com/OSGeo/gdal/releases/download/v3.8.3/gdal-3.8.3.tar.gz sha256: f7a30387a8239e9da26200f787a02136df2ee6473e86b36d05ad682761a049ea diff --git a/packages/geos/meta.yaml b/packages/geos/meta.yaml index 73a0ad44acf..a3d70337dcb 100644 --- a/packages/geos/meta.yaml +++ b/packages/geos/meta.yaml @@ -3,6 +3,7 @@ package: version: 3.12.1 tag: - library + - shared_library source: url: https://github.com/libgeos/geos/releases/download/3.12.1/geos-3.12.1.tar.bz2 sha256: d6ea7e492224b51193e8244fe3ec17c4d44d0777f3c32ca4fb171140549a0d03 diff --git a/packages/glpk/meta.yaml b/packages/glpk/meta.yaml index f2b567151fc..64d44134986 100644 --- a/packages/glpk/meta.yaml +++ b/packages/glpk/meta.yaml @@ -3,6 +3,7 @@ package: version: "5.0" tag: - library + - static_library source: sha256: 4a1013eebb50f728fc601bdd833b0b2870333c3b3e5a816eeba921d95bec6f15 url: https://ftp.gnu.org/gnu/glpk/glpk-5.0.tar.gz diff --git a/packages/libde265/meta.yaml b/packages/libde265/meta.yaml index 395ecc0f1dc..0e0d413f0dc 100644 --- a/packages/libde265/meta.yaml +++ b/packages/libde265/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.0.8 tag: - library + - static_library source: url: https://github.com/strukturag/libde265/releases/download/v1.0.8/libde265-1.0.8.tar.gz sha256: 24c791dd334fa521762320ff54f0febfd3c09fc978880a8c5fbc40a88f21d905 diff --git a/packages/libf2c/meta.yaml b/packages/libf2c/meta.yaml index 45858b5aef2..6ef46580e26 100644 --- a/packages/libf2c/meta.yaml +++ b/packages/libf2c/meta.yaml @@ -9,6 +9,7 @@ package: version: CLAPACK-3.2.1 tag: - library + - static_library source: sha256: 6dc4c382164beec8aaed8fd2acc36ad24232c406eda6db462bd4c41d5e455fac url: http://www.netlib.org/clapack/clapack.tgz diff --git a/packages/libgmp/meta.yaml b/packages/libgmp/meta.yaml index a6cb0cdcb62..64e06660aa0 100644 --- a/packages/libgmp/meta.yaml +++ b/packages/libgmp/meta.yaml @@ -3,6 +3,7 @@ package: version: 6.3.0 tag: - library + - static_library source: url: https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.xz sha256: a3c2b80201b89e68616f4ad30bc66aee4927c3ce50e33929ca819d5c43538898 diff --git a/packages/libgsl/meta.yaml b/packages/libgsl/meta.yaml index 35952c23b2f..c1ce049906f 100644 --- a/packages/libgsl/meta.yaml +++ b/packages/libgsl/meta.yaml @@ -3,6 +3,7 @@ package: version: "2.7" tag: - library + - static_library source: sha256: efbbf3785da0e53038be7907500628b466152dbc3c173a87de1b5eba2e23602b url: https://ftp.gnu.org/gnu/gsl/gsl-2.7.tar.gz diff --git a/packages/libhdf5/meta.yaml b/packages/libhdf5/meta.yaml index 5b8f87c6291..0fb10047c0f 100644 --- a/packages/libhdf5/meta.yaml +++ b/packages/libhdf5/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.12.1 tag: - library + - shared_library source: sha256: e6dde173c2d243551922d23a0387a79961205b018502e6a742acb30b61bc2d5f url: https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_12_1.tar.gz diff --git a/packages/libheif/meta.yaml b/packages/libheif/meta.yaml index 906ccd5339e..bcdb11fca86 100644 --- a/packages/libheif/meta.yaml +++ b/packages/libheif/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.12.0 tag: - library + - shared_library source: url: https://github.com/strukturag/libheif/releases/download/v1.12.0/libheif-1.12.0.tar.gz sha256: e1ac2abb354fdc8ccdca71363ebad7503ad731c84022cf460837f0839e171718 diff --git a/packages/libiconv/meta.yaml b/packages/libiconv/meta.yaml index b319c13cf26..9bb60cdc36e 100644 --- a/packages/libiconv/meta.yaml +++ b/packages/libiconv/meta.yaml @@ -3,6 +3,7 @@ package: version: "1.16" tag: - library + - static_library source: url: https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.16.tar.gz sha256: e6a1b1b589654277ee790cce3734f07876ac4ccfaecbee8afa0b649cf529cc04 diff --git a/packages/liblzma/meta.yaml b/packages/liblzma/meta.yaml index 8bb887091f7..83651a1b73d 100644 --- a/packages/liblzma/meta.yaml +++ b/packages/liblzma/meta.yaml @@ -3,6 +3,7 @@ package: version: 5.2.2 tag: - library + - static_library source: url: https://github.com/xz-mirror/xz/releases/download/v5.2.2/xz-5.2.2.tar.gz sha256: 73df4d5d34f0468bd57d09f2d8af363e95ed6cc3a4a86129d2f2c366259902a2 diff --git a/packages/libmagic/meta.yaml b/packages/libmagic/meta.yaml index 4740f44a9f8..f4993923937 100644 --- a/packages/libmagic/meta.yaml +++ b/packages/libmagic/meta.yaml @@ -3,6 +3,7 @@ package: version: "5.42" tag: - library + - shared_library source: url: https://github.com/file/file/archive/refs/tags/FILE5_42.tar.gz sha256: d7374d06451154a628831df58e835fa3263825d0ad593df0fb8a911418d27863 diff --git a/packages/libmpc/meta.yaml b/packages/libmpc/meta.yaml index 12ce61bfaaf..fa35e58daf2 100644 --- a/packages/libmpc/meta.yaml +++ b/packages/libmpc/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.3.1 tag: - library + - static_library source: url: https://ftp.gnu.org/gnu/mpc/mpc-1.3.1.tar.gz sha256: ab642492f5cf882b74aa0cb730cd410a81edcdbec895183ce930e706c1c759b8 diff --git a/packages/libmpfr/meta.yaml b/packages/libmpfr/meta.yaml index 4ffc50797af..58d3d5086a1 100644 --- a/packages/libmpfr/meta.yaml +++ b/packages/libmpfr/meta.yaml @@ -3,6 +3,7 @@ package: version: 4.2.1 tag: - library + - static_library source: url: https://ftp.gnu.org/gnu/mpfr/mpfr-4.2.1.tar.xz sha256: 277807353a6726978996945af13e52829e3abd7a9a5b7fb2793894e18f1fcbb2 diff --git a/packages/libnetcdf/meta.yaml b/packages/libnetcdf/meta.yaml index 8fa55f54298..f4d90ec7f6f 100644 --- a/packages/libnetcdf/meta.yaml +++ b/packages/libnetcdf/meta.yaml @@ -3,6 +3,7 @@ package: version: 4.9.2 tag: - library + - static_library source: sha256: bc104d101278c68b303359b3dc4192f81592ae8640f1aee486921138f7f88cb7 url: https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz diff --git a/packages/libproj/meta.yaml b/packages/libproj/meta.yaml index 0a80251025e..c25ccae5388 100644 --- a/packages/libproj/meta.yaml +++ b/packages/libproj/meta.yaml @@ -3,6 +3,7 @@ package: version: 9.3.1 tag: - library + - static_library source: sha256: b0f919cb9e1f42f803a3e616c2b63a78e4d81ecfaed80978d570d3a5e29d10bc url: https://download.osgeo.org/proj/proj-9.3.1.tar.gz diff --git a/packages/libtiff/meta.yaml b/packages/libtiff/meta.yaml index 6c738be7973..738bf0cb99a 100644 --- a/packages/libtiff/meta.yaml +++ b/packages/libtiff/meta.yaml @@ -3,6 +3,7 @@ package: version: 4.4.0 tag: - library + - static_library source: # TODO: The root certificate of `download.osgeo.org` has been expired and requires the latest version (2022.6.15) of certifi. # url: https://download.osgeo.org/libtiff/tiff-4.4.0.tar.gz diff --git a/packages/libwebp/meta.yaml b/packages/libwebp/meta.yaml index 4517b2c443b..4f02c4b271e 100644 --- a/packages/libwebp/meta.yaml +++ b/packages/libwebp/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.2.2 tag: - library + - static_library source: url: https://github.com/webmproject/libwebp/archive/refs/tags/v1.2.2.tar.gz sha256: 51e9297aadb7d9eb99129fe0050f53a11fcce38a0848fb2b0389e385ad93695e diff --git a/packages/libxml/meta.yaml b/packages/libxml/meta.yaml index 7a749c3d824..23f540e94a7 100644 --- a/packages/libxml/meta.yaml +++ b/packages/libxml/meta.yaml @@ -3,6 +3,7 @@ package: version: 2.9.10 tag: - library + - static_library source: sha256: aafee193ffb8fe0c82d4afef6ef91972cbaf5feea100edc2f262750611b4be1f url: http://xmlsoft.org/sources/libxml2-2.9.10.tar.gz diff --git a/packages/libxslt/meta.yaml b/packages/libxslt/meta.yaml index 6b5ccded197..b9d45b065dc 100644 --- a/packages/libxslt/meta.yaml +++ b/packages/libxslt/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.1.33 tag: - library + - static_library source: sha256: 8e36605144409df979cab43d835002f63988f3dc94d5d3537c12796db90e38c8 url: http://xmlsoft.org/sources/libxslt-1.1.33.tar.gz diff --git a/packages/libyaml/meta.yaml b/packages/libyaml/meta.yaml index f088e49c779..c920a4adf85 100644 --- a/packages/libyaml/meta.yaml +++ b/packages/libyaml/meta.yaml @@ -3,6 +3,7 @@ package: version: 0.2.1 tag: - library + - static_library source: url: https://github.com/yaml/libyaml/archive/0.2.1.zip sha256: 56070c9d4bf244a8dcc68e04613e5bbce5c8411ed97cdccc1f4b5fb46aebe5a8 diff --git a/packages/openblas/meta.yaml b/packages/openblas/meta.yaml index 1325649a512..ec80ac6585d 100644 --- a/packages/openblas/meta.yaml +++ b/packages/openblas/meta.yaml @@ -3,6 +3,7 @@ package: version: 0.3.26 tag: - library + - shared_library source: sha256: 4e6e4f5cb14c209262e33e6816d70221a2fe49eb69eaf0a06f065598ac602c68 url: https://github.com/OpenMathLib/OpenBLAS/releases/download/v0.3.26/OpenBLAS-0.3.26.tar.gz diff --git a/packages/openssl/meta.yaml b/packages/openssl/meta.yaml index b20c848554e..34956c2b92c 100644 --- a/packages/openssl/meta.yaml +++ b/packages/openssl/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.1.1w tag: - library + - shared_library source: url: https://www.openssl.org/source/openssl-1.1.1w.tar.gz sha256: cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 diff --git a/packages/ppl/meta.yaml b/packages/ppl/meta.yaml index df7edfd665d..e9627199177 100644 --- a/packages/ppl/meta.yaml +++ b/packages/ppl/meta.yaml @@ -3,6 +3,7 @@ package: version: "1.2" tag: - library + - static_library source: url: https://mirrors.mit.edu/sage/spkg/upstream/ppl/ppl-1.2.tar.bz2 sha256: 2d470b0c262904f190a19eac57fb5c2387b1bfc3510de25a08f3c958df62fdf1 diff --git a/packages/primecount/meta.yaml b/packages/primecount/meta.yaml index 0998da989d7..206e38d493b 100644 --- a/packages/primecount/meta.yaml +++ b/packages/primecount/meta.yaml @@ -3,6 +3,7 @@ package: version: "7.9" tag: - library + - static_library source: url: https://github.com/kimwalisch/primecount/archive/refs/tags/v7.9.tar.gz sha256: 872975ba2cbb43f5cc1ff5f5fda9ec4ec3f2be1eb3e3e906abe5d0b29a997f5b diff --git a/packages/primesieve/meta.yaml b/packages/primesieve/meta.yaml index 21a9c15905b..2f267252d50 100644 --- a/packages/primesieve/meta.yaml +++ b/packages/primesieve/meta.yaml @@ -3,6 +3,7 @@ package: version: "11.2" tag: - library + - static_library source: url: https://github.com/kimwalisch/primesieve/archive/refs/tags/v11.2.tar.gz sha256: 86c31bae9c378340b19669eafef8c5e45849adf7b9c92af1d212a2a2bfa0a5db diff --git a/packages/sharedlib-test/meta.yaml b/packages/sharedlib-test/meta.yaml index d7a645f2f41..89d8b914658 100644 --- a/packages/sharedlib-test/meta.yaml +++ b/packages/sharedlib-test/meta.yaml @@ -4,6 +4,7 @@ package: tag: - pyodide.test - library + - shared_library source: path: src diff --git a/packages/suitesparse/meta.yaml b/packages/suitesparse/meta.yaml index f2f73f52fa3..23ed88aea5d 100644 --- a/packages/suitesparse/meta.yaml +++ b/packages/suitesparse/meta.yaml @@ -3,6 +3,7 @@ package: version: 5.11.0 tag: - library + - shared_library source: sha256: fdd957ed06019465f7de73ce931afaf5d40e96e14ae57d91f60868b8c123c4c8 url: https://github.com/DrTimothyAldenDavis/SuiteSparse/archive/refs/tags/v5.11.0.tar.gz diff --git a/packages/zlib/meta.yaml b/packages/zlib/meta.yaml index a70bb77ab7a..c9cd5b7001e 100644 --- a/packages/zlib/meta.yaml +++ b/packages/zlib/meta.yaml @@ -3,6 +3,7 @@ package: version: 1.3.1 tag: - library + - static_library source: sha256: 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 url: https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz From f8f7f10ec8d545684ec503f6efbe2ef3747c0e47 Mon Sep 17 00:00:00 2001 From: Riya Sinha <33243383+riyavsinha@users.noreply.github.com> Date: Fri, 1 Nov 2024 03:13:17 -0700 Subject: [PATCH 11/29] Add `tiktoken` package (#5147) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/project/changelog.md | 1 + packages/tiktoken/meta.yaml | 23 +++++++++++++++++++++ packages/tiktoken/test_tiktoken.py | 32 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 packages/tiktoken/meta.yaml create mode 100644 packages/tiktoken/test_tiktoken.py diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 4cfb9317fb2..52a05d62838 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,7 @@ myst: ### Packages +- Added `tiktoken` v0.8.0 in {pr}`5147` - Upgraded `protobuf` to 5.28.3 {pr}`5136` - Upgraded `scikit-learn` to 1.5.2 {pr}`4823`, {pr}`5016`, {pr}`5072` - Upgraded `libcst` to 1.4.0 {pr}`4856` diff --git a/packages/tiktoken/meta.yaml b/packages/tiktoken/meta.yaml new file mode 100644 index 00000000000..ad14faf917d --- /dev/null +++ b/packages/tiktoken/meta.yaml @@ -0,0 +1,23 @@ +package: + name: tiktoken + version: 0.8.0 + top-level: + - tiktoken + - tiktoken_ext +source: + url: https://files.pythonhosted.org/packages/37/02/576ff3a6639e755c4f70997b2d315f56d6d71e0d046f4fb64cb81a3fb099/tiktoken-0.8.0.tar.gz + sha256: 9ccbb2740f24542534369c5635cfd9b2b3c2490754a78ac8831d99f89f94eeb2 +requirements: + executable: + - rustup + run: + - regex + - requests +about: + home: https://pypi.org/project/tiktoken + PyPI: https://pypi.org/project/tiktoken + summary: tiktoken is a fast BPE tokeniser for use with OpenAI's models + license: MIT +extra: + recipe-maintainers: + - riyavsinha diff --git a/packages/tiktoken/test_tiktoken.py b/packages/tiktoken/test_tiktoken.py new file mode 100644 index 00000000000..dcd9825516d --- /dev/null +++ b/packages/tiktoken/test_tiktoken.py @@ -0,0 +1,32 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["tiktoken"]) +def test_tiktoken(selenium): + import tiktoken + + # In the selenium environment, we cannot fetch a pre-trained tokenizer from tiktoken, so we create a basic one + def make_encoding(): + # This pattern splits on whitespace and punctuation + basic_pat_str = r"\w+|[^\w\s]+|\s" + basic_mergeable_ranks = { + b"Hello": 0, + b"world": 1, + b"!": 2, + b",": 3, + b"<|endoftext|>": 4, + b" ": 5, + } + basic_special_tokens = {"<|endoftext|>": 4} + return tiktoken.Encoding( + name="basic", + pat_str=basic_pat_str, + mergeable_ranks=basic_mergeable_ranks, + special_tokens=basic_special_tokens, + ) + + encoding = make_encoding() + test_str = "Hello, world!" + encoded = encoding.encode(test_str) + decoded = encoding.decode(encoded) + assert decoded == test_str From 4227cb5f7587ca1a9aef8e0675a05f419c1597f0 Mon Sep 17 00:00:00 2001 From: Riya Sinha <33243383+riyavsinha@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:36:58 -0700 Subject: [PATCH 12/29] Upgrade msgpack to 1.1.0 (#5144) --- docs/project/changelog.md | 1 + packages/msgpack/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 52a05d62838..6c711d55822 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,7 @@ myst: ### Packages +- Upgraded `msgpack` to 1.1.0 {pr}`5144` - Added `tiktoken` v0.8.0 in {pr}`5147` - Upgraded `protobuf` to 5.28.3 {pr}`5136` - Upgraded `scikit-learn` to 1.5.2 {pr}`4823`, {pr}`5016`, {pr}`5072` diff --git a/packages/msgpack/meta.yaml b/packages/msgpack/meta.yaml index 017d086aaef..e0e63ef45b8 100644 --- a/packages/msgpack/meta.yaml +++ b/packages/msgpack/meta.yaml @@ -1,11 +1,11 @@ package: name: msgpack - version: 1.0.8 + version: 1.1.0 top-level: - msgpack source: - sha256: 95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3 - url: https://files.pythonhosted.org/packages/08/4c/17adf86a8fbb02c144c7569dc4919483c01a2ac270307e2d59e1ce394087/msgpack-1.0.8.tar.gz + sha256: dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e + url: https://files.pythonhosted.org/packages/cb/d0/7555686ae7ff5731205df1012ede15dd9d927f6227ea151e901c7406af4f/msgpack-1.1.0.tar.gz about: home: https://msgpack.org/ PyPI: https://pypi.org/project/msgpack From 59da8bc5754c61d776dc8bcb0f88d885de78c643 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Sun, 3 Nov 2024 08:56:55 +0100 Subject: [PATCH 13/29] Bump to DuckDB 1.1.2 (#5142) Co-authored-by: Gyeongjae Choi --- docs/project/changelog.md | 1 + packages/duckdb/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 6c711d55822..92275dc488f 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -85,6 +85,7 @@ myst: - Upgraded `sympy` to 1.13.3 {pr}`5098` - Upgraded `tree-sitter` to 0.23.1 {pr}`5110` - Upgraded `PyYAML` to 6.0.2 {pr}`5137` +- Upgraded `duckdb` to 1.1.2 {pr}`5142` - Added `casadi` 3.6.6 {pr}`4936`, {pr}`5057` - Added `pyarrow` 17.0.0 {pr}`4950` - Added `rasterio` 1.13.10, `affine` 2.4.0 {pr}`4983` diff --git a/packages/duckdb/meta.yaml b/packages/duckdb/meta.yaml index ac78b3edde2..80f473c32d6 100644 --- a/packages/duckdb/meta.yaml +++ b/packages/duckdb/meta.yaml @@ -1,11 +1,11 @@ package: name: duckdb - version: 1.1.0 + version: 1.1.2 top-level: - duckdb source: - url: https://duckdb.github.io/duckdb-pyodide/wheels/duckdb-1.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl - sha256: 5f6dffea55b467b41b329e8117120403cbd63fd8096a046daa740dfc94dfcb66 + url: https://duckdb.github.io/duckdb-pyodide/wheels/duckdb-1.1.2-cp312-cp312-pyodide_2024_0_wasm32.whl + sha256: aa794366f7acb924ddd1ecb32a349148a3c1343219a7d9d84812d3dcf47a4404 about: home: https://github.com/duckdb/duckdb PyPI: https://pypi.org/project/duckdb From 330afe4a8b8697a9fcfeb1c1e63c8007e9c93e35 Mon Sep 17 00:00:00 2001 From: Riya Sinha <33243383+riyavsinha@users.noreply.github.com> Date: Mon, 4 Nov 2024 00:24:10 -0800 Subject: [PATCH 14/29] Upgrade `pydantic` and `pydantic_core` (#5151) --- docs/project/changelog.md | 2 ++ packages/pydantic/meta.yaml | 6 +++--- packages/pydantic_core/meta.yaml | 9 +++------ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 92275dc488f..1d98f04a6d2 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,8 @@ myst: ### Packages +- Upgraded `pydantic_core` to 2.25.1 {pr}`5151` +- Upgraded `pydantic` to 2.9.2 {pr}`5151` - Upgraded `msgpack` to 1.1.0 {pr}`5144` - Added `tiktoken` v0.8.0 in {pr}`5147` - Upgraded `protobuf` to 5.28.3 {pr}`5136` diff --git a/packages/pydantic/meta.yaml b/packages/pydantic/meta.yaml index 19a25c89dfc..4201d6e8b9f 100644 --- a/packages/pydantic/meta.yaml +++ b/packages/pydantic/meta.yaml @@ -1,11 +1,11 @@ package: name: pydantic - version: 2.8.2 + version: 2.9.2 top-level: - pydantic source: - sha256: 6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a - url: https://files.pythonhosted.org/packages/8c/99/d0a5dca411e0a017762258013ba9905cd6e7baa9a3fd1fe8b6529472902e/pydantic-2.8.2.tar.gz + sha256: d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f + url: https://files.pythonhosted.org/packages/a9/b7/d9e3f12af310e1120c21603644a1cd86f59060e040ec5c3a80b8f05fae30/pydantic-2.9.2.tar.gz requirements: run: - typing-extensions diff --git a/packages/pydantic_core/meta.yaml b/packages/pydantic_core/meta.yaml index 2f9e932ee8c..1070719ef73 100644 --- a/packages/pydantic_core/meta.yaml +++ b/packages/pydantic_core/meta.yaml @@ -1,14 +1,11 @@ package: name: pydantic_core - version: 2.23.1 + version: 2.25.1 top-level: - pydantic_core source: - url: https://files.pythonhosted.org/packages/60/a9/a64afaeecc30a42142f5e60bb61a1ec4817e90c2d1c0c7b242082f61ed00/pydantic_core-2.23.1.tar.gz - sha256: 12543919dbf3021e17b9e2cd5cd1b570c6585794fa487339b5b95564b9689452 -requirements: - executable: - - rustup + url: https://github.com/pydantic/pydantic-core/releases/download/v2.25.1/pydantic_core-2.25.1-cp312-cp312-emscripten_3_1_58_wasm32.whl + sha256: f4b77016b1511e54a8c6bd833bb68e190b96989ba9997f9e0f023cfb0f8a16c2 about: home: https://github.com/pydantic/pydantic-core PyPI: https://pypi.org/project/pydantic_core From 4f7e0df9f6763e594b35617b3cc1f5645f6d607c Mon Sep 17 00:00:00 2001 From: Yan Wong Date: Mon, 4 Nov 2024 22:37:21 +0000 Subject: [PATCH 15/29] Update tskit to 0.6.0 (#5157) --- docs/project/changelog.md | 1 + packages/tskit/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 1d98f04a6d2..e6d7031e5fb 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,7 @@ myst: ### Packages +- Upgraded `tskit` to 0.6.0 {pr}`5157` - Upgraded `pydantic_core` to 2.25.1 {pr}`5151` - Upgraded `pydantic` to 2.9.2 {pr}`5151` - Upgraded `msgpack` to 1.1.0 {pr}`5144` diff --git a/packages/tskit/meta.yaml b/packages/tskit/meta.yaml index 2e865c13f2e..b68dd79c0cd 100644 --- a/packages/tskit/meta.yaml +++ b/packages/tskit/meta.yaml @@ -1,11 +1,11 @@ package: name: tskit - version: 0.5.6 + version: 0.6.0 top-level: - tskit source: - url: https://files.pythonhosted.org/packages/aa/cc/6bdd65802bda99d758ced40c0204885d7c3ac8667e24be125fc72660c5bf/tskit-0.5.6.tar.gz - sha256: ddfe213f1cb063cdb6982177230a2805ecd7dfc7ccd73026e13878abffd2ce46 + url: https://files.pythonhosted.org/packages/01/dc/6962a28d6e2eac38f088fced8b0e62fff683bd5fa7e6c25bd38306ea6433/tskit-0.6.0.tar.gz + sha256: 9dda2acc84ae8dab3484dfedcb721807a4efe48e159f74f24b07789404b54106 requirements: run: - numpy From a5386c86b86135bda9c6f5d2d4f214726bacb7af Mon Sep 17 00:00:00 2001 From: swnf <50806201+swnf@users.noreply.github.com> Date: Tue, 5 Nov 2024 01:26:30 +0100 Subject: [PATCH 16/29] Add `soxr` to packages (#5150) --- docs/development/new-packages.md | 2 +- docs/project/changelog.md | 3 +- packages/soxr/meta.yaml | 25 +++++ .../0001-Fix-function-signatures.patch | 99 +++++++++++++++++++ packages/soxr/test_soxr.py | 24 +++++ 5 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 packages/soxr/meta.yaml create mode 100644 packages/soxr/patches/0001-Fix-function-signatures.patch create mode 100644 packages/soxr/test_soxr.py diff --git a/docs/development/new-packages.md b/docs/development/new-packages.md index f2983bc93dd..019171e6983 100644 --- a/docs/development/new-packages.md +++ b/docs/development/new-packages.md @@ -196,7 +196,7 @@ from pytest_pyodide import run_in_pyodide @run_in_pyodide(packages=["-tests", "pytest"]) def test_mytestname(selenium): import pytest - pytest.main(["--pyargs", "", "-k", "some_filter", ...]) + assert pytest.main(["--pyargs", "", "-k", "some_filter", ...]) == 0 ``` you can put whatever command line arguments you would pass to `pytest` as diff --git a/docs/project/changelog.md b/docs/project/changelog.md index e6d7031e5fb..1983c0829a9 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -68,7 +68,6 @@ myst: - Upgraded `pydantic_core` to 2.25.1 {pr}`5151` - Upgraded `pydantic` to 2.9.2 {pr}`5151` - Upgraded `msgpack` to 1.1.0 {pr}`5144` -- Added `tiktoken` v0.8.0 in {pr}`5147` - Upgraded `protobuf` to 5.28.3 {pr}`5136` - Upgraded `scikit-learn` to 1.5.2 {pr}`4823`, {pr}`5016`, {pr}`5072` - Upgraded `libcst` to 1.4.0 {pr}`4856` @@ -89,6 +88,8 @@ myst: - Upgraded `tree-sitter` to 0.23.1 {pr}`5110` - Upgraded `PyYAML` to 6.0.2 {pr}`5137` - Upgraded `duckdb` to 1.1.2 {pr}`5142` +- Added `soxr` 0.5.0.post1 {pr}`5150` +- Added `tiktoken` v0.8.0 in {pr}`5147` - Added `casadi` 3.6.6 {pr}`4936`, {pr}`5057` - Added `pyarrow` 17.0.0 {pr}`4950` - Added `rasterio` 1.13.10, `affine` 2.4.0 {pr}`4983` diff --git a/packages/soxr/meta.yaml b/packages/soxr/meta.yaml new file mode 100644 index 00000000000..3eabecdbc97 --- /dev/null +++ b/packages/soxr/meta.yaml @@ -0,0 +1,25 @@ +package: + name: soxr + version: 0.5.0.post1 + top-level: + - soxr +source: + url: https://files.pythonhosted.org/packages/02/c0/4429bf9b3be10e749149e286aa5c53775399ec62891c6b970456c6dca325/soxr-0.5.0.post1.tar.gz + sha256: 7092b9f3e8a416044e1fa138c8172520757179763b85dc53aa9504f4813cff73 + patches: + # Can be removed once the PR chirlu/soxr#13 (dead upstream) or dofuuz/soxr#3 (python soxr fork) is merged and released + - patches/0001-Fix-function-signatures.patch +requirements: + run: + - numpy +about: + home: https://github.com/dofuuz/python-soxr + PyPI: https://pypi.org/project/soxr + summary: High quality, one-dimensional sample-rate conversion library + license: LGPLv2+ +test: + imports: + - soxr +extra: + recipe-maintainers: + - swnf diff --git a/packages/soxr/patches/0001-Fix-function-signatures.patch b/packages/soxr/patches/0001-Fix-function-signatures.patch new file mode 100644 index 00000000000..37610333184 --- /dev/null +++ b/packages/soxr/patches/0001-Fix-function-signatures.patch @@ -0,0 +1,99 @@ +From 983ea3d49b2834e044ca5b492cfb80181e49666f Mon Sep 17 00:00:00 2001 +From: swnf <50806201+swnf@users.noreply.github.com> +Date: Sun, 3 Nov 2024 10:36:47 +0100 +Subject: [PATCH] Fix function signature mismatches + + +diff --git a/libsoxr/src/fft4g32.c b/libsoxr/src/fft4g32.c +index 7a31ba4..31aebca 100644 +--- a/libsoxr/src/fft4g32.c ++++ b/libsoxr/src/fft4g32.c +@@ -9,11 +9,12 @@ + + #if WITH_CR32 + #include "rdft_t.h" +-static void * null(void) {return 0;} +-static void forward (int length, void * setup, double * H) {lsx_safe_rdft_f(length, 1, H); (void)setup;} +-static void backward(int length, void * setup, double * H) {lsx_safe_rdft_f(length, -1, H); (void)setup;} ++static void * null(int length) {return 0; (void)length;} ++static void nothing(void * setup) {(void)setup;} ++static void forward(int length, void * setup, void * H, void * scratch) {lsx_safe_rdft_f(length, 1, (double * ) H); (void)setup; (void)scratch;} ++static void backward(int length, void * setup, void * H, void * scratch) {lsx_safe_rdft_f(length, -1, (double * ) H); (void)setup; (void)scratch;} + static int multiplier(void) {return 2;} +-static void nothing(void) {} ++static void nothing2(int length, void * setup, void * H, void * scratch) {(void)length; (void)setup; (void)H; (void)scratch;} + static int flags(void) {return 0;} + + fn_t _soxr_rdft32_cb[] = { +@@ -27,7 +28,7 @@ fn_t _soxr_rdft32_cb[] = { + (fn_t)_soxr_ordered_convolve_f, + (fn_t)_soxr_ordered_partial_convolve_f, + (fn_t)multiplier, +- (fn_t)nothing, ++ (fn_t)nothing2, + (fn_t)malloc, + (fn_t)calloc, + (fn_t)free, +diff --git a/libsoxr/src/fft4g32s.c b/libsoxr/src/fft4g32s.c +index 8ce9726..f00c97c 100644 +--- a/libsoxr/src/fft4g32s.c ++++ b/libsoxr/src/fft4g32s.c +@@ -5,11 +5,12 @@ + #include "util32s.h" + #include "rdft_t.h" + +-static void * null(void) {return 0;} +-static void nothing(void) {} +-static void forward (int length, void * setup, float * H) {lsx_safe_rdft_f(length, 1, H); (void)setup;} +-static void backward(int length, void * setup, float * H) {lsx_safe_rdft_f(length, -1, H); (void)setup;} ++static void * null(int length) {return 0; (void)length;} ++static void nothing(void * setup) {(void)setup;} ++static void forward (int length, void * setup, void * H, void * scratch) {lsx_safe_rdft_f(length, 1, (float*)H); (void)setup; (void)scratch;} ++static void backward(int length, void * setup, void * H, void * scratch) {lsx_safe_rdft_f(length, -1, (float*)H); (void)setup; (void)scratch;} + static int multiplier(void) {return 2;} ++static void nothing2(int length, void * setup, void * H, void * scratch) {(void)length; (void)setup; (void)H; (void)scratch;} + static int flags(void) {return RDFT_IS_SIMD;} + + fn_t _soxr_rdft32s_cb[] = { +@@ -23,7 +24,7 @@ fn_t _soxr_rdft32s_cb[] = { + (fn_t)ORDERED_CONVOLVE_SIMD, + (fn_t)ORDERED_PARTIAL_CONVOLVE_SIMD, + (fn_t)multiplier, +- (fn_t)nothing, ++ (fn_t)nothing2, + (fn_t)SIMD_ALIGNED_MALLOC, + (fn_t)SIMD_ALIGNED_CALLOC, + (fn_t)SIMD_ALIGNED_FREE, +diff --git a/libsoxr/src/fft4g64.c b/libsoxr/src/fft4g64.c +index 0018516..5798fd7 100644 +--- a/libsoxr/src/fft4g64.c ++++ b/libsoxr/src/fft4g64.c +@@ -7,11 +7,12 @@ + #include "soxr-config.h" + + #if WITH_CR64 +-static void * null(void) {return 0;} +-static void nothing(void) {} +-static void forward (int length, void * setup, double * H) {lsx_safe_rdft(length, 1, H); (void)setup;} +-static void backward(int length, void * setup, double * H) {lsx_safe_rdft(length, -1, H); (void)setup;} ++static void * null(int length) {return 0; (void)length;} ++static void nothing(void * setup) {(void)setup;} ++static void forward (int length, void * setup, void * H, void * scratch) {lsx_safe_rdft(length, 1, (double*)H); (void)setup; (void)scratch;} ++static void backward(int length, void * setup, void * H, void * scratch) {lsx_safe_rdft(length, -1, (double*)H); (void)setup; (void)scratch;} + static int multiplier(void) {return 2;} ++static void nothing2(int length, void * setup, void * H, void * scratch) {(void)length; (void)setup; (void)H; (void)scratch;} + static int flags(void) {return 0;} + + typedef void (* fn_t)(void); +@@ -26,7 +27,7 @@ fn_t _soxr_rdft64_cb[] = { + (fn_t)_soxr_ordered_convolve, + (fn_t)_soxr_ordered_partial_convolve, + (fn_t)multiplier, +- (fn_t)nothing, ++ (fn_t)nothing2, + (fn_t)malloc, + (fn_t)calloc, + (fn_t)free, +-- +2.43.0 + diff --git a/packages/soxr/test_soxr.py b/packages/soxr/test_soxr.py new file mode 100644 index 00000000000..9c8a667ec17 --- /dev/null +++ b/packages/soxr/test_soxr.py @@ -0,0 +1,24 @@ +import pytest +from pytest_pyodide import run_in_pyodide + + +@pytest.mark.parametrize("input_sr, output_sr", [(44100, 22050), (22050, 44100)]) +@run_in_pyodide(packages=["soxr", "numpy"]) +def test_resample(selenium, input_sr, output_sr): + import numpy as np + import soxr + + # Signal length in seconds + length = 5.0 + # Frequency in Hz + frequency = 42 + + input_sample_positions = np.arange(0, length, 1 / input_sr) + output_sample_positions = np.arange(0, length, 1 / output_sr) + + input_signal = np.sin(2 * np.pi * frequency * input_sample_positions) + predicted_output_signal = np.sin(2 * np.pi * frequency * output_sample_positions) + + output_signal = soxr.resample(input_signal, input_sr, output_sr) + + np.testing.assert_allclose(predicted_output_signal, output_signal, atol=0.0015) From 0adc6d050ac2513396cdd0eade2bd4b01849370d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 7 Nov 2024 07:54:12 +0100 Subject: [PATCH 17/29] bump_version.py: `ast.Str` will be removed in Py314 so use `ast.Constant` (#5164) --- tools/bump_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bump_version.py b/tools/bump_version.py index 1e5c81cab41..e5cec786793 100755 --- a/tools/bump_version.py +++ b/tools/bump_version.py @@ -6,7 +6,7 @@ import itertools import pathlib import re -from ast import Str +from ast import Constant from collections import namedtuple from collections.abc import Callable @@ -81,7 +81,7 @@ def build_version_pattern(pattern): @functools.lru_cache -def python_version_to_js_version(version: str) -> Str: +def python_version_to_js_version(version: str) -> Constant: """ Convert Python version name to JS version name These two are different in prerelease or dev versions. From ce29b59902dd776aa62584d0c3de345370ee8e9c Mon Sep 17 00:00:00 2001 From: Joel Ostblom Date: Thu, 7 Nov 2024 02:04:54 -0800 Subject: [PATCH 18/29] Update altair to 5.4.1 (#5124) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/project/changelog.md | 1 + packages/altair/meta.yaml | 10 ++++------ packages/altair/test_altair.py | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 1983c0829a9..971f898a208 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -86,6 +86,7 @@ myst: - Upgraded `duckdb` to 1.1.0 {pr}`5078` - Upgraded `sympy` to 1.13.3 {pr}`5098` - Upgraded `tree-sitter` to 0.23.1 {pr}`5110` +- Upgraded `altair` to 5.4.1 {pr}`5124` - Upgraded `PyYAML` to 6.0.2 {pr}`5137` - Upgraded `duckdb` to 1.1.2 {pr}`5142` - Added `soxr` 0.5.0.post1 {pr}`5150` diff --git a/packages/altair/meta.yaml b/packages/altair/meta.yaml index ff8b2ef53c0..6cf5d16dae5 100644 --- a/packages/altair/meta.yaml +++ b/packages/altair/meta.yaml @@ -1,11 +1,11 @@ package: name: altair - version: 5.3.0 + version: 5.4.1 top-level: - altair source: - url: https://files.pythonhosted.org/packages/46/30/2118537233fa72c1d91a81f5908a7e843a6601ccc68b76838ebc4951505f/altair-5.3.0-py3-none-any.whl - sha256: 7084a1dab4d83c5e7e5246b92dc1b4451a6c68fd057f3716ee9d315c8980e59a + url: https://files.pythonhosted.org/packages/9b/52/4a86a4fa1cc2aae79137cc9510b7080c3e5aede2310d14fae5486feec7f7/altair-5.4.1-py3-none-any.whl + sha256: 0fb130b8297a569d08991fb6fe763582e7569f8a04643bbd9212436e3be04aef about: home: https://altair-viz.github.io/ PyPI: https://pypi.org/project/altair @@ -19,7 +19,5 @@ requirements: - typing-extensions - Jinja2 - jsonschema - - numpy - - pandas - - toolz - packaging + - narwhals diff --git a/packages/altair/test_altair.py b/packages/altair/test_altair.py index 676603bb207..5159fa91c9e 100644 --- a/packages/altair/test_altair.py +++ b/packages/altair/test_altair.py @@ -6,17 +6,19 @@ @run_in_pyodide(packages=["altair"]) def test_altair(selenium): import altair as alt - import pandas as pd - data = pd.DataFrame( - { - "a": ["A", "B", "C", "D", "E", "F", "G", "H", "I"], - "b": [28, 55, 43, 91, 81, 53, 19, 87, 52], - } + data = alt.Data( + values=[ + {"a": "A", "b": 5}, + {"a": "B", "b": 3}, + {"a": "C", "b": 6}, + {"a": "D", "b": 7}, + {"a": "E", "b": 2}, + ] ) - c = alt.Chart(data).mark_bar().encode(x="a", y="b").to_dict() + c = alt.Chart(data).mark_bar().encode(x="a:N", y="b:Q").to_dict() assert c["mark"]["type"] == "bar" assert c["encoding"]["x"]["field"] == "a" assert c["encoding"]["y"]["type"] == "quantitative" - assert "name" in c["data"] + assert "values" in c["data"] From 782e8c6586433bff5ba03fd3b9563256716337fb Mon Sep 17 00:00:00 2001 From: Yan Wong Date: Thu, 7 Nov 2024 10:05:08 +0000 Subject: [PATCH 19/29] Update msprime to 1.3.3 (#5159) --- docs/project/changelog.md | 1 + packages/msprime/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 971f898a208..b66eb9a9f5e 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,7 @@ myst: ### Packages +- Upgraded `msprime` to 1.3.3 {pr}`5159` - Upgraded `tskit` to 0.6.0 {pr}`5157` - Upgraded `pydantic_core` to 2.25.1 {pr}`5151` - Upgraded `pydantic` to 2.9.2 {pr}`5151` diff --git a/packages/msprime/meta.yaml b/packages/msprime/meta.yaml index ae16db491d0..35940636089 100644 --- a/packages/msprime/meta.yaml +++ b/packages/msprime/meta.yaml @@ -1,11 +1,11 @@ package: name: msprime - version: 1.3.1 + version: 1.3.3 top-level: - msprime source: - url: https://files.pythonhosted.org/packages/9c/c6/d2c24086da4492fe2c9d281d237b5fd09e5bf5c92a8956efad7522cf96b1/msprime-1.3.1.tar.gz - sha256: b3f62cd516ad2e43c8412ea1f242acad1bc939391cfe9caa19625874e2e34835 + url: https://files.pythonhosted.org/packages/70/84/72a914d91c3e9cea91b8089ddc2a32cc9bdee50eda7e8900be6be70d1fdb/msprime-1.3.3.tar.gz + sha256: d8ae798076167f632b8fd7eccd5089faaa84c9034ca69422f886425f94a5b01b build: script: | export LIBGSL_INCLUDE_PATH=$(pkg-config --cflags-only-I --dont-define-prefix gsl) From 1c82138a272e77d5aae420adbc522f11e1fe4608 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 7 Nov 2024 21:04:14 +0900 Subject: [PATCH 20/29] Use pyodide-build submodule (#5152) Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- .circleci/config.yml | 10 ++++ .github/workflows/main.yml | 11 +++- .../workflows/update_cross_build_releases.yml | 2 + .gitmodules | 3 + Makefile | 4 +- Makefile.envs | 9 --- docs/development/contributing.md | 2 +- docs/development/maintainers.md | 23 +++++++- docs/development/new-packages.md | 2 +- pyodide-build | 1 + src/tests/test_cmdline_runner.py | 2 +- tools/check_and_install_pyodide_build.py | 59 ------------------- 12 files changed, 50 insertions(+), 78 deletions(-) create mode 100644 .gitmodules create mode 160000 pyodide-build delete mode 100755 tools/check_and_install_pyodide_build.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d98476b2a0..6a72acdc310 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,8 @@ jobs: - DISABLE_DYLINK: << parameters.disable_dylink >> steps: - checkout + - run: git submodule sync + - run: git submodule update --init - restore_cache: keys: @@ -132,6 +134,8 @@ jobs: resource_class: large steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -190,6 +194,8 @@ jobs: resource_class: large steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -219,6 +225,8 @@ jobs: <<: *defaults steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -246,6 +254,8 @@ jobs: <<: *defaults steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2327f9dc549..fcbb88f8d9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,6 +27,8 @@ jobs: needs: get_python_version steps: - uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ needs.get_python_version.outputs.PYVERSION }} @@ -34,8 +36,8 @@ jobs: shell: bash -l {0} run: | mkdir test-results - python3 -m pip install -r requirements.txt -r requirements-deploy.txt make pyodide_build + python3 -m pip install -r requirements.txt -r requirements-deploy.txt - name: Run tests shell: bash -l {0} run: | @@ -43,7 +45,7 @@ jobs: --junitxml=test-results/junit.xml \ --verbose \ --runtime=host \ - --cov=pyodide_build --cov=pyodide \ + --cov=pyodide \ src packages/_tests tools/ - uses: codecov/codecov-action@v4 with: @@ -68,7 +70,8 @@ jobs: steps: - uses: actions/checkout@v4 - + with: + submodules: recursive - name: Cache ccache output uses: actions/cache@v4 with: @@ -169,6 +172,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Download build artifact uses: actions/download-artifact@v4 diff --git a/.github/workflows/update_cross_build_releases.yml b/.github/workflows/update_cross_build_releases.yml index bcfbda5f1e3..4d75d8a03d7 100644 --- a/.github/workflows/update_cross_build_releases.yml +++ b/.github/workflows/update_cross_build_releases.yml @@ -15,6 +15,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..b32d197f231 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pyodide-build"] + path = pyodide-build + url = https://github.com/pyodide/pyodide-build diff --git a/Makefile b/Makefile index 218619f2cc1..f4f10870b40 100644 --- a/Makefile +++ b/Makefile @@ -225,8 +225,8 @@ $(eval $(call preprocess-js,js2python.js)) .PHONY: pyodide_build pyodide_build: - @echo "Ensuring required pyodide-build version is installed" - ./tools/check_and_install_pyodide_build.py "$(PYODIDE_BUILD_COMMIT)" --repo "$(PYODIDE_BUILD_REPO)" + @echo "Ensuring pyodide-build is installed" + pip install -e ./pyodide-build @which pyodide >/dev/null diff --git a/Makefile.envs b/Makefile.envs index 2b626695d60..b48c6b3529b 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -5,15 +5,6 @@ export PYODIDE_ABI_VERSION ?= 2024_0 export PYTHON_ARCHIVE_SHA256=d01ec6a33bc10009b09c17da95cc2759af5a580a7316b3a446eb4190e13f97b2 -# Update the following variables when you want to update the version of the -# pyodide-build version used in the build process. If you want to make breaking -# changes in pyodide-build, it is also useful to change the repository URL -# to your fork to test the changes are working as expected. - -# v0.27.3 -export PYODIDE_BUILD_COMMIT=fac0109aa2acf14469320b049d710dd42639bf94 -export PYODIDE_BUILD_REPO=https://github.com/pyodide/pyodide-build - ifdef CPYTHON_DEBUG export CPYTHON_ABI_FLAGS=d endif diff --git a/docs/development/contributing.md b/docs/development/contributing.md index 92283fc807a..b1c7529304c 100644 --- a/docs/development/contributing.md +++ b/docs/development/contributing.md @@ -29,7 +29,7 @@ To contribute code, see the following steps, 3. Clone your fork of Pyodide ``` - git clone https://github.com//pyodide.git + git clone --recursive https://github.com//pyodide.git ``` and add the upstream remote, ``` diff --git a/docs/development/maintainers.md b/docs/development/maintainers.md index f4caa870bf2..0d0e2033d27 100644 --- a/docs/development/maintainers.md +++ b/docs/development/maintainers.md @@ -193,8 +193,27 @@ note to update them independently. ## Updating pyodide-build -to change the version of pyodide-build, update the PYODIDE_BUILD_REPO and PYODIDE_BUILD_COMMIT -variables in `Makefile.envs`. +to change the version of pyodide-build, change the commit of the pyodide-build submodule. + +```bash +cd pyodide-build +git checkout "" +``` + +to test with the fork of pyodide-build, change the `.gitmodules` file to point to your fork and update the commit hash + +```ini +# .gitmodules +[submodule "pyodide-build"] + path = pyodide-build + url = https://github.com//pyodide-build +``` + +```bash +git submodule sync +cd pyodide-build +git checkout " str | None: - """ - Return the version of the pyodide-build package or the URL to the repository. - """ - freeze_result = sp.check_output( - [ - sys.executable, - "-m", - "pip", - "freeze", - ] - ) - - for line in freeze_result.decode().split("\n"): - if line.startswith("pyodide-build"): - try: - return line.split(" @ ")[1] - except IndexError: - print("pyodide-build is not installed from a VCS: ", line) - return None - - return None - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("commit", type=str) - parser.add_argument( - "--repo", type=str, default="https://github.com/pyodide/pyodide-build" - ) - - return parser.parse_args() - - -def main(): - args = parse_args() - install_url = f"git+{args.repo}@{args.commit}" - installed_url = get_pyodide_build_install_url() - - if not installed_url or installed_url != install_url: - sp.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - install_url, - ] - ) - - -if __name__ == "__main__": - main() From 24dfc07a4ecdcc702ea60d0bd3136e7e6b5ce06e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:32:39 +0800 Subject: [PATCH 21/29] Add missing Pyodide v0.26.3 cross-build environment metadata (#5168) --- pyodide-cross-build-environments.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyodide-cross-build-environments.json b/pyodide-cross-build-environments.json index f60588ccdbb..d69c3d5437e 100644 --- a/pyodide-cross-build-environments.json +++ b/pyodide-cross-build-environments.json @@ -8,6 +8,14 @@ "emscripten_version": "3.1.58", "min_pyodide_build_version": "0.26.0" }, + "0.26.3": { + "version": "0.26.3", + "url": "https://github.com/pyodide/pyodide/releases/download/0.26.3/xbuildenv-0.26.3.tar.bz2", + "sha256": "538b00bb7b329f96e6f981464bc7875109b9779f8d121e2140235c375f530675", + "python_version": "3.12.1", + "emscripten_version": "3.1.58", + "min_pyodide_build_version": "0.26.0" + }, "0.26.2": { "version": "0.26.2", "url": "https://github.com/pyodide/pyodide/releases/download/0.26.2/xbuildenv-0.26.2.tar.bz2", From 651b690a6c42184d8f910784dc35afcad34ab7c8 Mon Sep 17 00:00:00 2001 From: Hanno Rein Date: Thu, 7 Nov 2024 13:13:08 -0500 Subject: [PATCH 22/29] Bump rebound/reboundx to version 4.4.3/4.3.0 (#5163) --- docs/project/changelog.md | 2 + packages/rebound/meta.yaml | 8 ++- .../patches/0001-fix-install_name.patch | 38 ++++++++----- .../rebound/patches/0002-fix-output.patch | 57 +++++++++++++++++++ .../rebound/patches/0003-no-emscripten.patch | 49 ++++++++++++++++ packages/reboundx/meta.yaml | 6 +- 6 files changed, 141 insertions(+), 19 deletions(-) create mode 100644 packages/rebound/patches/0002-fix-output.patch create mode 100644 packages/rebound/patches/0003-no-emscripten.patch diff --git a/docs/project/changelog.md b/docs/project/changelog.md index b66eb9a9f5e..7f1e704f9bb 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,8 @@ myst: ### Packages +- Upgraded `rebound` to 4.4.3 {pr}`5163` +- Upgraded `reboundx` to 4.3.0 {pr}`5163` - Upgraded `msprime` to 1.3.3 {pr}`5159` - Upgraded `tskit` to 0.6.0 {pr}`5157` - Upgraded `pydantic_core` to 2.25.1 {pr}`5151` diff --git a/packages/rebound/meta.yaml b/packages/rebound/meta.yaml index a850257d793..717fad1f151 100644 --- a/packages/rebound/meta.yaml +++ b/packages/rebound/meta.yaml @@ -1,6 +1,6 @@ package: name: rebound - version: 3.24.2 + version: 4.4.3 top-level: - rebound build: @@ -12,10 +12,12 @@ build: cp librebound.cpython-*.so $WASM_LIBRARY_DIR/lib/ source: - url: https://files.pythonhosted.org/packages/5c/66/7564ac591bb088d7f35a59d560b5c26426bb8ef5415522232819fe11c45a/rebound-3.24.2.tar.gz - sha256: a3c5d4c6a10b9c1538e1051edcd6341ef3d13142b698ee9c4a1f112d9684d803 + url: https://files.pythonhosted.org/packages/9d/f5/93f5abaf2c796e35b775e6fd395c2fd3729c17701fcb90fb4ed052fdd465/rebound-4.4.3.tar.gz + sha256: 8dd2fc69f4db4609abeb50048583d0428c716cd3eab11df868358f5384d826cf patches: - patches/0001-fix-install_name.patch + - patches/0002-fix-output.patch + - patches/0003-no-emscripten.patch requirements: run: diff --git a/packages/rebound/patches/0001-fix-install_name.patch b/packages/rebound/patches/0001-fix-install_name.patch index 5cedb0b5d69..0e88a8e6bab 100644 --- a/packages/rebound/patches/0001-fix-install_name.patch +++ b/packages/rebound/patches/0001-fix-install_name.patch @@ -1,15 +1,27 @@ -Index: rebound-3.24.2/setup.py -=================================================================== ---- rebound-3.24.2.orig/setup.py -+++ rebound-3.24.2/setup.py -@@ -23,8 +23,8 @@ extra_link_args=[] +From 0a6cc79e6814d7e2dbc637541cad86cebc47f855 Mon Sep 17 00:00:00 2001 +From: Hanno Rein +Date: Wed, 6 Nov 2024 17:37:22 -0500 +Subject: [PATCH 3/3] install_name + +--- + setup.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/setup.py b/setup.py +index 6443cb27..965b2fc9 100644 +--- a/setup.py ++++ b/setup.py +@@ -23,8 +23,8 @@ except: + extra_link_args=[] if sys.platform == 'darwin': - from distutils import sysconfig - vars = sysconfig.get_config_vars() -- vars['LDSHARED'] = vars['LDSHARED'].replace('-bundle', '-shared') + config_vars = sysconfig.get_config_vars() +- config_vars['LDSHARED'] = config_vars['LDSHARED'].replace('-bundle', '-shared') - extra_link_args=['-Wl,-install_name,@rpath/librebound'+suffix] -+ # vars['LDSHARED'] = vars['LDSHARED'].replace('-bundle', '-shared') -+ # extra_link_args=['-Wl,-install_name,@rpath/librebound'+suffix] - - libreboundmodule = Extension('librebound', - sources = [ 'src/rebound.c', ++ #config_vars['LDSHARED'] = config_vars['LDSHARED'].replace('-bundle', '-shared') ++ #extra_link_args=['-Wl,-install_name,@rpath/librebound'+suffix] + if sys.platform == 'win32': + extra_compile_args=[ghash_arg, '-DLIBREBOUND', '-D_GNU_SOURCE', '-DSERVER'] + else: +-- +2.39.5 (Apple Git-154) + diff --git a/packages/rebound/patches/0002-fix-output.patch b/packages/rebound/patches/0002-fix-output.patch new file mode 100644 index 00000000000..1b7bc3296de --- /dev/null +++ b/packages/rebound/patches/0002-fix-output.patch @@ -0,0 +1,57 @@ +From f875c41a58ae4b3d59d97a5787ec3e546c737860 Mon Sep 17 00:00:00 2001 +From: Hanno Rein +Date: Wed, 6 Nov 2024 17:21:47 -0500 +Subject: [PATCH] output + +--- + src/output.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/output.c b/src/output.c +index e897743a..fdd4ea45 100644 +--- a/src/output.c ++++ b/src/output.c +@@ -349,7 +349,7 @@ void reb_simulation_output_timing(struct reb_simulation* r, const double tmax){ + r->output_timing_last = temp; + }else{ + #ifdef __EMSCRIPTEN__ +- reb_remove_last_line(); ++ // reb_remove_last_line(); + #else + printf("\r"); + #endif +--- a/src/output.c ++++ b/src/output.c +@@ -257,17 +257,17 @@ void profiling_stop(int cat){ + + #ifdef __EMSCRIPTEN__ + // fflush does not work in emscripten. Workaround. +-EM_JS(void, reb_remove_last_line, (), { +- var output = document.getElementById("output"); +- if (output){ +- const lastIndex1 = output.value.lastIndexOf("\n"); +- const lastIndex2 = output.value.lastIndexOf("\n",lastIndex1-1); +- const lastIndexNtot = output.value.lastIndexOf("N_tot="); +- if(lastIndex1>0 && lastIndex20 && lastIndex2 +Date: Thu, 7 Nov 2024 09:40:09 -0500 +Subject: [PATCH 4/4] no emscripten + +--- + src/rebound.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/rebound.c b/src/rebound.c +index 82f6ac60..accea978 100644 +--- a/src/rebound.c ++++ b/src/rebound.c +@@ -665,7 +665,7 @@ int reb_check_exit(struct reb_simulation* const r, const double tmax, double* la + while(r->status == REB_STATUS_PAUSED || r->status == REB_STATUS_SCREENSHOT){ + // Wait for user to disable paused simulation + #ifdef __EMSCRIPTEN__ +- emscripten_sleep(100); ++ // emscripten_sleep(100); + #else + usleep(1000); + #endif +@@ -820,16 +820,16 @@ static void* reb_simulation_integrate_raw(void* args){ + } + reb_run_heartbeat(r); + #ifdef __EMSCRIPTEN__ +- double t0 = emscripten_performance_now(); ++// double t0 = emscripten_performance_now(); + #endif + while(reb_check_exit(r,thread_info->tmax,&last_full_dt)<0){ + #ifdef __EMSCRIPTEN__ +- double t1 = emscripten_performance_now(); +- if (t1-t0>1000./120.){ // max framerate 120Hz +- t0 = t1; +- emscripten_sleep(0); // allow drawing and event handling +- } +- ++// double t1 = emscripten_performance_now(); ++// if (t1-t0>1000./120.){ // max framerate 120Hz ++// t0 = t1; ++// emscripten_sleep(0); // allow drawing and event handling ++// } ++// + #endif + #ifdef OPENGL + if (r->display_data){ +-- +2.39.5 (Apple Git-154) + diff --git a/packages/reboundx/meta.yaml b/packages/reboundx/meta.yaml index fa52882cb4d..1ff826103a9 100644 --- a/packages/reboundx/meta.yaml +++ b/packages/reboundx/meta.yaml @@ -1,6 +1,6 @@ package: name: reboundx - version: 3.10.1 + version: 4.3.0 top-level: - reboundx build: @@ -10,8 +10,8 @@ build: ldflags: | -L$(WASM_LIBRARY_DIR)/lib/ source: - url: https://files.pythonhosted.org/packages/14/08/c1c0b83c90ea43ea8e218beadb13332259e930a2ca62f85024be89d811f3/reboundx-3.10.1.tar.gz - sha256: 65b2fcecf296661aad15a81a8d7d7b7b96cc2ba834c89e49dba24445f146d642 + url: https://files.pythonhosted.org/packages/2c/c2/32bb671c3f6944755b90c75d20f387daf4b9fc1e6919264cb0e9e26af8d7/reboundx-4.3.0.tar.gz + sha256: 94fc5b0993b623faa9964800f87fbc97236f7f12cb639c659d25702f1e0ac0ec requirements: run: - rebound From 2a19f2ae44741c046e88525305fe60227252ccfc Mon Sep 17 00:00:00 2001 From: Nick Altmann Date: Thu, 7 Nov 2024 20:30:43 +0000 Subject: [PATCH 23/29] Add `rateslib` to packages (#5146) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- packages/rateslib/meta.yaml | 23 +++++++++++++++++++++++ packages/rateslib/test_rateslib.py | 17 +++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 packages/rateslib/meta.yaml create mode 100644 packages/rateslib/test_rateslib.py diff --git a/packages/rateslib/meta.yaml b/packages/rateslib/meta.yaml new file mode 100644 index 00000000000..0df2e18ebf8 --- /dev/null +++ b/packages/rateslib/meta.yaml @@ -0,0 +1,23 @@ +package: + name: rateslib + version: 1.5.0 + top-level: + - rateslib +source: + url: https://files.pythonhosted.org/packages/02/d7/7695bb1b6a240ecfae77804d3fc242e30582fd83973f76dda03b23d921a6/rateslib-1.5.0.tar.gz + sha256: 7c7af2707f20974a70403689f84905757a4ed084e4c92229388d2fc30b331b04 +requirements: + run: + - numpy + - pandas + - matplotlib + executable: + - rustup +about: + home: https://github.com/attack68/rateslib + PyPI: https://pypi.org/project/rateslib + summary: A fixed income library for trading interest rates + license: CC-BY-NC-ND-4.0 +extra: + recipe-maintainers: + - NickAltmann diff --git a/packages/rateslib/test_rateslib.py b/packages/rateslib/test_rateslib.py new file mode 100644 index 00000000000..d352dd28177 --- /dev/null +++ b/packages/rateslib/test_rateslib.py @@ -0,0 +1,17 @@ +import pytest +from pytest_pyodide import run_in_pyodide + + +@pytest.mark.driver_timeout(60) +@run_in_pyodide(packages=["rateslib"]) +def test_add_tenor(selenium): + import datetime + + import rateslib as rl + + start_date = datetime.datetime(2024, 10, 29) + end_date = datetime.datetime(2024, 10, 31) + + test_date = rl.add_tenor(start_date, "2b", "F", "nyc") + + assert end_date == test_date From 912ccca6f0fc9722e2e69ec2ad1868d62c27c10a Mon Sep 17 00:00:00 2001 From: airen1986 <45193110+airen1986@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:13:10 +0530 Subject: [PATCH 24/29] Set MAXIMUM_MEMORY to 4GB and rename TOTAL_MEMORY to INITIAL_MEMORY (#5167) MAXIMUM_MEMORY defaults to 2GB but it can be 4GB. Also the "TOTAL_MEMORY" setting has changed name to "INITIAL_MEMORY". --- Makefile.envs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile.envs b/Makefile.envs index b48c6b3529b..54fea4eb4a7 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -142,8 +142,9 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \ -s USE_ZLIB \ -s USE_BZIP2 \ -s FORCE_FILESYSTEM=1 \ - -s TOTAL_MEMORY=20971520 \ + -s INITIAL_MEMORY=20971520 \ -s ALLOW_MEMORY_GROWTH=1 \ + -s MAXIMUM_MEMORY=4GB \ -s EXPORT_ALL=1 \ -s FS_DEBUG=1 \ -s STACK_SIZE=5MB \ @@ -230,8 +231,9 @@ ifeq ($(DISABLE_DYLINK), 1) -s USE_ZLIB \ -s USE_BZIP2 \ -s FORCE_FILESYSTEM=1 \ - -s TOTAL_MEMORY=20971520 \ + -s INITIAL_MEMORY=20971520 \ -s ALLOW_MEMORY_GROWTH=1 \ + -s MAXIMUM_MEMORY=4GB \ -s STACK_SIZE=5MB \ -s ALLOW_TABLE_GROWTH \ -s FS_DEBUG=1 \ From ecf3f163a83776b6e7ec2122ed155977b24b5c26 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 8 Nov 2024 21:06:21 +0100 Subject: [PATCH 25/29] Drop duplicated array tests (#5174) --- src/tests/test_typeconversions.py | 228 ------------------------------ 1 file changed, 228 deletions(-) diff --git a/src/tests/test_typeconversions.py b/src/tests/test_typeconversions.py index 36151a80d18..6e9c9afdfe9 100644 --- a/src/tests/test_typeconversions.py +++ b/src/tests/test_typeconversions.py @@ -1657,234 +1657,6 @@ class A: assert res.a == 2 -@pytest.mark.parametrize("n", [1 << 31, 1 << 32, 1 << 33, 1 << 63, 1 << 64, 1 << 65]) -@run_in_pyodide -def test_very_large_length(selenium, n): - from unittest import TestCase - - from pyodide.code import run_js - - raises = TestCase().assertRaises( - OverflowError, msg=f"length {n} of object is larger than INT_MAX (2147483647)" - ) - - o = run_js(f"({{length : {n}}})") - with raises: - len(o) - - # 1. Set toStringTag to NodeList to force JsProxy to feature detect this object - # as an array - # 2. Return a very large length - # 3. JsProxy_subscript_array should successfully handle this and propagate the error. - a = run_js(f"({{[Symbol.toStringTag] : 'NodeList', length: {n}}})") - with raises: - a[-1] - - -@pytest.mark.parametrize( - "n", [-1, -2, -3, -100, -1 << 31, -1 << 32, -1 << 33, -1 << 63, -1 << 64, -1 << 65] -) -@run_in_pyodide -def test_negative_length(selenium, n): - from unittest import TestCase - - from pyodide.code import run_js - - raises = TestCase().assertRaises( - ValueError, msg=f"length {n} of object is negative" - ) - - o = run_js(f"({{length : {n}}})") - with raises: - len(o) - - # 1. Set toStringTag to NodeList to force JsProxy to feature detect this object - # as an array - # 2. Return a negative length - # 3. JsProxy_subscript_array should successfully handle this and propagate the error. - a = run_js(f"({{[Symbol.toStringTag] : 'NodeList', length: {n}}})") - with raises: - a[-1] - - -@std_hypothesis_settings -@given(l=st.lists(st.integers()), slice=st.slices(50)) -@example(l=[0, 1], slice=slice(None, None, -1)) -@example(l=list(range(4)), slice=slice(None, None, -2)) -@example(l=list(range(10)), slice=slice(-1, 12)) -@example(l=list(range(10)), slice=slice(12, -1)) -@example(l=list(range(10)), slice=slice(12, -1, -1)) -@example(l=list(range(10)), slice=slice(-1, 12, 2)) -@example(l=list(range(10)), slice=slice(12, -1, -1)) -@example(l=list(range(10)), slice=slice(12, -1, -2)) -@run_in_pyodide -def test_array_slices(selenium, l, slice): - expected = l[slice] - from pyodide.ffi import JsArray, to_js - - jsl = to_js(l) - assert isinstance(jsl, JsArray) - result = jsl[slice] - assert result.to_py() == expected - - -@std_hypothesis_settings -@given(l=st.lists(st.integers()), slice=st.slices(50)) -@example(l=[0, 1], slice=slice(None, None, -1)) -@example(l=list(range(4)), slice=slice(None, None, -2)) -@example(l=list(range(10)), slice=slice(-1, 12)) -@example(l=list(range(10)), slice=slice(12, -1)) -@example(l=list(range(10)), slice=slice(12, -1, -1)) -@example(l=list(range(10)), slice=slice(-1, 12, 2)) -@example(l=list(range(10)), slice=slice(12, -1, -1)) -@example(l=list(range(10)), slice=slice(12, -1, -2)) -@run_in_pyodide -def test_array_slice_del(selenium, l, slice): - from pyodide.ffi import JsArray, to_js - - jsl = to_js(l) - assert isinstance(jsl, JsArray) - del l[slice] - del jsl[slice] - assert jsl.to_py() == l - - -@st.composite -def list_slice_and_value(draw): - l = draw(st.lists(st.integers())) - step_one = draw(st.booleans()) - if step_one: - start = draw(st.integers(0, max(len(l) - 1, 0)) | st.none()) - stop = draw(st.integers(start, len(l)) | st.none()) - if draw(st.booleans()) and start is not None: - start -= len(l) - if draw(st.booleans()) and stop is not None: - stop -= len(l) - s = slice(start, stop) - vals = draw(st.lists(st.integers())) - else: - s = draw(st.slices(50)) - vals_len = len(l[s]) - vals = draw(st.lists(st.integers(), min_size=vals_len, max_size=vals_len)) - return (l, s, vals) - - -@std_hypothesis_settings -@given(lsv=list_slice_and_value()) -@example(lsv=(list(range(5)), slice(5, 2), [])) -@example(lsv=(list(range(5)), slice(2, 5, -1), [])) -@example(lsv=(list(range(5)), slice(5, 2), [-1, -2, -3])) -@run_in_pyodide -def test_array_slice_assign_1(selenium, lsv): - from pyodide.ffi import JsArray, to_js - - [l, s, v] = lsv - jsl = to_js(l) - assert isinstance(jsl, JsArray) - l[s] = v - jsl[s] = v - assert jsl.to_py() == l - - -@run_in_pyodide -def test_array_slice_assign_2(selenium): - import pytest - - from pyodide.ffi import JsArray, to_js - - l = list(range(10)) - with pytest.raises(ValueError) as exc_info_1a: - l[0:4:2] = [1, 2, 3, 4] - - jsl = to_js(l) - assert isinstance(jsl, JsArray) - with pytest.raises(ValueError) as exc_info_1b: - jsl[0:4:2] = [1, 2, 3, 4] - - l = list(range(10)) - with pytest.raises(ValueError) as exc_info_2a: - l[0:4:2] = [] - - with pytest.raises(ValueError) as exc_info_2b: - jsl[0:4:2] = [] - - with pytest.raises(TypeError) as exc_info_3a: - l[:] = 1 # type: ignore[call-overload] - - with pytest.raises(TypeError) as exc_info_3b: - jsl[:] = 1 # type: ignore[call-overload] - - assert exc_info_1a.value.args == exc_info_1b.value.args - assert exc_info_2a.value.args == exc_info_2b.value.args - assert exc_info_3a.value.args == exc_info_3b.value.args - - -@std_hypothesis_settings -@given(l1=st.lists(st.integers()), l2=st.lists(st.integers())) -@example(l1=[], l2=[]) -@example(l1=[], l2=[1]) -@run_in_pyodide -def test_array_extend(selenium_module_scope, l1, l2): - from pyodide.ffi import to_js - - l1js1 = to_js(l1) - l1js1.extend(l2) - - l1js2 = to_js(l1) - l1js2 += l2 - - l1.extend(l2) - - assert l1 == l1js1.to_py() - assert l1 == l1js2.to_py() - - -@run_in_pyodide -def test_typed_array(selenium): - from pyodide.code import run_js - - a = run_js("self.a = new Uint8Array([1,2,3,4]); a") - assert a[0] == 1 - assert a[-1] == 4 - a[-2] = 7 - assert run_js("self.a[2]") == 7 - - import pytest - - with pytest.raises(TypeError, match="does ?n[o']t support item deletion"): - del a[0] - - msg = "Slice subscripting isn't implemented for typed arrays" - with pytest.raises(NotImplementedError, match=msg): - a[:] - - msg = "Slice assignment isn't implemented for typed arrays" - with pytest.raises(NotImplementedError, match=msg): - a[:] = [-1, -2, -3, -4] - - assert not hasattr(a, "extend") - with pytest.raises(TypeError): - a += [1, 2, 3] - - -@pytest.mark.xfail_browsers(node="No document in node") -@run_in_pyodide -def test_html_array(selenium): - from pyodide.code import run_js - - x = run_js("document.querySelectorAll('*')") - assert run_js("(a, b) => a === b[0]")(x[0], x) - assert run_js("(a, b) => a === Array.from(b).pop()")(x[-1], x) - - import pytest - - with pytest.raises(TypeError, match="does ?n[o']t support item assignment"): - x[0] = 0 - - with pytest.raises(TypeError, match="does ?n[o']t support item deletion"): - del x[0] - - @run_in_pyodide def test_bind_attrs(selenium): from typing import Annotated From 870a67c9b15b27b4b4afe060eabe6e08110c221c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 9 Nov 2024 18:50:02 +0800 Subject: [PATCH 26/29] Bump `crc32c` to version 2.7.1 (#5169) --- docs/project/changelog.md | 1 + packages/crc32c/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 7f1e704f9bb..8e806b154a5 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -64,6 +64,7 @@ myst: ### Packages +- Upgraded `crc32c` to 2.7.1 {pr}`5169` - Upgraded `rebound` to 4.4.3 {pr}`5163` - Upgraded `reboundx` to 4.3.0 {pr}`5163` - Upgraded `msprime` to 1.3.3 {pr}`5159` diff --git a/packages/crc32c/meta.yaml b/packages/crc32c/meta.yaml index 02a1483f756..988633c0a03 100644 --- a/packages/crc32c/meta.yaml +++ b/packages/crc32c/meta.yaml @@ -1,11 +1,11 @@ package: name: crc32c - version: "2.4" + version: 2.7.1 top-level: - crc32c source: - url: https://files.pythonhosted.org/packages/6f/19/47ac8d1d5b81a83272fe56d4cf425274437fd619ed0af9a5f9805484748c/crc32c-2.4.tar.gz - sha256: d985c4d9b1a1fd16c593d83f8735a8e4e156790a95338a1e0b199aac51ca1e5e + url: https://files.pythonhosted.org/packages/7f/4c/4e40cc26347ac8254d3f25b9f94710b8e8df24ee4dddc1ba41907a88a94d/crc32c-2.7.1.tar.gz + sha256: f91b144a21eef834d64178e01982bb9179c354b3e9e5f4c803b0e5096384968c about: home: https://github.com/ICRAR/crc32c PyPI: https://pypi.org/project/crc32c From 4c2878f01ec3053444ee187b1370b8c08f6dd798 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 9 Nov 2024 22:30:01 +0800 Subject: [PATCH 27/29] Add `zfp` and `zfpy` (#5172) Co-authored-by: Gyeongjae Choi --- .pre-commit-config.yaml | 2 +- docs/project/changelog.md | 1 + packages/libzfp/meta.yaml | 34 +++++++++ packages/zfpy/meta.yaml | 37 ++++++++++ .../zfpy/patches/Modernise-packaging.patch | 69 +++++++++++++++++ packages/zfpy/test_zfpy.py | 74 +++++++++++++++++++ 6 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 packages/libzfp/meta.yaml create mode 100644 packages/zfpy/meta.yaml create mode 100644 packages/zfpy/patches/Modernise-packaging.patch create mode 100644 packages/zfpy/test_zfpy.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b44923b4573..d27f0871a9c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,7 +82,7 @@ repos: name: mypy-tests args: [--ignore-missing-imports] files: ^(packages/|docs|/conftest.py|src/tests) - exclude: (^packages/.*/setup.py|/src|^packages/aiohttp/aiohttp_patch.py$) + exclude: (^packages/.*/setup.py|/src|^packages/aiohttp/aiohttp_patch.py$|^packages/zfpy/test_zfpy.py$) additional_dependencies: *mypy-deps - repo: https://github.com/pre-commit/mirrors-prettier diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 8e806b154a5..04a3cb4ea8d 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -105,6 +105,7 @@ myst: - Added `tree-sitter-java` 0.23.2 {pr}`5102` - Added `tree-sitter-python` 0.23.2 {pr}`5102` - Added `Narwhals` 1.9.4 {pr}`5121` +- Added `libzfp` and `zfpy` 1.0.1 {pr}`5172` ## Version 0.26.3 diff --git a/packages/libzfp/meta.yaml b/packages/libzfp/meta.yaml new file mode 100644 index 00000000000..7f66a86166d --- /dev/null +++ b/packages/libzfp/meta.yaml @@ -0,0 +1,34 @@ +package: + name: libzfp + version: 1.0.1 + tag: + - library + - static_library +source: + url: https://github.com/LLNL/zfp/archive/refs/tags/1.0.1.tar.gz + sha256: 4984db6a55bc919831966dd17ba5e47ca7ac58668f4fd278ebd98cd2200da66f + +build: + type: static_library + exports: requested + script: | + mkdir -p build + cd build + emcmake cmake ${CMAKE_ARGS} \ + -DBUILD_CFP=ON \ + -DBUILD_UTILITIES=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DZFP_ENABLE_PIC=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-s WASM=1" \ + -DZFP_WITH_OPENMP=OFF \ + -DCMAKE_INSTALL_PREFIX=${WASM_LIBRARY_DIR} \ + .. + + emmake make -j ${PYODIDE_JOBS:-3} install + +about: + home: https://zfp.llnl.gov/ + summary: Open-source software for compressed floating-point arrays + license: BSD-3-Clause diff --git a/packages/zfpy/meta.yaml b/packages/zfpy/meta.yaml new file mode 100644 index 00000000000..b883a3a76df --- /dev/null +++ b/packages/zfpy/meta.yaml @@ -0,0 +1,37 @@ +package: + name: zfpy + version: 1.0.1 + top-level: + - zfpy + +source: + # There's no sdist for zfpy on PyPI, so we get the tarball from the GitHub release instead + url: https://github.com/LLNL/zfp/archive/refs/tags/1.0.1.tar.gz + sha256: 4984db6a55bc919831966dd17ba5e47ca7ac58668f4fd278ebd98cd2200da66f + patches: + - patches/Modernise-packaging.patch + +build: + exports: requested + cflags: | + -I$(PYTHONINCLUDE) + -I$(WASM_LIBRARY_DIR)/include + ldflags: | + -L$(WASM_LIBRARY_DIR)/lib + +requirements: + host: + - libzfp + - numpy + run: + - numpy + +test: + imports: + - zfpy + +about: + home: https://zfp.llnl.gov/ + PyPI: https://pypi.org/project/zfpy/ + summary: zfp compression in Python + license: BSD-3-Clause diff --git a/packages/zfpy/patches/Modernise-packaging.patch b/packages/zfpy/patches/Modernise-packaging.patch new file mode 100644 index 00000000000..411f7835bf2 --- /dev/null +++ b/packages/zfpy/patches/Modernise-packaging.patch @@ -0,0 +1,69 @@ +From f36ef86e67185b16b65445092c40a3d75d0be45c Mon Sep 17 00:00:00 2001 +From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> +Date: Fri, 8 Nov 2024 02:46:00 +0530 +Subject: [PATCH] Modernise packaging + +This patch introduces a PEP 517/518 compliant build system using setuptools and +declarative metadata in pyproject.toml, and fixes syntax errors in setup.py. The +sources for the extension module are correctly specified as Cython source files. + +Some part of the credits go to the upstream +PR at https://github.com/LLNL/zfp/pull/234 + +Co-Authored-By: David M. Rogers <2243447+frobnitzem@users.noreply.github.com> +--- + pyproject.toml | 13 +++++++++++++ + setup.py | 18 +++++++++--------- + 2 files changed, 22 insertions(+), 9 deletions(-) + create mode 100644 pyproject.toml + +diff --git a/pyproject.toml b/pyproject.toml +new file mode 100644 +index 0000000..88adcc8 +--- /dev/null ++++ b/pyproject.toml +@@ -0,0 +1,13 @@ ++[build-system] ++requires = ["setuptools", "cython", "numpy"] ++build-backend = "setuptools.build_meta" ++ ++[project] ++name = "zfpy" ++version = "1.0.1" ++authors = [ ++ { name = "Peter Lindstrom", email = "zfp@llnl.gov" }, ++ { name = "Danielle Asher", email = "zfp@llnl.gov" }, ++] ++description = "zfp compression in Python" ++requires-python = ">=3.0" +diff --git a/setup.py b/setup.py +index fa2da6e..919fa7c 100644 +--- a/setup.py ++++ b/setup.py +@@ -2,14 +2,14 @@ from setuptools import setup, Extension + import numpy as np + + setup( +- name="zfpy", +- version="1.0.1", +- author="Peter Lindstrom, Danielle Asher", +- author_email="zfp@llnl.gov", + url="https://zfp.llnl.gov", +- description="zfp compression in Python", +- long_description="zfp is a compressed format for representing multidimensional floating-point and integer arrays. zfp provides compressed-array classes that support high throughput read and write random access to individual array elements. zfp also supports serial and parallel compression of whole arrays using both lossless and lossy compression with error tolerances. zfp is primarily written in C and C++ but also includes Python and Fortran bindings.", +- ext_modules=[Extension("zfpy", ["build/python/zfpy.c"], +- include_dirs=["include", np.get_include()], +- libraries=["zfp"], library_dirs=["build/lib64", "build/lib/Release"]), language_level = "3"] ++ ext_modules= ++ [ ++ Extension( ++ name="zfpy", ++ sources=["python/zfpy.pyx"], ++ include_dirs=["include", np.get_include()], ++ libraries=["zfp"], ++ library_dirs=["build/lib64", "build/lib"]) ++ ], + ) +-- +2.39.3 (Apple Git-146) + diff --git a/packages/zfpy/test_zfpy.py b/packages/zfpy/test_zfpy.py new file mode 100644 index 00000000000..0896f323d32 --- /dev/null +++ b/packages/zfpy/test_zfpy.py @@ -0,0 +1,74 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["zfpy", "numpy"]) +def test_compression(selenium): + import numpy as np + import zfpy + + my_array = np.arange(1, 20) + compressed_data = zfpy.compress_numpy(my_array) + decompressed_array = zfpy.decompress_numpy(compressed_data) + np.testing.assert_array_equal(my_array, decompressed_array) + + +@run_in_pyodide(packages=["zfpy", "numpy"]) +def test_compression_with_tolerance(selenium): + import numpy as np + import zfpy + + my_array = np.linspace(0, 1, 1000) + compressed_data = zfpy.compress_numpy(my_array, tolerance=1e-3) + decompressed_array = zfpy.decompress_numpy(compressed_data) + np.testing.assert_allclose(my_array, decompressed_array, atol=1e-3) + + +@run_in_pyodide(packages=["zfpy", "numpy"]) +def test_different_dimensions(selenium) -> None: + import numpy as np + import zfpy + + np.random.seed(42) + + # Test arrays; from 1D to 4D + for dimensions in range(1, 5): + # 1. test with uniform dimensions + shape1 = tuple([5] * dimensions) + array = np.random.rand(*shape1).astype(np.float64) + compressed1 = zfpy.compress_numpy(array, write_header=True) + decompressed1 = zfpy.decompress_numpy(compressed1) + np.testing.assert_array_equal(decompressed1, array) + + # 2. test with increasing dimensions + shape2 = tuple(range(2, 2 + dimensions)) + array = np.random.rand(*shape2).astype(np.float64) + compressed2 = zfpy.compress_numpy(array, write_header=True) + decompressed2 = zfpy.decompress_numpy(compressed2) + np.testing.assert_array_equal(decompressed2, array) + + +@run_in_pyodide(packages=["zfpy", "numpy"]) +def test_different_dtypes(selenium) -> None: + """Test ZFP compression/decompression with different numeric dtypes.""" + import numpy as np + import zfpy + + np.random.seed(42) + + shape = (5, 5) + num_elements = np.prod(shape) + + # Test floating-point types + for dtype in [np.float32, np.float64]: + elements = np.random.random_sample(num_elements) + array = np.reshape(elements, shape).astype(dtype) + compressed1 = zfpy.compress_numpy(array, write_header=True) + decompressed1 = zfpy.decompress_numpy(compressed1) + np.testing.assert_array_equal(decompressed1, array) + + # Test integer types + for dtype in [np.int32, np.int64]: + array = np.random.randint(low=-(2**30), high=2**30, size=shape, dtype=dtype) + compressed2 = zfpy.compress_numpy(array, write_header=True) + decompressed2 = zfpy.decompress_numpy(compressed2) + np.testing.assert_array_equal(decompressed2, array) From 993a8ee02f2bc3c2b5e1fca4c454d2eb2a0a968b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 10 Nov 2024 12:43:04 +0100 Subject: [PATCH 28/29] requirements.txt: Pin selenium==4.25.0 to fix ci failures (#5182) --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index a520fb49b29..55363655789 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,6 @@ pytest-cov pytest-httpserver pytest-benchmark pytest-pyodide==0.58.3 +setuptools; python_version >= '3.12' +# TODO(cclauss): Remove the Selenium line. +selenium==4.25.0 From 39e62fcb35d79c4340ce0e12062ea0cf402f1300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Gyarmati?= Date: Mon, 11 Nov 2024 10:34:50 +0100 Subject: [PATCH 29/29] Add `clingo` package (#5184) Adds pypi.org/project/clingo, a powerful Answer Set Programming (ASP) solver. --- docs/project/changelog.md | 1 + packages/clingo/meta.yaml | 21 +++++++++++++++++++++ packages/clingo/test_clingo.py | 25 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 packages/clingo/meta.yaml create mode 100644 packages/clingo/test_clingo.py diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 04a3cb4ea8d..24a00742aa4 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -106,6 +106,7 @@ myst: - Added `tree-sitter-python` 0.23.2 {pr}`5102` - Added `Narwhals` 1.9.4 {pr}`5121` - Added `libzfp` and `zfpy` 1.0.1 {pr}`5172` +- Added `clingo` 5.7.1 {pr}`5184` ## Version 0.26.3 diff --git a/packages/clingo/meta.yaml b/packages/clingo/meta.yaml new file mode 100644 index 00000000000..8adeda01d0e --- /dev/null +++ b/packages/clingo/meta.yaml @@ -0,0 +1,21 @@ +package: + name: clingo + version: 5.7.1 + top-level: + - clingo +source: + url: https://files.pythonhosted.org/packages/3b/b3/c7b464426f70fe100cb4f3e5b45623e3eb82ea50f1dffdc2f820f1418fe0/clingo-5.7.1.tar.gz + sha256: 17400a1894da46b2d4941a4a85012c98fa8b3c67a5c6b4c73dcd8c8facbc059f +about: + home: https://github.com/potassco/clingo + PyPI: https://pypi.org/project/clingo + summary: CFFI-based bindings to the clingo solver. + license: MIT +requirements: + host: + - cffi + run: + - cffi +extra: + recipe-maintainers: + - peter-gy diff --git a/packages/clingo/test_clingo.py b/packages/clingo/test_clingo.py new file mode 100644 index 00000000000..5ce43335304 --- /dev/null +++ b/packages/clingo/test_clingo.py @@ -0,0 +1,25 @@ +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide(packages=["clingo"]) +def test_clingo(selenium): + from clingo.control import Control + from clingo.symbol import Number + + class Context: + def inc(self, x): + return Number(x.number + 1) + + def seq(self, x, y): + return [x, y] + + def on_model(m): + print(m) + + ctl = Control() + program = "\n".join(["p(@inc(10)).", "q(@seq(1,2))."]) + ctl.add("base", [], program) + + ctl.ground([("base", [])], context=Context()) + solution = ctl.solve(on_model=on_model) + assert solution.satisfiable