From 5f2a8682aef96133579f5a8e9a7c7ede9ad2f758 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Wed, 2 Nov 2022 10:24:05 -0400 Subject: [PATCH] qsv now links against Python 3.11 for the `py` command (#576) * `python`: we now link against python 3.11 - the latest stable python * change all GH workflows to use python 3.11 * add empty DLLs directory to suppress python 3.11 warning as per https://github.com/python/cpython/issues/98790 --- .github/workflows/publish-nightly.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/rust-beta.yml | 2 +- .github/workflows/rust-macos.yml | 2 +- .github/workflows/rust-nightly-bleeding-edge.yml | 2 +- .github/workflows/rust-windows.yml | 5 ++++- .github/workflows/rust.yml | 2 +- .github/workflows/test-publish.yml | 2 +- README.md | 4 ++-- src/cmd/python.rs | 4 ++-- 10 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publish-nightly.yml b/.github/workflows/publish-nightly.yml index 54e2f2111..81755fc35 100644 --- a/.github/workflows/publish-nightly.yml +++ b/.github/workflows/publish-nightly.yml @@ -78,7 +78,7 @@ jobs: ref: ${{ needs.analyze-tags.outputs.previous-tag }} - uses: actions/setup-python@v4.3.0 with: - python-version: '3.10' + python-version: '3.11' - name: build prep for x86_64-unknown-linux-musl if: ${{ matrix.job.musl-prep }} run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index db7931ba0..51d1dbe31 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -149,7 +149,7 @@ jobs: ref: ${{ needs.analyze-tags.outputs.previous-tag }} - uses: actions/setup-python@v4.3.0 with: - python-version: '3.10' + python-version: '3.11' - name: build prep for aarch64-apple-darwin if: ${{ matrix.job.build-prep }} run: | diff --git a/.github/workflows/rust-beta.yml b/.github/workflows/rust-beta.yml index 3d00db9d8..8b7a365e9 100644 --- a/.github/workflows/rust-beta.yml +++ b/.github/workflows/rust-beta.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4.3.0 with: - python-version: '3.10' + python-version: '3.11' - name: Update Rust Beta run: | rustup update beta diff --git a/.github/workflows/rust-macos.yml b/.github/workflows/rust-macos.yml index 09472711b..7004b2eee 100644 --- a/.github/workflows/rust-macos.yml +++ b/.github/workflows/rust-macos.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4.3.0 with: - python-version: '3.10' + python-version: '3.11' - name: Update Rust run: rustup update - name: Setup Rust-cache diff --git a/.github/workflows/rust-nightly-bleeding-edge.yml b/.github/workflows/rust-nightly-bleeding-edge.yml index 77fbe3844..611463e09 100644 --- a/.github/workflows/rust-nightly-bleeding-edge.yml +++ b/.github/workflows/rust-nightly-bleeding-edge.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4.3.0 with: - python-version: '3.10' + python-version: '3.11' - name: Install and Run Redis run: | sudo apt-get install redis-server diff --git a/.github/workflows/rust-windows.yml b/.github/workflows/rust-windows.yml index 925fb14d1..5fbf77a8d 100644 --- a/.github/workflows/rust-windows.yml +++ b/.github/workflows/rust-windows.yml @@ -18,13 +18,16 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4.3.0 with: - python-version: '3.10' + python-version: '3.11' - name: Update Rust run: rustup update - name: Setup Rust-cache uses: Swatinem/rust-cache@v1 with: key: qsv-cache + - name: make empty DLLs directory to prevent spurious python warning + run: | + mkdir DLLs - name: Run tests run: cargo test --verbose --locked --features=apply,fetch,foreach,generate,lua,python,full - name: Run luajit tests separately as its mutually exclusive with lua feature diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b66fa6a1d..039c0a86a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4.3.0 with: - python-version: '3.10' + python-version: '3.11' - name: Install and Run Redis run: | sudo apt-get install redis-server diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml index 342da88a3..ae293273f 100644 --- a/.github/workflows/test-publish.yml +++ b/.github/workflows/test-publish.yml @@ -131,7 +131,7 @@ jobs: ref: 'master' - uses: actions/setup-python@v4.3.0 with: - python-version: '3.10.7' + python-version: '3.11' - name: build prep for aarch64-apple-darwin if: ${{ matrix.job.build-prep }} run: | diff --git a/README.md b/README.md index 4119e9c12..19f313ad5 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ See [FAQ](https://github.com/jqnatividad/qsv/discussions/categories/faq) for mor | [luajit](/src/cmd/luajit.rs#L2)[^1] | Execute a [LuaJIT](https://luajit.org/luajit.html) 2.0 (a Just-In-Time compiler for Lua 5.1) script over CSV lines to transform, aggregate or filter them. [LuaJIT is even faster still than Lua](https://luajit.org/performance_x86.html). | | [partition](/src/cmd/partition.rs#L2) | Partition a CSV based on a column value. | | [pseudo](/src/cmd/pseudo.rs#L2) | [Pseudonymise](https://en.wikipedia.org/wiki/Pseudonymization) the value of the given column by replacing them with an incremental identifier. | -| [py](/src/cmd/python.rs#L2)[^1] | Evaluate a Python expression over CSV lines to transform or filter them. Python's [f-strings](https://www.freecodecamp.org/news/python-f-strings-tutorial-how-to-use-f-strings-for-string-formatting/) is particularly useful for extended formatting, [with the ability to evaluate Python expressions as well](https://github.com/jqnatividad/qsv/blob/4cd00dca88addf0d287247fa27d40563b6d46985/src/cmd/python.rs#L23-L31).
Consider using the `lua`/`luajit` commands instead if you're having Python version issues ([Python 3.6 and up supported, with Python 3.10 required on prebuilt qsv](#python)) as it's much faster, embedded, can do aggregations & has no external dependencies. | +| [py](/src/cmd/python.rs#L2)[^1] | Evaluate a Python expression over CSV lines to transform or filter them. Python's [f-strings](https://www.freecodecamp.org/news/python-f-strings-tutorial-how-to-use-f-strings-for-string-formatting/) is particularly useful for extended formatting, [with the ability to evaluate Python expressions as well](https://github.com/jqnatividad/qsv/blob/4cd00dca88addf0d287247fa27d40563b6d46985/src/cmd/python.rs#L23-L31).
Consider using the `lua`/`luajit` commands instead if you're having Python version issues ([Python 3.6 and up supported, with Python 3.11 required on prebuilt qsv](#python)) as it's much faster, embedded, can do aggregations & has no external dependencies. | | [rename](/src/cmd/rename.rs#L2) | Rename the columns of a CSV efficiently. | | [replace](/src/cmd/replace.rs#L2) | Replace CSV data using a regex. | | [reverse](/src/cmd/reverse.rs#L2)[^3] | Reverse order of rows in a CSV. Unlike the `sort --reverse` command, it preserves the order of rows with the same key. | @@ -236,7 +236,7 @@ Note that this will happen as soon as the qsv binary is invoked, even if you're If you don't need to run the `py` command, simply use `qsvnp` ("np" stands for "no python"), `qsvlite`, or `qsvdp`. If you need the `py` command, the [prebuilt qsv binary](https://github.com/jqnatividad/qsv/releases/latest) is compiled, as a policy, -using the current stable Python minor version (currently Python 3.10) at the time of release. +using the current stable Python minor version (currently Python 3.11) at the time of release. If you require a different Python version (Python 3.6 and up are supported), you'll need to install/compile from source, making sure you have the development libraries for the desired Python version installed when doing so. diff --git a/src/cmd/python.rs b/src/cmd/python.rs index 566f5a4c3..7c96a8817 100644 --- a/src/cmd/python.rs +++ b/src/cmd/python.rs @@ -66,8 +66,8 @@ Some usage examples: 2,b2,b3,b4,7d594b33f82bdcbc1cfa6f924a84c4cd 3,c2,c3,c4,6eabbfdbfd9ab6ae7737fb2b82f6a1af - NOTE: The prebuilt qsv binaries are linked against Python 3.10 and will require access - to the Python 3.10 shared libraries (libpython* on Linux/macOS, python*.dll on Windows) + NOTE: The prebuilt qsv binaries are linked against Python 3.11 and will require access + to the Python 3.11 shared libraries (libpython* on Linux/macOS, python*.dll on Windows) during runtime for the py command to run. Note that qsv with the `python` feature enabled will panic on startup even if you're not