Run tests on CI with cargo-nextest

.config/nextest.toml

@@ -0,0 +1,64 @@
+[store]
+# The directory under the workspace root at which nextest-related files are
+# written. Profile-specific storage is currently written to dir/<profile-name>.
+dir = "target/nextest"
+
+# This section defines the default nextest profile. Custom profiles are layered
+# on top of the default profile.
+[profile.default]
+# "retries" defines the number of times a test should be retried. If set to a
+# non-zero value, tests that succeed on a subsequent attempt will be marked as
+# non-flaky. Can be overridden through the `--retries` option.
+retries = 2
+
+# Show these test statuses in the output.
+#
+# The possible values this can take are:
+# * none: no output
+# * fail: show failed (including exec-failed) tests
+# * retry: show flaky and retried tests
+# * slow: show slow tests
+# * pass: show passed tests
+# * skip: show skipped tests (most useful for CI)
+# * all: all of the above
+#
+# Each value includes all the values above it; for example, "slow" includes
+# failed and retried tests.
+#
+# Can be overridden through the `--status-level` flag.
+status-level = "pass"
+
+# "failure-output" defines when test failures are output to standard output.
+# Accepted values are
+# * "immediate": output failures as soon as they happen
+# * "final": output failures at the end of the test run
+# * "immediate-final": output failures as soon as they happen and at the end of
+#   the test run
+# * "never": don't output failures at all
+#
+# For large test suites and CI it is generally useful to use "immediate-final".
+#
+# Can be overridden through the `--failure-output` option.
+failure-output = "immediate"
+
+# "success-output" controls output on success. This should generally be set to
+# "never".
+success-output = "never"
+
+# Cancel the test run on the first failure. For CI runs, consider setting this
+# to false.
+fail-fast = false
+
+# Treat a test that takes longer than this as slow, and print a message.
+slow-timeout = "60s"
+
+[profile.default.junit]
+# Output a JUnit report into the given file inside 'store.dir/<profile-name>'.
+# If unspecified, JUnit is not written out.
+
+# path = "junit.xml"
+
+# The name of the top-level "report" element in JUnit report. If aggregating
+# reports across different test runs, it may be useful to provide separate names
+# for each report.
+report-name = "nextest-run"

.github/workflows/ci.yml

@@ -163,6 +163,9 @@ jobs:
           # needed to correctly format errors, see #1865
           components: rust-src
 
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
+
       - uses: Swatinem/rust-cache@v1
         with:
           key: cargo-${{ matrix.platform.python-architecture }}-${{ matrix.platform.os }}-${{ matrix.msrv }}
@@ -193,13 +196,13 @@ jobs:
       # Run tests (except on PyPy, because no embedding API).
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test (no features)
-        run: cargo test --no-default-features --lib --tests
+        run: cargo nextest run --no-default-features --lib --tests
 
       # --no-default-features when used with `cargo build/test -p` doesn't seem to work!
       - name: Test pyo3-build-config (no features)
         run: |
           cd pyo3-build-config
-          cargo test --no-default-features
+          cargo nextest run --no-default-features
 
       - name: Build (all additive features)
         run: cargo build --lib --tests --no-default-features --features "full ${{ matrix.extra_features }}"
@@ -211,23 +214,23 @@ jobs:
       # Run tests (except on PyPy, because no embedding API).
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test
-        run: cargo test --no-default-features --features "full ${{ matrix.extra_features }}"
+        run: cargo nextest run --no-default-features --features "full ${{ matrix.extra_features }}"
 
       # Run tests again, but in abi3 mode
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test (abi3)
-        run: cargo test --no-default-features --features "abi3 full ${{ matrix.extra_features }}"
+        run: cargo nextest run --no-default-features --features "abi3 full ${{ matrix.extra_features }}"
 
       # Run tests again, for abi3-py37 (the minimal Python version)
       - if: ${{ (!startsWith(matrix.python-version, 'pypy')) && (matrix.python-version != '3.7') }}
         name: Test (abi3-py37)
-        run: cargo test --no-default-features --features "abi3-py37 full ${{ matrix.extra_features }}"
+        run: cargo nextest run --no-default-features --features "abi3-py37 full ${{ matrix.extra_features }}"
 
       - name: Test proc-macro code
-        run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml
+        run: cargo nextest run --manifest-path=pyo3-macros-backend/Cargo.toml
 
       - name: Test build config
-        run: cargo test --manifest-path=pyo3-build-config/Cargo.toml
+        run: cargo nextest run --manifest-path=pyo3-build-config/Cargo.toml
 
       - name: Test python examples and tests
         shell: bash
@@ -304,6 +307,8 @@ jobs:
           components: llvm-tools-preview
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
       - run: python -m pip install -U pip nox
       - run: cargo xtask coverage --output-lcov coverage.lcov
       - uses: codecov/codecov-action@v2

src/marker.rs

@@ -994,9 +994,6 @@ mod tests {
         const GIL_NOT_HELD: c_int = 0;
         const GIL_HELD: c_int = 1;
 
-        let state = unsafe { crate::ffi::PyGILState_Check() };
-        assert_eq!(state, GIL_NOT_HELD);
-
         {
             let gil = Python::acquire_gil();
             let _py = gil.python();

xtask/src/llvm_cov.rs

@@ -12,44 +12,59 @@ pub fn run(opts: CoverageOpts) -> Result<()> {
 
     cli::run(
         Command::new("cargo")
-            .args(&["test", "--manifest-path", "pyo3-build-config/Cargo.toml"])
+            .args(&[
+                "nextest",
+                "run",
+                "--manifest-path",
+                "pyo3-build-config/Cargo.toml",
+            ])
             .envs(&env),
     )?;
     cli::run(
         Command::new("cargo")
-            .args(&["test", "--manifest-path", "pyo3-macros-backend/Cargo.toml"])
+            .args(&[
+                "nextest",
+                "run",
+                "--manifest-path",
+                "pyo3-macros-backend/Cargo.toml",
+            ])
             .envs(&env),
     )?;
     cli::run(
         Command::new("cargo")
-            .args(&["test", "--manifest-path", "pyo3-macros/Cargo.toml"])
+            .args(&[
+                "nextest",
+                "run",
+                "--manifest-path",
+                "pyo3-macros/Cargo.toml",
+            ])
             .envs(&env),
     )?;
 
-    cli::run(Command::new("cargo").arg("test").envs(&env))?;
+    cli::run(Command::new("cargo").arg("nextest").arg("run").envs(&env))?;
     cli::run(
         Command::new("cargo")
-            .args(&["test", "--features", "abi3"])
+            .args(&["nextest", "run", "--features", "abi3"])
             .envs(&env),
     )?;
     cli::run(
         Command::new("cargo")
-            .args(&["test", "--features", "full"])
+            .args(&["nextest", "run", "--features", "full"])
             .envs(&env),
     )?;
     cli::run(
         Command::new("cargo")
-            .args(&["test", "--features", "abi3 full"])
+            .args(&["nextest", "run", "--features", "abi3 full"])
             .envs(&env),
     )?;
 
     crate::pytests::run(&env)?;
 
     match opts.output_lcov {
-        Some(path) => {
-            cli::run(llvm_cov_command(&["--no-run", "--lcov", "--output-path", &path]).envs(&env))?
-        }
-        None => cli::run(llvm_cov_command(&["--no-run", "--summary-only"]).envs(&env))?,
+        Some(path) => cli::run(
+            llvm_cov_command(&["nextest", "--no-run", "--lcov", "--output-path", &path]).envs(&env),
+        )?,
+        None => cli::run(llvm_cov_command(&["nextest", "--no-run", "--summary-only"]).envs(&env))?,
     }
 
     Ok(())