diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ead99e4cdfd..4a4e2b3c215 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,7 +84,7 @@ repos: hooks: - id: ruff types_or: [python, pyi, jupyter] - args: [--show-source] + args: [--output-format=full] # A variant of the ruff hook above that automatically applies proposed fixes. # Runs only manually. # Run this hook (and all other manual hooks if any) with: @@ -94,7 +94,7 @@ repos: hooks: - id: ruff types_or: [python, pyi, jupyter] - args: [--show-source, --fix, --exit-non-zero-on-fix] + args: [--output-format=full, --fix, --exit-non-zero-on-fix] stages: [manual] # Enable on demand, run with # pre-commit run --all-files clang-format diff --git a/Applications/Utils/Tests.cmake b/Applications/Utils/Tests.cmake index fac5f9084ee..55f99585035 100644 --- a/Applications/Utils/Tests.cmake +++ b/Applications/Utils/Tests.cmake @@ -1423,9 +1423,11 @@ AddTest( ) if(NOT OGS_USE_PETSC) - NotebookTest(NOTEBOOKFILE ../../web/content/docs/tutorials/bhe_meshing/notebook-bhe_meshing.md - PYTHON_PACKAGES openpyxl - RUNTIME 10) + if(NOT WIN32) + NotebookTest(NOTEBOOKFILE ../../web/content/docs/tutorials/bhe_meshing/notebook-bhe_meshing.md + PYTHON_PACKAGES openpyxl + RUNTIME 10) + endif() NotebookTest(NOTEBOOKFILE ../../web/content/docs/tutorials/mesh_tutorial/notebook-mesh_tutorial.md RUNTIME 10) NotebookTest(NOTEBOOKFILE ../../web/content/docs/tutorials/Inclined_bhe_meshing/notebook-inclined_bhe_meshing.md diff --git a/ProcessLib/ComponentTransport/Tests.cmake b/ProcessLib/ComponentTransport/Tests.cmake index 3d014099242..8a6c7cff766 100644 --- a/ProcessLib/ComponentTransport/Tests.cmake +++ b/ProcessLib/ComponentTransport/Tests.cmake @@ -832,7 +832,7 @@ if(NOT OGS_USE_PETSC) OgsTest(PROJECTFILE Parabolic/ComponentTransport/FCT_test/1d_step_func.prj RUNTIME 5) endif() -if(NOT OGS_USE_PETSC) +if(NOT OGS_USE_PETSC AND NOT WIN32) NotebookTest( NOTEBOOKFILE Parabolic/ComponentTransport/ReactiveTransport/DecayChain/GlobalImplicitApproach/performance_measurements.ipynb RUNTIME 200 diff --git a/Tests/Data/HydroMechanics/StaggeredScheme/ConsolidationBenchmark/consolidation_benchmark.prj b/Tests/Data/HydroMechanics/StaggeredScheme/ConsolidationBenchmark/consolidation_benchmark.prj index 902dc7490ac..91935914612 100644 --- a/Tests/Data/HydroMechanics/StaggeredScheme/ConsolidationBenchmark/consolidation_benchmark.prj +++ b/Tests/Data/HydroMechanics/StaggeredScheme/ConsolidationBenchmark/consolidation_benchmark.prj @@ -300,7 +300,7 @@ ConsolidationBenchmark_ts_12_t_6.000000.vtu epsilon - 1e-15 + 2e-15 1e-15 diff --git a/Tests/Data/HydroMechanics/StaggeredScheme/ConsolidationBenchmark/consolidation_benchmark_mono.prj b/Tests/Data/HydroMechanics/StaggeredScheme/ConsolidationBenchmark/consolidation_benchmark_mono.prj index b71ec0f34d3..7222cf0a5a6 100644 --- a/Tests/Data/HydroMechanics/StaggeredScheme/ConsolidationBenchmark/consolidation_benchmark_mono.prj +++ b/Tests/Data/HydroMechanics/StaggeredScheme/ConsolidationBenchmark/consolidation_benchmark_mono.prj @@ -238,7 +238,7 @@ ConsolidationBenchmark_mono_ts_6_t_6.000000.vtu epsilon - 1e-15 + 2e-15 1e-15 diff --git a/Tests/Data/Notebooks/testrunner.py b/Tests/Data/Notebooks/testrunner.py index 59fc58f91a9..71ca7ad1a18 100644 --- a/Tests/Data/Notebooks/testrunner.py +++ b/Tests/Data/Notebooks/testrunner.py @@ -9,10 +9,11 @@ import jupytext import nbformat +import papermill import toml from nbclient.exceptions import DeadKernelError from nbconvert import HTMLExporter -from nbconvert.preprocessors import CellExecutionError, ExecutePreprocessor +from nbconvert.preprocessors import CellExecutionError def save_to_website(exec_notebook_file): @@ -108,11 +109,7 @@ def check_and_modify_frontmatter(): src="https://upload.wikimedia.org/wikipedia/commons/3/38/Jupyter_logo.svg" alt=""> This page is based on a Jupyter notebook.""" if is_jupytext: - download_file_name = ( - Path(convert_notebook_file) - .rename(Path(convert_notebook_file).with_suffix(".ipynb")) - .name - ) + download_file_name = Path(convert_notebook_file).name text += f""" """ text += f""" @@ -186,17 +183,27 @@ def check_and_modify_frontmatter(): else: with notebook_file_path.open(encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) - ep = ExecutePreprocessor(kernel_name="python3") # Run the notebook print(f"[Start] {notebook_filename}") start = timer() try: - ep.preprocess(nb, {"metadata": {"path": notebook_file_path.parent}}) + # Run with papermill instead of nbconvert for printing notebook + # outputs on the command line + nb = papermill.execute.execute_notebook( + nb, + None, + kernel_name="python3", + cwd=notebook_file_path.parent, + log_output=True, + progress_bar=False, + stdout_file=sys.stdout, + stderr_file=sys.stderr, + ) except DeadKernelError: out = None - msg = 'Error executing the notebook "%s".\n\n' % notebook_filename - msg += 'See notebook "%s" for the traceback.' % convert_notebook_file + msg = f'Error executing the notebook "{notebook_filename}".\n\n' + msg += f'See notebook "{convert_notebook_file}" for the traceback.' print(msg) notebook_success = False with convert_notebook_file.open(mode="w", encoding="utf-8") as f: @@ -204,6 +211,7 @@ def check_and_modify_frontmatter(): except CellExecutionError: notebook_success = False end = timer() + print(f"[End] {notebook_filename}") # Write new notebook with convert_notebook_file.open(mode="w", encoding="utf-8") as f: diff --git a/Tests/Data/requirements-dev.txt b/Tests/Data/requirements-dev.txt index 3d158732b79..8362ab3cc62 100644 --- a/Tests/Data/requirements-dev.txt +++ b/Tests/Data/requirements-dev.txt @@ -2,3 +2,4 @@ nbconvert==7.2.9 nbformat==5.7.3 toml==0.10.2 jupytext==1.14.5 +papermill==2.6.0 diff --git a/pyproject.toml b/pyproject.toml index 22c897a893e..036b6d9a14b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ build = ["cp3{9,10,11,12}-*"] skip = ["*musllinux*"] test-extras = "test" test-command = "pytest {project}/Tests/Python" -build-verbosity = "1" +build-verbosity = 1 [tool.cibuildwheel.linux] # quay.io/pypa/manylinux_2_28 works too @@ -85,6 +85,11 @@ environment-pass = [ "cmake.args" = "-G;Visual Studio 16 2019;--preset;wheel" [tool.ruff] +include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"] +exclude = ["scripts/doc", "*ci-skip*"] +line-length = 80 + +[tool.ruff.lint] select = [ "E", "F", @@ -120,21 +125,16 @@ extend-ignore = [ # https://github.com/charliermarsh/ruff/issues/2142: "RUF005", ] -target-version = "py39" typing-modules = ["mypackage._compat.typing"] unfixable = [ "T20", # Removes print statements "F841", # Removes unused variables ] -include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"] -exclude = ["scripts/doc", "*ci-skip*"] flake8-unused-arguments.ignore-variadic-names = true -line-length = 80 -[tool.ruff.lint] # allow en-dash allowed-confusables = ["–"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "tests/**" = ["T20"] "*.ipynb" = ["E402"] # Top-level imports diff --git a/scripts/ci/extends/template-build-win.yml b/scripts/ci/extends/template-build-win.yml index 851494877c0..aca415906b3 100644 --- a/scripts/ci/extends/template-build-win.yml +++ b/scripts/ci/extends/template-build-win.yml @@ -33,7 +33,11 @@ if($env:BUILD_CTEST -eq "true" -And $ctest_condition -eq $true) { ctest --preset=$env:CMAKE_PRESET -LE large --output-junit Tests/ctest.xml -M Experimental --group $ctest_group --test-dir $build_directory_full -T Test --no-tests=error + $ctestExitCode = $LASTEXITCODE ctest --test-dir $build_directory_full -T submit; $null + if ($ctestExitCode -ne 0) { + exit $ctestExitCode + } } - | if($env:CHECK_WARNINGS -eq "true" -and (cat $log_file | Select-String -Pattern ': warning') ) diff --git a/scripts/ci/jobs/code-quality.yml b/scripts/ci/jobs/code-quality.yml index 98a444f8b2c..eaca2461039 100644 --- a/scripts/ci/jobs/code-quality.yml +++ b/scripts/ci/jobs/code-quality.yml @@ -21,8 +21,8 @@ ruff: needs: [] tags: [envinf] script: - - pipx run ruff --exit-zero . - - pipx run ruff --exit-zero --output-format gitlab . > ruff-report.json + - pipx run ruff check --exit-zero . + - pipx run ruff check --exit-zero --output-format gitlab . > ruff-report.json artifacts: reports: codequality: ruff-report.json diff --git a/scripts/ci/jobs/jupyter.yml b/scripts/ci/jobs/jupyter.yml index 0e2407e670a..30d10948abb 100644 --- a/scripts/ci/jobs/jupyter.yml +++ b/scripts/ci/jobs/jupyter.yml @@ -30,7 +30,7 @@ test notebooks via wheel: # status file and sh -c to workaround xvfb-run problems: # /usr/bin/xvfb-run: line 186: kill: () - No such process" find . -type f -iname '*.ipynb' \ - | grep -vP '\.ipynb_checkpoints|\.ci-skip\.ipynb$|_out|\.venv|PhaseField|mtest' \ + | grep -vP '\.ipynb_checkpoints|\.ci-skip\.ipynb$|_out|\.venv|PhaseField|mtest|sen_shear|PETSc' \ | xargs xvfb-run -a \ sh -c 'statf="$1"; shift; "$@" || echo "$?" >"$statf"' \ -- "$status_file" \