From 5e52055e3c113714295ddade43712274150dd87f Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Mon, 29 Apr 2024 11:18:03 +0200 Subject: [PATCH 1/4] [mkl,eigen] Refactored interdependent setup. - [cmake] Also use MKLs OpenMP implementation for serial configs. - [cmake] Renamed OGS_USE_PETSC_MKL_EIGEN_OPENMP to OGS_USE_MKL_EIGEN_OPENMP. - [cmake] Option OGS_EIGEN_PARALLEL_BACKEND replaces other configs. Removed: - OGS_USE_MKL_EIGEN_OPENMP - OGS_EIGEN_USE_MKL - Also when OGS_USE_MKL=ON then OGS_EIGEN_PARALLEL_BACKEND defaults to MKL. - Updated eigen-unsupported version to inlude fix for MKL backend. Fix: https://gitlab.com/libeigen/eigen/-/merge_requests/851 - Moved OGS_EIGEN_PARALLEL_BACKEND initialization. - [cmake] Only link to OpenMP when eigen backend is set to OpenMP. - [cmake] Eigen includes OpenMP or MKL depending on backend. --- CMakeLists.txt | 4 -- MathLib/CMakeLists.txt | 17 ++++---- scripts/cmake/Dependencies.cmake | 17 +++++++- scripts/cmake/Find.cmake | 42 ++++++++++++------- .../advanced/configuration-options/index.md | 2 +- web/data/versions.json | 2 +- 6 files changed, 54 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ee7c8907db..63d265dc27a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,10 +129,6 @@ else() ${OGS_EIGEN_DYNAMIC_SHAPE_MATRICES} ) endif() - -cmake_dependent_option( - OGS_EIGEN_USE_MKL "Defines EIGEN_USE_MKL_ALL." OFF "OGS_USE_MKL" OFF -) # Eigen End # Debug diff --git a/MathLib/CMakeLists.txt b/MathLib/CMakeLists.txt index 8161718785c..5bc7d73b72f 100644 --- a/MathLib/CMakeLists.txt +++ b/MathLib/CMakeLists.txt @@ -27,13 +27,14 @@ set_target_properties(MathLib PROPERTIES LINKER_LANGUAGE CXX) target_link_libraries( MathLib - PUBLIC BaseLib - $<$:LAPACK::LAPACK> - $<$:${LIS_LIBRARIES}> - $<$:CVODE::CVODE> - $<$:PkgConfig::PETSC> - Eigen3::Eigen - $<$:OpenMP::OpenMP_CXX> + PUBLIC + BaseLib + $<$:LAPACK::LAPACK> + $<$:${LIS_LIBRARIES}> + $<$:CVODE::CVODE> + $<$:PkgConfig::PETSC> + Eigen3::Eigen + $<$,$>>:OpenMP::OpenMP_CXX> PRIVATE $<$:MKL::MKL> ) if(OGS_USE_MKL AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") @@ -56,7 +57,7 @@ target_compile_definitions( $<$:USE_EIGEN_UNSUPPORTED> $<$:EIGEN_INITIALIZE_MATRICES_BY_NAN> $<$:EIGEN_INITIALIZE_MATRICES_BY_NAN> - $<$,$>:EIGEN_USE_MKL_ALL> + $<$,$>>:EIGEN_USE_MKL_ALL> ) if(OGS_USE_MKL) set_source_files_properties( diff --git a/scripts/cmake/Dependencies.cmake b/scripts/cmake/Dependencies.cmake index 7a957fe0ebc..fa9ae04bd3b 100644 --- a/scripts/cmake/Dependencies.cmake +++ b/scripts/cmake/Dependencies.cmake @@ -113,6 +113,13 @@ set(_eigen_version ${ogs.minimum_version.eigen}) set(_eigen_url https://gitlab.com/libeigen/eigen/-/archive/${_eigen_version}/eigen-${_eigen_version}.tar.gz ) +if(OGS_EIGEN_PARALLEL_BACKEND STREQUAL "MKL" AND NOT OGS_USE_EIGEN_UNSUPPORTED) + message( + FATAL_ERROR + "OGS_EIGEN_PARALLEL_BACKEND=MKL requires OGS_USE_EIGEN_UNSUPPORTED!" + ) +endif() + if(OGS_USE_EIGEN_UNSUPPORTED) set(_eigen_version 3.4.90) set(_eigen_url @@ -133,8 +140,16 @@ if(Eigen3_ADDED) add_library(Eigen3::Eigen INTERFACE IMPORTED) target_include_directories( Eigen3::Eigen SYSTEM INTERFACE ${Eigen3_SOURCE_DIR} - ${OpenMP_CXX_INCLUDE_DIRS} ) + if(${OGS_EIGEN_PARALLEL_BACKEND} STREQUAL "OpenMP") + target_include_directories( + Eigen3::Eigen SYSTEM INTERFACE ${OpenMP_CXX_INCLUDE_DIRS} + ) + else() + target_include_directories( + Eigen3::Eigen SYSTEM INTERFACE ${MKL_INCLUDE} + ) + endif() endif() if(OGS_USE_MFRONT) diff --git a/scripts/cmake/Find.cmake b/scripts/cmake/Find.cmake index f61f6be02fc..e2616a71a2e 100644 --- a/scripts/cmake/Find.cmake +++ b/scripts/cmake/Find.cmake @@ -59,26 +59,34 @@ endif() # geotiff ## find_package(GEOTIFF) -cmake_dependent_option( - OGS_USE_PETSC_MKL_EIGEN_OPENMP - "When PETSc and MKL is used, shall OpenMP be used for Eigen (or Intels iomp if false (default))?" - OFF - "OGS_USE_PETSC;OGS_USE_MKL" - OFF +# blas / lapack / MKL +set(_eigen_parallel_backend OpenMP MKL) +if(OGS_USE_MKL) + set(OGS_EIGEN_PARALLEL_BACKEND MKL + CACHE STRING "The parallelization backend of Eigen." + ) +else() + set(OGS_EIGEN_PARALLEL_BACKEND OpenMP + CACHE STRING "The parallelization backend of Eigen." + ) +endif() +set_property( + CACHE OGS_EIGEN_PARALLEL_BACKEND PROPERTY STRINGS + ${_eigen_parallel_backend} ) -if(NOT (OGS_USE_PETSC AND OGS_USE_MKL) OR OGS_USE_PETSC_MKL_EIGEN_OPENMP) +if(OGS_EIGEN_PARALLEL_BACKEND STREQUAL "OpenMP") # this pulls in libgomp dependency, when MKL is enabled libiomp5 is used. find_package(OpenMP COMPONENTS C CXX) endif() -# blas / lapack / MKL if(OGS_USE_MKL) if(APPLE) set(_mac_ld_prefix "DY") endif() - if(NOT GUIX_BUILD AND (NOT DEFINED ENV{MKLROOT} OR - (NOT "$ENV{${_mac_ld_prefix}LD_LIBRARY_PATH}" - MATCHES "intel" AND NOT WIN32)) + if(NOT GUIX_BUILD + AND (NOT DEFINED ENV{MKLROOT} + OR (NOT "$ENV{${_mac_ld_prefix}LD_LIBRARY_PATH}" MATCHES "intel" + AND NOT WIN32)) ) message( FATAL_ERROR @@ -115,13 +123,17 @@ if(OGS_USE_MKL) endif() set(PKG_CONFIG_ARGN "--define-variable=MKLROOT=${MKLROOT}") set(ENV{PKG_CONFIG_PATH} ${MKLROOT}/bin/pkgconfig) - # TODO: Using -seq instead of -iomp; library iomp5 is missing - # See https://gitlab.opengeosys.org/ogs/inf/guix-ogs/-/issues/1 - pkg_search_module(MKL REQUIRED IMPORTED_TARGET mkl-dynamic-${MKL_INTERFACE}-seq) + # TODO: Using -seq instead of -iomp; library iomp5 is missing See + # https://gitlab.opengeosys.org/ogs/inf/guix-ogs/-/issues/1 + pkg_search_module( + MKL REQUIRED IMPORTED_TARGET mkl-dynamic-${MKL_INTERFACE}-seq + ) add_library(MKL::MKL ALIAS PkgConfig::MKL) else() find_package(MKL CONFIG REQUIRED PATHS $ENV{MKLROOT} ${MKLROOT}) - find_file(MKL_SETVARS setvars.sh PATHS ${MKL_ROOT}/../.. NO_DEFAULT_PATH) + find_file(MKL_SETVARS setvars.sh PATHS ${MKL_ROOT}/../.. + NO_DEFAULT_PATH + ) endif() endif() diff --git a/web/content/docs/devguide/advanced/configuration-options/index.md b/web/content/docs/devguide/advanced/configuration-options/index.md index 581afed3c49..6b054bf6d26 100644 --- a/web/content/docs/devguide/advanced/configuration-options/index.md +++ b/web/content/docs/devguide/advanced/configuration-options/index.md @@ -52,4 +52,4 @@ CMake switches to enable / disable parts of OGS. To enable 64-bit array indices in MKL add `-DMKL_INTERFACE=ilp64` on the first CMake run (with a clean CMake cache) but this seems [not supported by Eigen](https://libeigen.gitlab.io/docs/TopicUsingIntelMKL.html). -- `OGS_USE_PETSC_MKL_EIGEN_OPENMP` - Also enables OpenMP multithreading when PETSc and MKL is used. This pulls in another OpenMP implementation besides the Intel MKL OpenMP. **Experimental feature!** +- `OGS_EIGEN_PARALLEL_BACKEND` - Defaults to `OpenMP`. Defaults to `MKL` when `OGS_USE_MKL=ON`. May be set to `OpenMP` when MKL is on which also enables OpenMP multithreading. This pulls in another OpenMP implementation besides the Intel MKL OpenMP which is an **experimental feature!**. If you want to use OpenMP parallelized assembly with MKL enabled you also need to explicitly set `OGS_EIGEN_PARALLEL_BACKEND=OpenMP`! diff --git a/web/data/versions.json b/web/data/versions.json index 453a98337bc..14a01fd47d4 100644 --- a/web/data/versions.json +++ b/web/data/versions.json @@ -14,7 +14,7 @@ "hugo": "0.122.0", "boost": "1.73.0", "eigen": "3.4.0", - "eigen-unsupported": "9441d94dccccd5db8d64179516fdc5b53994a047", + "eigen-unsupported": "81e012c909475ab945a6730ec098b5bee7548184", "vtk": "9.3.0", "petsc": "3.18.6", "qt": "5.14.2", From 48a8958785c412f08a32f2efd6ab57da92774308 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 7 May 2024 09:45:12 +0200 Subject: [PATCH 2/4] [cmake] Fixed a warning regarding search paths. Python should be copied to lib-folder at install time, not at configure time. file(INSTALL ..) is an alias to file(COPY ...) and therefore a bit misleading, use install(FILE ...) instead! --- scripts/cmake/packaging/Pack.cmake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/cmake/packaging/Pack.cmake b/scripts/cmake/packaging/Pack.cmake index df2dfd61a62..11e827756ae 100644 --- a/scripts/cmake/packaging/Pack.cmake +++ b/scripts/cmake/packaging/Pack.cmake @@ -76,9 +76,7 @@ if(WIN32) DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} ) else() - file(INSTALL ${Python_LIBRARIES} DESTINATION ${CMAKE_INSTALL_LIBDIR} - FOLLOW_SYMLINK_CHAIN - ) + install(FILES ${Python_LIBRARIES} DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() configure_file(Documentation/README.txt.in ${PROJECT_BINARY_DIR}/README.txt) From bd49b86513fc2bd4eee0819a5dc6cb4f1831dc05 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Wed, 8 May 2024 13:23:22 +0200 Subject: [PATCH 3/4] Update eigen to fix dupl. symbols error. Also added , former code implicitly used an assert() function from eigen which was recently renamed to eigen_assert(). --- MathLib/LinAlg/Eigen/EigenMatrix.h | 1 + MathLib/Point3d.h | 1 + web/data/versions.json | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/MathLib/LinAlg/Eigen/EigenMatrix.h b/MathLib/LinAlg/Eigen/EigenMatrix.h index d69ee834f4f..a7644e07817 100644 --- a/MathLib/LinAlg/Eigen/EigenMatrix.h +++ b/MathLib/LinAlg/Eigen/EigenMatrix.h @@ -11,6 +11,7 @@ #pragma once #include +#include #include #include diff --git a/MathLib/Point3d.h b/MathLib/Point3d.h index 0aeafed2da8..2eb46ee2252 100644 --- a/MathLib/Point3d.h +++ b/MathLib/Point3d.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include #include "mathlib_export.h" diff --git a/web/data/versions.json b/web/data/versions.json index 14a01fd47d4..4e19cd3ae67 100644 --- a/web/data/versions.json +++ b/web/data/versions.json @@ -14,7 +14,7 @@ "hugo": "0.122.0", "boost": "1.73.0", "eigen": "3.4.0", - "eigen-unsupported": "81e012c909475ab945a6730ec098b5bee7548184", + "eigen-unsupported": "9000b3767770f6dd0f4cfb12f4e19c71921885a4", "vtk": "9.3.0", "petsc": "3.18.6", "qt": "5.14.2", From 91732ee59b085d50170b04d45fe3a2dc5e7ce23f Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 7 May 2024 12:55:31 +0200 Subject: [PATCH 4/4] [T] Relax tolerance due to eigen update. --- ProcessLib/HeatConduction/Tests.cmake | 2 +- ProcessLib/RichardsMechanics/Tests.cmake | 6 +++--- Tests/Data/LIE/Mechanics/mohr_coulomb_load_path_nu0p3.prj | 2 +- .../Ehlers/MFront/two_material_ids_single_solid.prj | 6 +++--- Tests/Data/Mechanics/ModifiedCamClay/square_1e0_shear.prj | 2 +- Tests/Data/RichardsMechanics/double_porosity_swelling.prj | 2 +- Tests/Data/RichardsMechanics/orthotropic_swelling_xy.prj | 6 +++--- Tests/Data/TH2M/H2M/OrthotropicSwelling/square.prj | 6 +++--- .../Confined_Compression/HM_confined_compression_gas.prj | 4 ++-- .../HM_confined_compression_liquid.prj | 4 ++-- Tests/Data/TH2M/HM/flow_fully_saturated.prj | 8 ++++---- Tests/Data/TH2M/HM/flow_fully_saturated_gas.prj | 8 ++++---- 12 files changed, 28 insertions(+), 28 deletions(-) diff --git a/ProcessLib/HeatConduction/Tests.cmake b/ProcessLib/HeatConduction/Tests.cmake index e41ebb662c7..5277158dec8 100644 --- a/ProcessLib/HeatConduction/Tests.cmake +++ b/ProcessLib/HeatConduction/Tests.cmake @@ -379,7 +379,7 @@ AddTest( TESTER vtkdiff DIFF_DATA square_1e2_axi_ts_10_t_1.000000.vtu square_1e2_axi_ts_10_t_1.000000.vtu temperature temperature 2e-15 0 - square_1e2_axi_ts_10_t_1.000000.vtu square_1e2_axi_ts_10_t_1.000000.vtu heat_flux heat_flux 1e-14 0 + square_1e2_axi_ts_10_t_1.000000.vtu square_1e2_axi_ts_10_t_1.000000.vtu heat_flux heat_flux 2e-14 0 REQUIREMENTS NOT OGS_USE_MPI ) # WEDGE 1x1 HEAT CONDUCTION TEST -- same setup as above test but in cartesian coordinates diff --git a/ProcessLib/RichardsMechanics/Tests.cmake b/ProcessLib/RichardsMechanics/Tests.cmake index dc9ba762df6..1a4e6ea8af6 100644 --- a/ProcessLib/RichardsMechanics/Tests.cmake +++ b/ProcessLib/RichardsMechanics/Tests.cmake @@ -60,9 +60,9 @@ AddTest( confined_compression_fully_saturated_ts_120_t_1000.000000.vtu confined_compression_fully_saturated_restart_ts_100_t_1000.000000.vtu epsilon epsilon 5e-14 0 confined_compression_fully_saturated_ts_420_t_4000.000000.vtu confined_compression_fully_saturated_restart_ts_400_t_4000.000000.vtu epsilon epsilon 5e-14 0 - confined_compression_fully_saturated_ts_20_t_100.000000.vtu confined_compression_fully_saturated_restart_ts_0_t_100.000000.vtu saturation saturation 2.5e-15 0 - confined_compression_fully_saturated_ts_120_t_1000.000000.vtu confined_compression_fully_saturated_restart_ts_100_t_1000.000000.vtu saturation saturation 2.5e-15 0 - confined_compression_fully_saturated_ts_420_t_4000.000000.vtu confined_compression_fully_saturated_restart_ts_400_t_4000.000000.vtu saturation saturation 2.5e-15 0 + confined_compression_fully_saturated_ts_20_t_100.000000.vtu confined_compression_fully_saturated_restart_ts_0_t_100.000000.vtu saturation saturation 3e-15 0 + confined_compression_fully_saturated_ts_120_t_1000.000000.vtu confined_compression_fully_saturated_restart_ts_100_t_1000.000000.vtu saturation saturation 3e-15 0 + confined_compression_fully_saturated_ts_420_t_4000.000000.vtu confined_compression_fully_saturated_restart_ts_400_t_4000.000000.vtu saturation saturation 3e-15 0 confined_compression_fully_saturated_ts_20_t_100.000000.vtu confined_compression_fully_saturated_restart_ts_0_t_100.000000.vtu velocity velocity 1e-16 0 confined_compression_fully_saturated_ts_120_t_1000.000000.vtu confined_compression_fully_saturated_restart_ts_100_t_1000.000000.vtu velocity velocity 1e-16 0 diff --git a/Tests/Data/LIE/Mechanics/mohr_coulomb_load_path_nu0p3.prj b/Tests/Data/LIE/Mechanics/mohr_coulomb_load_path_nu0p3.prj index d3f637d55c3..41e84ecb3f3 100644 --- a/Tests/Data/LIE/Mechanics/mohr_coulomb_load_path_nu0p3.prj +++ b/Tests/Data/LIE/Mechanics/mohr_coulomb_load_path_nu0p3.prj @@ -308,7 +308,7 @@ out_mohr_coulomb_load_path_nu0p3_ts_(.*)_t_2.500000.vtu sigma - 7.0e-3 + 8.0e-3 0 diff --git a/Tests/Data/Mechanics/Ehlers/MFront/two_material_ids_single_solid.prj b/Tests/Data/Mechanics/Ehlers/MFront/two_material_ids_single_solid.prj index 66bd7ecbdf0..14b838883dd 100644 --- a/Tests/Data/Mechanics/Ehlers/MFront/two_material_ids_single_solid.prj +++ b/Tests/Data/Mechanics/Ehlers/MFront/two_material_ids_single_solid.prj @@ -208,13 +208,13 @@ two_material_ids_single_solid_t_.*.vtu sigma - 2e-13 + 3e-13 0 two_material_ids_single_solid_t_.*.vtu epsilon - 2e-15 + 3e-15 0 @@ -232,7 +232,7 @@ two_material_ids_single_solid_t_.*.vtu NodalForces - 5e-13 + 6e-13 0 diff --git a/Tests/Data/Mechanics/ModifiedCamClay/square_1e0_shear.prj b/Tests/Data/Mechanics/ModifiedCamClay/square_1e0_shear.prj index 234ce8cc65d..69f3aa85238 100644 --- a/Tests/Data/Mechanics/ModifiedCamClay/square_1e0_shear.prj +++ b/Tests/Data/Mechanics/ModifiedCamClay/square_1e0_shear.prj @@ -235,7 +235,7 @@ square_1e0_shear_ts_100_t_1.000000.vtu sigma 1e-12 - 1e-13 + 2e-13 diff --git a/Tests/Data/RichardsMechanics/double_porosity_swelling.prj b/Tests/Data/RichardsMechanics/double_porosity_swelling.prj index 2949eecc1fd..e35add3e617 100644 --- a/Tests/Data/RichardsMechanics/double_porosity_swelling.prj +++ b/Tests/Data/RichardsMechanics/double_porosity_swelling.prj @@ -352,7 +352,7 @@ double_porosity_swelling_.*.vtu porosity - 2e-5 + 2.1e-5 0 diff --git a/Tests/Data/RichardsMechanics/orthotropic_swelling_xy.prj b/Tests/Data/RichardsMechanics/orthotropic_swelling_xy.prj index b25c3885d5b..8379c4333ba 100644 --- a/Tests/Data/RichardsMechanics/orthotropic_swelling_xy.prj +++ b/Tests/Data/RichardsMechanics/orthotropic_swelling_xy.prj @@ -324,8 +324,8 @@ orthotropic_swelling_xy_ts_.*.vtu saturation - 2e-15 - 1e-15 + 4e-15 + 4e-15 orthotropic_swelling_xy_ts_.*.vtu @@ -342,7 +342,7 @@ orthotropic_swelling_xy_ts_.*.vtu swelling_stress - 1e-15 + 2e-15 0 diff --git a/Tests/Data/TH2M/H2M/OrthotropicSwelling/square.prj b/Tests/Data/TH2M/H2M/OrthotropicSwelling/square.prj index 0726ab51ab7..de0f4825519 100644 --- a/Tests/Data/TH2M/H2M/OrthotropicSwelling/square.prj +++ b/Tests/Data/TH2M/H2M/OrthotropicSwelling/square.prj @@ -464,8 +464,8 @@ square_ts_.*.vtu saturation - 2e-15 - 1e-15 + 3e-15 + 4e-15 square_ts_.*.vtu @@ -482,7 +482,7 @@ square_ts_.*.vtu swelling_stress - 1e-15 + 2e-15 0 diff --git a/Tests/Data/TH2M/HM/Confined_Compression/HM_confined_compression_gas.prj b/Tests/Data/TH2M/HM/Confined_Compression/HM_confined_compression_gas.prj index 91676a26672..193620a6d76 100644 --- a/Tests/Data/TH2M/HM/Confined_Compression/HM_confined_compression_gas.prj +++ b/Tests/Data/TH2M/HM/Confined_Compression/HM_confined_compression_gas.prj @@ -515,8 +515,8 @@ HM_confined_compression_gas_ts_.*.vtu porosity - 1e-15 - 1e-15 + 2e-15 + 2e-15 HM_confined_compression_gas_ts_.*.vtu diff --git a/Tests/Data/TH2M/HM/Confined_Compression/HM_confined_compression_liquid.prj b/Tests/Data/TH2M/HM/Confined_Compression/HM_confined_compression_liquid.prj index 97001751570..7791d571bf4 100644 --- a/Tests/Data/TH2M/HM/Confined_Compression/HM_confined_compression_liquid.prj +++ b/Tests/Data/TH2M/HM/Confined_Compression/HM_confined_compression_liquid.prj @@ -522,8 +522,8 @@ HM_confined_compression_liquid_ts_.*.vtu porosity - 1e-15 - 1e-15 + 2e-15 + 2e-15 HM_confined_compression_liquid_ts_.*.vtu diff --git a/Tests/Data/TH2M/HM/flow_fully_saturated.prj b/Tests/Data/TH2M/HM/flow_fully_saturated.prj index 36903d0786e..4c1072d2563 100644 --- a/Tests/Data/TH2M/HM/flow_fully_saturated.prj +++ b/Tests/Data/TH2M/HM/flow_fully_saturated.prj @@ -468,14 +468,14 @@ flow_fully_saturated_ts_.*.vtu liquid_density - 3e-15 - 3e-15 + 4e-15 + 4e-15 flow_fully_saturated_ts_.*.vtu gas_density - 3e-15 - 3e-15 + 4e-15 + 4e-15 flow_fully_saturated_ts_.*.vtu diff --git a/Tests/Data/TH2M/HM/flow_fully_saturated_gas.prj b/Tests/Data/TH2M/HM/flow_fully_saturated_gas.prj index bc0ff250642..d6207efe014 100644 --- a/Tests/Data/TH2M/HM/flow_fully_saturated_gas.prj +++ b/Tests/Data/TH2M/HM/flow_fully_saturated_gas.prj @@ -519,14 +519,14 @@ flow_fully_saturated_gas_ts_.*.vtu liquid_density - 3e-15 - 3e-15 + 4e-15 + 4e-15 flow_fully_saturated_gas_ts_.*.vtu gas_density - 3e-15 - 3e-15 + 4e-15 + 4e-15 flow_fully_saturated_gas_ts_.*.vtu