diff --git a/Apps/Regrid_Util.F90 b/Apps/Regrid_Util.F90 index 072ab7373b96..9906beed1586 100644 --- a/Apps/Regrid_Util.F90 +++ b/Apps/Regrid_Util.F90 @@ -26,6 +26,7 @@ module regrid_util_support_mod integer :: deflate, shave integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level logical :: use_weights contains procedure :: create_grid @@ -98,6 +99,7 @@ subroutine process_command_line(this,rc) this%deflate=0 this%quantize_algorithm=0 this%quantize_level=0 + this%zstandard_level=0 this%use_weights = .false. nargs = command_argument_count() do i=1,nargs @@ -161,6 +163,9 @@ subroutine process_command_line(this,rc) case('-quantize_level') call get_command_argument(i+1,astr) read(astr,*)this%quantize_level + case('-zstandard_level') + call get_command_argument(i+1,astr) + read(astr,*)this%zstandard_level case('-file_weights') this%use_weights = .true. case('--help') @@ -432,7 +437,7 @@ subroutine main() call ESMF_ClockSet(clock,currtime=time,_RC) if (.not. writer_created) then - call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,_RC) + call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,zstandard_level=support%zstandard_level,_RC) writer_created=.true. end if diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c5c4fdae965..6e5c62c38ab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +## [2.49.0] - 2024-10-04 + +### Added + +- Added zstandard compression support + - Note this requires netCDF-C to have been compiled with zstandard support. We have a CMake test to check for this + and enabling zstandard output in History will fail if the library does not support it + +### Changed + +- ExtDataDriver.x now uses ExtData2G by default +- Update `components.yaml` + - ecbuild geos/v1.4.0 + - Fixes bug between GCC, macOS, and the `-pipe` flag + +### Fixed + +- Workaround for NVHPC 24.9 involving `use` statement in `block` construct + ## [2.48.0] - 2024-09-24 ### Added diff --git a/CMakeLists.txt b/CMakeLists.txt index 212ffb6e0d23..0f06772e49dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ endif () project ( MAPL - VERSION 2.48.0 + VERSION 2.49.0 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF # Set the possible values of build type for cmake-gui diff --git a/Tests/ExtDataDriverGridComp.F90 b/Tests/ExtDataDriverGridComp.F90 index d4d11b038ed4..6d99db879902 100644 --- a/Tests/ExtDataDriverGridComp.F90 +++ b/Tests/ExtDataDriverGridComp.F90 @@ -224,7 +224,7 @@ subroutine initialize_gc(gc, import_state, export_state, clock, rc) !EOR enableTimers = ESMF_UtilStringUpperCase(enableTimers, rc = status) _VERIFY(status) - call MAPL_GetResource(maplobj,use_extdata2g,"USE_EXTDATA2G:",default=.false.,_RC) + call MAPL_GetResource(maplobj,use_extdata2g,"USE_EXTDATA2G:",default=.true.,_RC) if (enableTimers /= 'YES') then call MAPL_ProfDisable(rc = status) diff --git a/Tests/ExtData_Testing_Framework/CMakeLists.txt b/Tests/ExtData_Testing_Framework/CMakeLists.txt index afcb2b5be93f..4ac289cb31a2 100644 --- a/Tests/ExtData_Testing_Framework/CMakeLists.txt +++ b/Tests/ExtData_Testing_Framework/CMakeLists.txt @@ -29,62 +29,84 @@ set(QUANTIZE_TESTS "case34" ) -foreach(TEST_CASE ${TEST_CASES_1G}) +# We have one test that requires netcdf zstandard support +set(ZSTD_TESTS + "case35" +) + +# We have one test which requires *both* Quantize and ZSTD support +set(QUANTIZE_AND_ZSTD_TESTS + "case36" +) + +file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G) + +foreach(TEST_CASE ${TEST_CASES_2G}) + + # Skip tests that require Quantize support if we don't have it + if (NOT NETCDF_HAS_QUANTIZE AND ${TEST_CASE} IN_LIST QUANTIZE_TESTS) + continue() + endif() + + # Skip tests that require ZSTD support if we don't have it + if (NOT NETCDF_HAS_ZSTD AND ${TEST_CASE} IN_LIST ZSTD_TESTS) + continue() + endif() + + # Skip tests that require both Quantize and ZSTD support if we don't have it + if (NOT (NETCDF_HAS_QUANTIZE AND NETCDF_HAS_ZSTD) AND ${TEST_CASE} IN_LIST QUANTIZE_AND_ZSTD_TESTS) + continue() + endif() + if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) else() set(num_procs "1") endif() add_test( - NAME "ExtData1G_${TEST_CASE}" + NAME "ExtData2G_${TEST_CASE}" COMMAND ${CMAKE_COMMAND} -DTEST_CASE=${TEST_CASE} -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} -DMY_BINARY_DIR=${CMAKE_BINARY_DIR}/bin -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} - -DIS_EXTDATA2G=NO + -DIS_EXTDATA2G=YES -P ${CMAKE_CURRENT_SOURCE_DIR}/run_extdata.cmake ) if (${num_procs} GREATER ${cutoff}) - set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_BIG_TESTS") + set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_BIG_TESTS") elseif (${TEST_CASE} IN_LIST SLOW_TESTS) - set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_SLOW_TESTS") + set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_SLOW_TESTS") else() - set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_SMALL_TESTS") + set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_SMALL_TESTS;ESSENTIAL") endif() endforeach() -file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G) - -foreach(TEST_CASE ${TEST_CASES_2G}) - - # Skip tests that require Quantize support if we don't have it - if (NOT NETCDF_HAS_QUANTIZE AND ${TEST_CASE} IN_LIST QUANTIZE_TESTS) - continue() - endif() +file(STRINGS "test_cases/extdata_1g_cases.txt" TEST_CASES_1G) +foreach(TEST_CASE ${TEST_CASES_1G}) if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) else() set(num_procs "1") endif() add_test( - NAME "ExtData2G_${TEST_CASE}" + NAME "ExtData1G_${TEST_CASE}" COMMAND ${CMAKE_COMMAND} -DTEST_CASE=${TEST_CASE} -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} -DMY_BINARY_DIR=${CMAKE_BINARY_DIR}/bin -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} - -DIS_EXTDATA2G=YES + -DIS_EXTDATA2G=NO -P ${CMAKE_CURRENT_SOURCE_DIR}/run_extdata.cmake ) if (${num_procs} GREATER ${cutoff}) - set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_BIG_TESTS") + set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_BIG_TESTS") elseif (${TEST_CASE} IN_LIST SLOW_TESTS) - set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_SLOW_TESTS") + set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_SLOW_TESTS") else() - set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_SMALL_TESTS;ESSENTIAL") + set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_SMALL_TESTS") endif() endforeach() diff --git a/Tests/ExtData_Testing_Framework/run_extdata.cmake b/Tests/ExtData_Testing_Framework/run_extdata.cmake index e5b8af69d042..ccdd4d003c49 100644 --- a/Tests/ExtData_Testing_Framework/run_extdata.cmake +++ b/Tests/ExtData_Testing_Framework/run_extdata.cmake @@ -10,14 +10,14 @@ macro(run_case CASE) else() set(num_procs "1") endif() - if (${IS_EXTDATA2G} STREQUAL "YES") - file(APPEND "${tempdir}/CAP1.rc" "USE_EXTDATA2G: .true.") - file(APPEND "${tempdir}/CAP2.rc" "USE_EXTDATA2G: .true.") + if (${IS_EXTDATA2G} STREQUAL "NO") + file(APPEND "${tempdir}/CAP1.rc" "USE_EXTDATA2G: .false.") + file(APPEND "${tempdir}/CAP2.rc" "USE_EXTDATA2G: .false.") if (EXISTS "${tempdir}/CAP3.rc") - file(APPEND "${tempdir}/CAP3.rc" "USE_EXTDATA2G: .true.") + file(APPEND "${tempdir}/CAP3.rc" "USE_EXTDATA2G: .false.") endif() if (EXISTS "${tempdir}/CAP4.rc") - file(APPEND "${tempdir}/CAP4.rc" "USE_EXTDATA2G: .true.") + file(APPEND "${tempdir}/CAP4.rc" "USE_EXTDATA2G: .false.") endif() endif() execute_process( diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc new file mode 100644 index 000000000000..83ad27a2c551 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc @@ -0,0 +1,24 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: GenerateExports + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc new file mode 100644 index 000000000000..2e79954523bd --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc @@ -0,0 +1,29 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: CompareImports + +IMPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc new file mode 100644 index 000000000000..680d0ffa9c5b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc @@ -0,0 +1,4 @@ +CASES:: +CAP1.rc +CAP2.rc +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc new file mode 100644 index 000000000000..ce2690d6937b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc @@ -0,0 +1,25 @@ +ROOT_NAME: Root +ROOT_CF: AGCM1.rc +HIST_CF: HISTORY1.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20040115 210000 +20040215 210000 +20040315 210000 +20040415 210000 +20040515 210000 +20040615 210000 +20040715 210000 +20040815 210000 +20040915 210000 +20041015 210000 +20041115 210000 +20041215 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc new file mode 100644 index 000000000000..4e9e1bb95026 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc @@ -0,0 +1,15 @@ +ROOT_NAME: Root +ROOT_CF: AGCM2.rc +HIST_CF: HISTORY2.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20041125 210000 +20041126 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc new file mode 100644 index 000000000000..a45d1dd13f7f --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc @@ -0,0 +1,13 @@ +#CASE_SENSITIVE_VARIABLE_NAMES: .false. +Ext_AllowExtrap: .false. +Prefetch: .true. +#DEBUG_LEVEL: 20 + +PrimaryExports%% +VAR2D NA N N 0 none none VAR2D case1.%y4.nc4 +VAR3D NA N N 0 none none VAR3D case1.%y4.nc4 +%% + + +DerivedExports%% +%% diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc new file mode 100644 index 000000000000..632008de4f12 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc @@ -0,0 +1,14 @@ +GRID_LABELS: +:: + +COLLECTIONS: case1 +:: + + case1.template: '%y4.nc4', + case1.format: 'CFIO', + case1.frequency: 010000, + case1.duration: 000000, + case1.zstandard_level: 2, + case1.fields: 'VAR2D', 'Root', + 'VAR3D', 'Root', + :: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc new file mode 100644 index 000000000000..2895432e995a --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc @@ -0,0 +1,5 @@ +GRID_LABELS: +:: + +COLLECTIONS: +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/README b/Tests/ExtData_Testing_Framework/test_cases/case35/README new file mode 100644 index 000000000000..9a6d7597262d --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/README @@ -0,0 +1 @@ +Case, 12-month/12 time 2004 file with 2 updates, non-climatology diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml b/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml new file mode 100644 index 000000000000..e2ddb90675ab --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml @@ -0,0 +1,5 @@ +Collections: + fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" } +Exports: + VAR2D: {variable: VAR2D, collection: fstream1} + VAR3D: {variable: VAR3D, collection: fstream1} diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc new file mode 100644 index 000000000000..83ad27a2c551 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc @@ -0,0 +1,24 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: GenerateExports + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc new file mode 100644 index 000000000000..2e79954523bd --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc @@ -0,0 +1,29 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: CompareImports + +IMPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc new file mode 100644 index 000000000000..680d0ffa9c5b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc @@ -0,0 +1,4 @@ +CASES:: +CAP1.rc +CAP2.rc +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc new file mode 100644 index 000000000000..ce2690d6937b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc @@ -0,0 +1,25 @@ +ROOT_NAME: Root +ROOT_CF: AGCM1.rc +HIST_CF: HISTORY1.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20040115 210000 +20040215 210000 +20040315 210000 +20040415 210000 +20040515 210000 +20040615 210000 +20040715 210000 +20040815 210000 +20040915 210000 +20041015 210000 +20041115 210000 +20041215 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc new file mode 100644 index 000000000000..4e9e1bb95026 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc @@ -0,0 +1,15 @@ +ROOT_NAME: Root +ROOT_CF: AGCM2.rc +HIST_CF: HISTORY2.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20041125 210000 +20041126 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc new file mode 100644 index 000000000000..a45d1dd13f7f --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc @@ -0,0 +1,13 @@ +#CASE_SENSITIVE_VARIABLE_NAMES: .false. +Ext_AllowExtrap: .false. +Prefetch: .true. +#DEBUG_LEVEL: 20 + +PrimaryExports%% +VAR2D NA N N 0 none none VAR2D case1.%y4.nc4 +VAR3D NA N N 0 none none VAR3D case1.%y4.nc4 +%% + + +DerivedExports%% +%% diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc new file mode 100644 index 000000000000..014b2e83e177 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc @@ -0,0 +1,16 @@ +GRID_LABELS: +:: + +COLLECTIONS: case1 +:: + + case1.template: '%y4.nc4', + case1.format: 'CFIO', + case1.frequency: 010000, + case1.duration: 000000, + case1.zstandard_level: 2, + case1.quantization_algorithm: 'granular_bitround', + case1.quantization_level: 5, + case1.fields: 'VAR2D', 'Root', + 'VAR3D', 'Root', + :: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc new file mode 100644 index 000000000000..2895432e995a --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc @@ -0,0 +1,5 @@ +GRID_LABELS: +:: + +COLLECTIONS: +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/README b/Tests/ExtData_Testing_Framework/test_cases/case36/README new file mode 100644 index 000000000000..9a6d7597262d --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/README @@ -0,0 +1 @@ +Case, 12-month/12 time 2004 file with 2 updates, non-climatology diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml b/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml new file mode 100644 index 000000000000..e2ddb90675ab --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml @@ -0,0 +1,5 @@ +Collections: + fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" } +Exports: + VAR2D: {variable: VAR2D, collection: fstream1} + VAR3D: {variable: VAR3D, collection: fstream1} diff --git a/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt index 76be9d4d54f8..b79e4a5c9d29 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt +++ b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt @@ -32,3 +32,5 @@ case31 case32 case33 case34 +case35 +case36 diff --git a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md index f3700c12e5ee..65802345b833 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md +++ b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md @@ -39,3 +39,6 @@ path_to_script/run_extdatadriver_cases.py --builddir path_to_geos_install/bin -- 32. Case1 with deflate compression and NetCDF bitgroom quantization (only enabled if netcdf built with quantization support) 33. Case1 with deflate compression and NetCDF bitround quantization (only enabled if netcdf built with quantization support) 34. Case1 with deflate compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization support) +35. Case1 with zstandard compression (only enabled if netcdf built with zstandard support) +36. Case1 with zstandard compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization + support and zstandard support) diff --git a/base/Base/Base_Base_implementation.F90 b/base/Base/Base_Base_implementation.F90 index 20b27e36deea..a070bee5cba3 100644 --- a/base/Base/Base_Base_implementation.F90 +++ b/base/Base/Base_Base_implementation.F90 @@ -1954,6 +1954,7 @@ module function MAPL_LatLonGridCreate (Name, vm, & block use MAPL_Constants, only: MAPL_DEGREES_TO_RADIANS_R8 + use iso_fortran_env, only: REAL64 real(kind=REAL64), allocatable :: lons(:) real(kind=REAL64), allocatable :: lats(:) @@ -2599,7 +2600,7 @@ module subroutine MAPL_GetHorzIJIndex(npts,II,JJ,lon,lat,lonR8,latR8,Grid, rc) type(ESMF_CoordSys_Flag) :: coordSys character(len=ESMF_MAXSTR) :: grid_type - _RETURN_IF(npts == 0 ) + _RETURN_IF(npts == 0 ) ! if the grid is present then we can just get the prestored edges and the dimensions of the grid ! this also means we are running on a distributed grid ! if grid not present then the we just be running outside of ESMF and the user must diff --git a/components.yaml b/components.yaml index 1796c1de157f..4360ab32a56f 100644 --- a/components.yaml +++ b/components.yaml @@ -17,4 +17,4 @@ ESMA_cmake: ecbuild: local: ./ESMA_cmake/ecbuild remote: ../ecbuild.git - tag: geos/v1.3.0 + tag: geos/v1.4.0 diff --git a/gridcomps/History/MAPL_HistoryCollection.F90 b/gridcomps/History/MAPL_HistoryCollection.F90 index 39f3c0b723d1..336773e0400d 100644 --- a/gridcomps/History/MAPL_HistoryCollection.F90 +++ b/gridcomps/History/MAPL_HistoryCollection.F90 @@ -83,6 +83,7 @@ module MAPL_HistoryCollectionMod character(len=ESMF_MAXSTR) :: quantize_algorithm_string integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level integer :: slices integer :: Root integer :: Psize diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index 7aa0afff2a1c..adc88d215dbb 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -836,6 +836,18 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute ( cfg, list(n)%deflate, default=0, & label=trim(string) // 'deflate:' ,_RC ) + ! We only allow deflate level to be between 0 and 9 + _ASSERT( .not. (list(n)%deflate < 0 .or. list(n)%deflate > 9), 'deflate level must be between 0 and 9') + + call ESMF_ConfigGetAttribute ( cfg, list(n)%zstandard_level, default=0, & + label=trim(string) // 'zstandard_level:' ,_RC ) + + ! We only allow zstandard level to be between 0 and 22 + _ASSERT( .not. (list(n)%zstandard_level < 0 .or. list(n)%zstandard_level > 22), 'zstandard level must be between 0 and 22') + + ! We only allow either deflate or zstandard compression to be used, not both + _ASSERT( .not. (list(n)%deflate > 0 .and. list(n)%zstandard_level > 0), 'deflate and zstandard_level cannot be used together') + call ESMF_ConfigGetAttribute ( cfg, list(n)%quantize_algorithm_string, default='NONE', & label=trim(string) // 'quantize_algorithm:' ,_RC ) @@ -2405,6 +2417,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call list(n)%xsampler%set_param(deflation=list(n)%deflate,_RC) call list(n)%xsampler%set_param(quantize_algorithm=list(n)%quantize_algorithm,_RC) call list(n)%xsampler%set_param(quantize_level=list(n)%quantize_level,_RC) + call list(n)%xsampler%set_param(zstandard_level=list(n)%zstandard_level,_RC) call list(n)%xsampler%set_param(chunking=list(n)%chunkSize,_RC) call list(n)%xsampler%set_param(nbits_to_keep=list(n)%nbits_to_keep,_RC) call list(n)%xsampler%set_param(regrid_method=list(n)%regrid_method,_RC) @@ -2415,6 +2428,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call list(n)%mGriddedIO%set_param(deflation=list(n)%deflate,_RC) call list(n)%mGriddedIO%set_param(quantize_algorithm=list(n)%quantize_algorithm,_RC) call list(n)%mGriddedIO%set_param(quantize_level=list(n)%quantize_level,_RC) + call list(n)%mGriddedIO%set_param(zstandard_level=list(n)%zstandard_level,_RC) call list(n)%mGriddedIO%set_param(chunking=list(n)%chunkSize,_RC) call list(n)%mGriddedIO%set_param(nbits_to_keep=list(n)%nbits_to_keep,_RC) call list(n)%mGriddedIO%set_param(regrid_method=list(n)%regrid_method,_RC) @@ -2493,6 +2507,9 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) print *, 'Quantize Alg: ', trim(list(n)%quantize_algorithm_string) print *, 'Quantize Lvl: ', list(n)%quantize_level end if + if (list(n)%zstandard_level > 0) then + print *, 'Zstandard Lvl: ', list(n)%zstandard_level + end if if (associated(list(n)%chunksize)) then print *, ' ChunkSize: ', list(n)%chunksize end if diff --git a/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 b/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 index 0fb038bff536..cf051b2b66a8 100644 --- a/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 +++ b/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 @@ -90,6 +90,7 @@ module MAPL_EpochSwathMod integer :: deflateLevel = 0 integer :: quantizeAlgorithm = 1 integer :: quantizeLevel = 0 + integer :: zstandardLevel = 0 integer, allocatable :: chunking(:) logical :: itemOrderAlphabetical = .true. integer :: fraction @@ -536,11 +537,12 @@ subroutine Create_bundle_RH(this,items,bundle,tunit,timeInfo,vdata,ogrid,rc) end subroutine Create_Bundle_RH - subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,rc) + subroutine set_param(this,deflation,quantize_algorithm,quantize_level,zstandard_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,rc) class (sampler), intent(inout) :: this integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(in) :: chunking(:) integer, optional, intent(in) :: nbits_to_keep integer, optional, intent(in) :: regrid_method @@ -555,6 +557,7 @@ subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,n if (present(deflation)) this%deflateLevel = deflation if (present(quantize_algorithm)) this%quantizeAlgorithm = quantize_algorithm if (present(quantize_level)) this%quantizeLevel = quantize_level + if (present(zstandard_level)) this%zstandardLevel = zstandard_level if (present(chunking)) then allocate(this%chunking,source=chunking,_STAT) end if diff --git a/griddedio/FieldBundleWrite.F90 b/griddedio/FieldBundleWrite.F90 index 1fb4e134e304..36e7d57f9b59 100644 --- a/griddedio/FieldBundleWrite.F90 +++ b/griddedio/FieldBundleWrite.F90 @@ -29,7 +29,7 @@ module MAPL_ESMFFieldBundleWrite contains - subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,deflate,quantize_algorithm,quantize_level,rc) + subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,deflate,quantize_algorithm,quantize_level,zstandard_level,rc) type(ESMF_FieldBundle), intent(inout) :: bundle type(ESMF_Clock), intent(inout) :: clock character(len=*), intent(in) :: output_file @@ -37,20 +37,21 @@ subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,defla integer, optional, intent(in) :: deflate integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc integer :: status type(FieldBundleWriter) :: newWriter - call newWriter%create_from_bundle(bundle,clock,output_file=output_File,n_steps=1,time_interval=0,nbits_to_keep=nbits_to_keep,deflate=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,rc=status) + call newWriter%create_from_bundle(bundle,clock,output_file=output_File,n_steps=1,time_interval=0,nbits_to_keep=nbits_to_keep,deflate=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,zstandard_level=zstandard_level,rc=status) _VERIFY(status) call newWriter%write_to_file(rc=status) _VERIFY(status) _RETURN(_SUCCESS) end subroutine write_bundle_single_time - subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_steps,time_interval,nbits_to_keep,deflate,quantize_algorithm,quantize_level,rc) + subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_steps,time_interval,nbits_to_keep,deflate,quantize_algorithm,quantize_level,zstandard_level,rc) class(FieldBundleWRiter), intent(inout) :: this type(ESMF_FieldBundle), intent(inout) :: bundle type(ESMF_Clock), intent(inout) :: clock @@ -62,6 +63,7 @@ subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_step integer, optional, intent(in) :: deflate integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc type(TimeData) :: time_info @@ -85,7 +87,7 @@ subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_step time_interval_=0 end if - call this%cfio%set_param(nbits_to_keep=nbits_to_keep,deflation=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level) + call this%cfio%set_param(nbits_to_keep=nbits_to_keep,deflation=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,zstandard_level=zstandard_level) time_info = TimeData(clock,file_steps,time_interval_,offset) call ESMF_FieldBundleGet(bundle, fieldCount=num_fields,rc=status) _VERIFY(status) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index 7eb3cf53c7f7..06ec0754460e 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -54,6 +54,7 @@ module MAPL_GriddedIOMod integer :: deflateLevel = 0 integer :: quantizeAlgorithm = MAPL_NOQUANTIZE integer :: quantizeLevel = 0 + integer :: zstandardLevel = 0 integer, allocatable :: chunking(:) logical :: itemOrderAlphabetical = .true. integer :: fraction @@ -250,11 +251,12 @@ subroutine destroy(this, rc) end subroutine destroy - subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,regrid_hints,rc) + subroutine set_param(this,deflation,quantize_algorithm,quantize_level,zstandard_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,regrid_hints,rc) class (MAPL_GriddedIO), intent(inout) :: this integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(in) :: chunking(:) integer, optional, intent(in) :: nbits_to_keep integer, optional, intent(in) :: regrid_method @@ -270,6 +272,7 @@ subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,n if (present(deflation)) this%deflateLevel = deflation if (present(quantize_algorithm)) this%quantizeAlgorithm = quantize_algorithm if (present(quantize_level)) this%quantizeLevel = quantize_level + if (present(zstandard_level)) this%zstandardLevel = zstandard_level if (present(chunking)) then allocate(this%chunking,source=chunking,stat=status) _VERIFY(status) @@ -409,7 +412,7 @@ subroutine CreateVariable(this,itemName,rc) _FAIL( 'Unsupported field rank') end if end if - v = Variable(type=PFIO_REAL32,dimensions=vdims,chunksizes=this%chunking,deflation=this%deflateLevel,quantize_algorithm=this%quantizeAlgorithm,quantize_level=this%quantizeLevel) + v = Variable(type=PFIO_REAL32,dimensions=vdims,chunksizes=this%chunking,deflation=this%deflateLevel,quantize_algorithm=this%quantizeAlgorithm,quantize_level=this%quantizeLevel,zstandard_level=this%zstandardLevel) call v%add_attribute('units',trim(units)) call v%add_attribute('long_name',trim(longName)) call v%add_attribute('standard_name',trim(longName)) diff --git a/pfio/CMakeLists.txt b/pfio/CMakeLists.txt index 3c64d0598926..3c6b826eb4f3 100644 --- a/pfio/CMakeLists.txt +++ b/pfio/CMakeLists.txt @@ -114,6 +114,27 @@ else () message(STATUS "netCDF does not have quantize capability") endif () +################################################################ +# Check to see if zstandard capability is present in netcdf-c. # +################################################################ + +check_c_source_compiles(" + #include + #if !NC_HAS_ZSTD + choke me + #endif + int main(void) { + return 0; + } + " + NETCDF_HAS_ZSTD) +if (NETCDF_HAS_ZSTD) + message(STATUS "netCDF has zstandard capability") + add_definitions(-DNF_HAS_ZSTD) +else () + message(STATUS "netCDF does not have zstandard capability") +endif () + if (BUILD_WITH_PFLOGGER) find_package (PFLOGGER REQUIRED) endif () diff --git a/pfio/NetCDF4_FileFormatter.F90 b/pfio/NetCDF4_FileFormatter.F90 index 7ffc315bbbba..1527d8eb5524 100644 --- a/pfio/NetCDF4_FileFormatter.F90 +++ b/pfio/NetCDF4_FileFormatter.F90 @@ -709,6 +709,7 @@ subroutine def_variables(this, cf, unusable, varname, rc) integer :: deflation integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level character(len=:), pointer :: var_name character(len=:), pointer :: dim_name class (Variable), pointer :: var @@ -783,6 +784,18 @@ subroutine def_variables(this, cf, unusable, varname, rc) #endif end if + zstandard_level = var%get_zstandard_level() + if (zstandard_level /= 0) then +#ifdef NF_HAS_ZSTD + !$omp critical + status = nf90_def_var_zstandard(this%ncid, varid, zstandard_level) + !$omp end critical + _VERIFY(status) +#else + _FAIL("netcdf was not built with zstandard support") +#endif + end if + call this%put_var_attributes(var, varid, rc=status) _VERIFY(status) diff --git a/pfio/Variable.F90 b/pfio/Variable.F90 index 624954bfd162..84958a172945 100644 --- a/pfio/Variable.F90 +++ b/pfio/Variable.F90 @@ -30,6 +30,7 @@ module pFIO_VariableMod integer :: deflation = 0 ! default no compression integer :: quantize_algorithm = 0 ! default no quantization integer :: quantize_level = 0 ! default no quantize_level + integer :: zstandard_level = 0 ! default no zstandard integer, allocatable :: chunksizes(:) contains procedure :: get_type @@ -51,6 +52,7 @@ module pFIO_VariableMod procedure :: set_deflation procedure :: get_quantize_algorithm procedure :: get_quantize_level + procedure :: get_zstandard_level procedure :: is_attribute_present generic :: operator(==) => equal generic :: operator(/=) => not_equal @@ -69,7 +71,7 @@ module pFIO_VariableMod contains - function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflation, quantize_algorithm, quantize_level, rc) result(var) + function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflation, quantize_algorithm, quantize_level, zstandard_level, rc) result(var) type (Variable) :: var integer, optional, intent(in) :: type class (KeywordEnforcer), optional, intent(in) :: unusable @@ -79,6 +81,7 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc integer:: empty(0) @@ -87,6 +90,7 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat var%deflation = 0 var%quantize_algorithm = 0 var%quantize_level = 0 + var%zstandard_level = 0 var%chunksizes = empty var%dimensions = StringVector() var%attributes = StringAttributeMap() @@ -120,6 +124,10 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat var%quantize_level = quantize_level endif + if (present(zstandard_level)) then + var%zstandard_level = zstandard_level + endif + _RETURN(_SUCCESS) _UNUSED_DUMMY(unusable) contains @@ -313,6 +321,13 @@ function get_quantize_level(this) result(quantizeLevel) quantizeLevel=this%quantize_level end function get_quantize_level + function get_zstandard_level(this) result(zstandardLevel) + class (Variable), target, intent(In) :: this + integer :: zstandardLevel + + zstandardLevel=this%zstandard_level + end function get_zstandard_level + logical function equal(a, b) class (Variable), target, intent(in) :: a type (Variable), target, intent(in) :: b @@ -388,6 +403,7 @@ subroutine serialize(this, buffer, rc) buffer = [buffer, serialize_intrinsic(this%deflation)] buffer = [buffer, serialize_intrinsic(this%quantize_algorithm)] buffer = [buffer, serialize_intrinsic(this%quantize_level)] + buffer = [buffer, serialize_intrinsic(this%zstandard_level)] if( .not. allocated(this%chunksizes)) then buffer =[buffer,[1]] @@ -453,6 +469,9 @@ subroutine deserialize(this, buffer, rc) call deserialize_intrinsic(buffer(n:),this%quantize_level) length = serialize_buffer_length(this%quantize_level) n = n + length + call deserialize_intrinsic(buffer(n:),this%zstandard_level) + length = serialize_buffer_length(this%zstandard_level) + n = n + length call deserialize_intrinsic(buffer(n:),this%chunksizes) _RETURN(_SUCCESS) end subroutine deserialize