Merge pull request #3071 from GEOS-ESM/develop

GitFlow: Merge develop into main for 2.49 release

Apps/Regrid_Util.F90

@@ -26,6 +26,7 @@ module regrid_util_support_mod
       integer :: deflate, shave
       integer :: quantize_algorithm
       integer :: quantize_level
+      integer :: zstandard_level
       logical :: use_weights
    contains
       procedure :: create_grid
@@ -98,6 +99,7 @@ subroutine process_command_line(this,rc)
     this%deflate=0
     this%quantize_algorithm=0
     this%quantize_level=0
+    this%zstandard_level=0
     this%use_weights = .false.
     nargs = command_argument_count()
     do i=1,nargs
@@ -161,6 +163,9 @@ subroutine process_command_line(this,rc)
       case('-quantize_level')
          call get_command_argument(i+1,astr)
          read(astr,*)this%quantize_level
+      case('-zstandard_level')
+         call get_command_argument(i+1,astr)
+         read(astr,*)this%zstandard_level
       case('-file_weights')
          this%use_weights = .true.
       case('--help')
@@ -432,7 +437,7 @@ subroutine main()
 
          call ESMF_ClockSet(clock,currtime=time,_RC)
          if (.not. writer_created) then
-            call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,_RC)
+            call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,zstandard_level=support%zstandard_level,_RC)
             writer_created=.true.
          end if
 

CHANGELOG.md

@@ -17,6 +17,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Deprecated
 
+## [2.49.0] - 2024-10-04
+
+### Added
+
+- Added zstandard compression support
+  - Note this requires netCDF-C to have been compiled with zstandard support. We have a CMake test to check for this
+    and enabling zstandard output in History will fail if the library does not support it
+
+### Changed
+
+- ExtDataDriver.x now uses ExtData2G by default
+- Update `components.yaml`
+  - ecbuild geos/v1.4.0
+    - Fixes bug between GCC, macOS, and the `-pipe` flag
+
+### Fixed
+
+- Workaround for NVHPC 24.9 involving `use` statement in `block` construct
+
 ## [2.48.0] - 2024-09-24
 
 ### Added

CMakeLists.txt

@@ -8,7 +8,7 @@ endif ()
 
 project (
   MAPL
-  VERSION 2.48.0
+  VERSION 2.49.0
   LANGUAGES Fortran CXX C)  # Note - CXX is required for ESMF
 
 # Set the possible values of build type for cmake-gui

Tests/ExtDataDriverGridComp.F90

@@ -224,7 +224,7 @@ subroutine initialize_gc(gc, import_state, export_state, clock, rc)
     !EOR
     enableTimers = ESMF_UtilStringUpperCase(enableTimers, rc = status)
     _VERIFY(status)
-    call MAPL_GetResource(maplobj,use_extdata2g,"USE_EXTDATA2G:",default=.false.,_RC)
+    call MAPL_GetResource(maplobj,use_extdata2g,"USE_EXTDATA2G:",default=.true.,_RC)
 
     if (enableTimers /= 'YES') then
        call MAPL_ProfDisable(rc = status)

Tests/ExtData_Testing_Framework/CMakeLists.txt

@@ -29,62 +29,84 @@ set(QUANTIZE_TESTS
   "case34"
 )
 
-foreach(TEST_CASE ${TEST_CASES_1G})
+# We have one test that requires netcdf zstandard support
+set(ZSTD_TESTS
+  "case35"
+)
+
+# We have one test which requires *both* Quantize and ZSTD support
+set(QUANTIZE_AND_ZSTD_TESTS
+  "case36"
+)
+
+file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G)
+
+foreach(TEST_CASE ${TEST_CASES_2G})
+
+  # Skip tests that require Quantize support if we don't have it
+  if (NOT NETCDF_HAS_QUANTIZE AND ${TEST_CASE} IN_LIST QUANTIZE_TESTS)
+    continue()
+  endif()
+
+  # Skip tests that require ZSTD support if we don't have it
+  if (NOT NETCDF_HAS_ZSTD AND ${TEST_CASE} IN_LIST ZSTD_TESTS)
+    continue()
+  endif()
+
+  # Skip tests that require both Quantize and ZSTD support if we don't have it
+  if (NOT (NETCDF_HAS_QUANTIZE AND NETCDF_HAS_ZSTD) AND ${TEST_CASE} IN_LIST QUANTIZE_AND_ZSTD_TESTS)
+    continue()
+  endif()
+
   if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc)
     file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs)
   else()
     set(num_procs "1")
   endif()
   add_test(
-    NAME "ExtData1G_${TEST_CASE}"
+    NAME "ExtData2G_${TEST_CASE}"
     COMMAND ${CMAKE_COMMAND}
       -DTEST_CASE=${TEST_CASE}
       -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE}
       -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG}
       -DMY_BINARY_DIR=${CMAKE_BINARY_DIR}/bin
       -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}
-      -DIS_EXTDATA2G=NO
+      -DIS_EXTDATA2G=YES
       -P ${CMAKE_CURRENT_SOURCE_DIR}/run_extdata.cmake
   )
   if (${num_procs} GREATER ${cutoff})
-    set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_BIG_TESTS")
+    set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_BIG_TESTS")
   elseif (${TEST_CASE} IN_LIST SLOW_TESTS)
-    set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_SLOW_TESTS")
+    set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_SLOW_TESTS")
   else()
-    set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_SMALL_TESTS")
+    set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_SMALL_TESTS;ESSENTIAL")
   endif()
 endforeach()
 
-file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G)
-
-foreach(TEST_CASE ${TEST_CASES_2G})
-
-  # Skip tests that require Quantize support if we don't have it
-  if (NOT NETCDF_HAS_QUANTIZE AND ${TEST_CASE} IN_LIST QUANTIZE_TESTS)
-    continue()
-  endif()
+file(STRINGS "test_cases/extdata_1g_cases.txt" TEST_CASES_1G)
 
+foreach(TEST_CASE ${TEST_CASES_1G})
   if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc)
     file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs)
   else()
     set(num_procs "1")
   endif()
   add_test(
-    NAME "ExtData2G_${TEST_CASE}"
+    NAME "ExtData1G_${TEST_CASE}"
     COMMAND ${CMAKE_COMMAND}
       -DTEST_CASE=${TEST_CASE}
       -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE}
       -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG}
       -DMY_BINARY_DIR=${CMAKE_BINARY_DIR}/bin
       -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}
-      -DIS_EXTDATA2G=YES
+      -DIS_EXTDATA2G=NO
       -P ${CMAKE_CURRENT_SOURCE_DIR}/run_extdata.cmake
   )
   if (${num_procs} GREATER ${cutoff})
-    set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_BIG_TESTS")
+    set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_BIG_TESTS")
   elseif (${TEST_CASE} IN_LIST SLOW_TESTS)
-    set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_SLOW_TESTS")
+    set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_SLOW_TESTS")
   else()
-    set_tests_properties ("ExtData2G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA2G_SMALL_TESTS;ESSENTIAL")
+    set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_SMALL_TESTS")
   endif()
 endforeach()

Tests/ExtData_Testing_Framework/run_extdata.cmake

@@ -10,14 +10,14 @@ macro(run_case CASE)
     else()
       set(num_procs "1")
     endif()
-    if (${IS_EXTDATA2G} STREQUAL "YES")
-       file(APPEND "${tempdir}/CAP1.rc" "USE_EXTDATA2G: .true.")
-       file(APPEND "${tempdir}/CAP2.rc" "USE_EXTDATA2G: .true.")
+    if (${IS_EXTDATA2G} STREQUAL "NO")
+       file(APPEND "${tempdir}/CAP1.rc" "USE_EXTDATA2G: .false.")
+       file(APPEND "${tempdir}/CAP2.rc" "USE_EXTDATA2G: .false.")
        if (EXISTS  "${tempdir}/CAP3.rc")
-          file(APPEND "${tempdir}/CAP3.rc" "USE_EXTDATA2G: .true.")
+          file(APPEND "${tempdir}/CAP3.rc" "USE_EXTDATA2G: .false.")
        endif()
        if (EXISTS  "${tempdir}/CAP4.rc")
-          file(APPEND "${tempdir}/CAP4.rc" "USE_EXTDATA2G: .true.")
+          file(APPEND "${tempdir}/CAP4.rc" "USE_EXTDATA2G: .false.")
        endif()
     endif()
     execute_process(

Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc

@@ -0,0 +1,24 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc

@@ -0,0 +1,29 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: CompareImports
+
+IMPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc

@@ -0,0 +1,4 @@
+CASES::
+CAP1.rc
+CAP2.rc
+::

Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc

@@ -0,0 +1,25 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM1.rc
+HIST_CF: HISTORY1.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20040115 210000
+20040215 210000
+20040315 210000
+20040415 210000
+20040515 210000
+20040615 210000
+20040715 210000
+20040815 210000
+20040915 210000
+20041015 210000
+20041115 210000
+20041215 210000
+::
+

Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc

@@ -0,0 +1,15 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM2.rc
+HIST_CF: HISTORY2.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20041125 210000
+20041126 210000
+::
+

Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc

@@ -0,0 +1,13 @@
+#CASE_SENSITIVE_VARIABLE_NAMES: .false.
+Ext_AllowExtrap: .false.
+Prefetch: .true.
+#DEBUG_LEVEL: 20
+
+PrimaryExports%%
+VAR2D NA  N N 0   none     none VAR2D case1.%y4.nc4
+VAR3D NA  N N 0   none     none VAR3D case1.%y4.nc4
+%%
+
+
+DerivedExports%%
+%%

Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc

@@ -0,0 +1,14 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: case1
+::
+
+  case1.template:  '%y4.nc4',
+  case1.format:    'CFIO',
+  case1.frequency:  010000,
+  case1.duration: 000000,
+  case1.zstandard_level: 2,
+  case1.fields: 'VAR2D', 'Root',
+                'VAR3D', 'Root',
+                          ::

Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc

@@ -0,0 +1,5 @@
+GRID_LABELS:
+::
+
+COLLECTIONS:
+::

Tests/ExtData_Testing_Framework/test_cases/case35/README

@@ -0,0 +1 @@
+Case, 12-month/12 time 2004 file with 2 updates, non-climatology

Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml

@@ -0,0 +1,5 @@
+Collections:
+   fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" }
+Exports:
+   VAR2D: {variable: VAR2D, collection: fstream1}
+   VAR3D: {variable: VAR3D, collection: fstream1}

Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc

@@ -0,0 +1,24 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000