diff --git a/.circleci/config.yml b/.circleci/config.yml index 597bb333..34dde007 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,8 +22,8 @@ aliases: name: setup_miniconda command: | source $BASH_ENV - if [[ $OS == 'osx-64' ]]; then - curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o miniconda.sh + if [[ $OS == 'osx-arm64' ]]; then + curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o miniconda.sh else curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh fi @@ -75,8 +75,8 @@ aliases: export ARTIFACT_DIR=`pwd`/artifacts/$OS mkdir -p $ARTIFACT_DIR - if [[ $OS == 'osx-64' ]]; then - export OS_NAME="osx_64" + if [[ $OS == 'osx-arm64' ]]; then + export OS_NAME="osx_arm64" else export OS_NAME="linux_64" fi @@ -123,12 +123,12 @@ executors: CONDA_COMPILERS: "gcc_linux-64 gfortran_linux-64" macos: macos: - xcode: "13.4.1" - resource_class: macos.x86.medium.gen2 + xcode: "15.4.0" + resource_class: macos.m1.medium.gen1 environment: - OS: "osx-64" + OS: "osx-arm64" PROJECT_DIR: "workdir/macos" - CONDA_COMPILERS: "clang_osx-64 gfortran_osx-64" + CONDA_COMPILERS: "clang_osx-arm64 gfortran_osx-arm64" jobs: build: @@ -196,8 +196,8 @@ workflows: - build: matrix: parameters: - os: [ linux ] - python_version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + os: [ linux, macos ] + python_version: [ "3.10" ] name: build-<< matrix.os >>-<< matrix.python_version >> - upload: diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 44f04844..1f1af0d4 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -28,7 +28,7 @@ jobs: C_COMPILER: clang_osx-64 FORTRAN_COMPILER: gfortran_osx-64 PROJECT_DIR: workdir/macos_64 - python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_version: ['3.10'] runs-on: ${{ matrix.runner.RUNNER_OS }} env: PACKAGE_NAME: cmor diff --git a/Lib/__init__.py b/Lib/__init__.py index a6ef98d7..bc42ef04 100644 --- a/Lib/__init__.py +++ b/Lib/__init__.py @@ -11,8 +11,9 @@ close, grid, set_grid_mapping, time_varying_grid_coordinate, dataset_json, set_cur_dataset_attribute, get_cur_dataset_attribute, has_cur_dataset_attribute, set_variable_attribute, get_variable_attribute, - has_variable_attribute, get_final_filename, set_deflate, set_furtherinfourl, - set_climatology, get_climatology, set_terminate_signal, get_terminate_signal) + has_variable_attribute, get_final_filename, set_deflate, set_zstandard, + set_quantize, set_furtherinfourl, set_climatology, get_climatology, + set_terminate_signal, get_terminate_signal) try: from check_CMOR_compliant import checkCMOR diff --git a/Lib/pywrapper.py b/Lib/pywrapper.py index 9a9b8c8d..9ff636c8 100644 --- a/Lib/pywrapper.py +++ b/Lib/pywrapper.py @@ -1032,6 +1032,41 @@ def set_deflate(var_id, shuffle, deflate, deflate_level): return _cmor.set_deflate(var_id, shuffle, deflate, deflate_level) +def set_zstandard(var_id, zstandar_level): + """Sets Zstandard compression on a cmor variable + Usage: + cmor.set_zstandard(var_id, szstandar_level) + Where: + var_id: is cmor variable id + zstandar_level: Compression level. Must be set from -131072 to 22 + + """ + + return _cmor.set_zstandard(var_id, zstandar_level) + + +def set_quantize(var_id, quantize_mode, quantize_nsd): + """Sets quantization on a cmor variable + Usage: + cmor.set_quantize(var_id, quantize_mode, quantize_nsd) + Where: + var_id: is cmor variable id + quantize_mode: Quantization mode. Can be set to the following. + 0: No quantization mode + 1: BitGroom + 2: Granular BitRound + 3: BitRound + quantize_nsd: Number of significant digits. If quantize_mode is set to + 1 or 2, then the value can be set from 1 to 7 for floats + and 1 to 23 for doubles. If quantize_mode is set to 3, then + the value can be set from 1 to 15 for floats and 1 to 52 + for doubles. The value is ignore if quantize_mode is 0. + + """ + + return _cmor.set_quantize(var_id, quantize_mode, quantize_nsd) + + def has_variable_attribute(var_id, name): """determines if the a cmor variable has an attribute Usage: diff --git a/Makefile.in b/Makefile.in index 86382f0b..a0b046cb 100644 --- a/Makefile.in +++ b/Makefile.in @@ -225,6 +225,7 @@ test_python: python env TEST_NAME=Test/test_cmor_python_not_enough_times_written.py make test_a_python env TEST_NAME=Test/test_python_forecast_coordinates.py make test_a_python env TEST_NAME=Test/test_cmor_CMIP6Plus.py make test_a_python + env TEST_NAME=Test/test_cmor_zstandard_and_quantize.py make test_a_python test_cmip6_cv: python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentnotset.py make test_a_python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentbad.py make test_a_python diff --git a/Src/_cmormodule.c b/Src/_cmormodule.c index da1036aa..762ee27a 100644 --- a/Src/_cmormodule.c +++ b/Src/_cmormodule.c @@ -151,6 +151,52 @@ static PyObject *PyCMOR_set_deflate(PyObject * self, PyObject * args) return (Py_BuildValue("i", ierr)); } +/************************************************************************/ +/* PyCMOR_set_zstandard() */ +/************************************************************************/ +static PyObject *PyCMOR_set_zstandard(PyObject * self, PyObject * args) +{ + signal(signal_to_catch, signal_handler); + int ierr, var_id, zstandard_level; + + if (!PyArg_ParseTuple + (args, "ii", &var_id, &zstandard_level)) + return NULL; + + ierr = cmor_set_zstandard(var_id, zstandard_level); + + if (ierr != 0 || raise_exception) { + raise_exception = 0; + PyErr_Format(CMORError, exception_message, "set_zstandard"); + return NULL; + } + + return (Py_BuildValue("i", ierr)); +} + +/************************************************************************/ +/* PyCMOR_set_quantize() */ +/************************************************************************/ +static PyObject *PyCMOR_set_quantize(PyObject * self, PyObject * args) +{ + signal(signal_to_catch, signal_handler); + int ierr, var_id, quantize_mode, quantize_nsd; + + if (!PyArg_ParseTuple + (args, "iii", &var_id, &quantize_mode, &quantize_nsd)) + return NULL; + + ierr = cmor_set_quantize(var_id, quantize_mode, quantize_nsd); + + if (ierr != 0 || raise_exception) { + raise_exception = 0; + PyErr_Format(CMORError, exception_message, "set_quantize"); + return NULL; + } + + return (Py_BuildValue("i", ierr)); +} + /************************************************************************/ /* PyCMOR_set_variable_attribute() */ /************************************************************************/ @@ -1146,6 +1192,8 @@ static PyMethodDef MyExtractMethods[] = { {"set_furtherinfourl", PyCMOR_set_furtherinfourl, METH_VARARGS}, {"get_final_filename", PyCMOR_getFinalFilename, METH_VARARGS}, {"set_deflate", PyCMOR_set_deflate, METH_VARARGS}, + {"set_zstandard", PyCMOR_set_zstandard, METH_VARARGS}, + {"set_quantize", PyCMOR_set_quantize, METH_VARARGS}, {"set_terminate_signal", PyCMOR_set_terminate_signal, METH_VARARGS}, {"get_terminate_signal", PyCMOR_get_terminate_signal, METH_VARARGS}, {NULL, NULL} /*sentinel */ diff --git a/Src/cmor.c b/Src/cmor.c index abd6bdf5..a8e60f6d 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -9,6 +9,7 @@ #include "cmor.h" #include "cmor_locale.h" #include +#include #include #include #include @@ -41,6 +42,20 @@ int nc_def_var_chunking(int i, int j, int k, size_t * l) }; #endif +#ifndef H5Z_FILTER_ZSTD +int nc_def_var_zstandard(int i, int j, int k) +{ + return (0); +}; +#endif + +#ifndef NC_QUANTIZE_BITGROOM +int nc_def_var_quantize(int i, int j, int k, int l) +{ + return (0); +}; +#endif + /* -------------------------------------------------------------------- */ /* function declaration */ /* -------------------------------------------------------------------- */ @@ -720,6 +735,9 @@ void cmor_reset_variable(int var_id) cmor_vars[var_id].shuffle = 0; cmor_vars[var_id].deflate = 1; cmor_vars[var_id].deflate_level = 1; + cmor_vars[var_id].zstandard_level = 3; + cmor_vars[var_id].quantize_mode = 0; + cmor_vars[var_id].quantize_nsd = 1; cmor_vars[var_id].nomissing = 1; cmor_vars[var_id].iunits[0] = '\0'; cmor_vars[var_id].ounits[0] = '\0'; @@ -1869,7 +1887,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, ia; + int ics, icd, icdl, icz, icqm, icqn, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2066,8 +2084,24 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, icd = cmor_tables[nTableID].vars[nTableID].deflate; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], - ics, icd, icdl); + icz = + cmor_tables[nTableID].vars[nTableID].zstandard_level; + icqm = + cmor_tables[nTableID].vars[nTableID].quantize_mode; + icqn = + cmor_tables[nTableID].vars[nTableID].quantize_nsd; + + ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], + icqm, icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3432,7 +3466,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3784,9 +3818,21 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - - ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, - icdl); + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; + + ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, + icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4039,7 +4085,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4306,9 +4352,29 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - - ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], - ics, icd, icdl); + icz = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + zstandard_level; + icqm = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_mode; + icqn = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_nsd; + ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], + icqm, icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " @@ -5093,7 +5159,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, int nVarRefTblID = cmor_vars[var_id].ref_table_id; int nelts; int *int_list = NULL; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int bChunk; cmor_add_traceback("cmor_create_var_attributes"); /* -------------------------------------------------------------------- */ @@ -5167,7 +5233,18 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - ierr = nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; + ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn); + + // Only use zstandard compression if deflate is disabled + if (icd != 0) { + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, 0, 0); + ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, diff --git a/Src/cmor_CV.c b/Src/cmor_CV.c index fa9da49e..ac8f46c9 100644 --- a/Src/cmor_CV.c +++ b/Src/cmor_CV.c @@ -2539,6 +2539,9 @@ int cmor_CV_variable(int *var_id, char *name, char *units, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; + cmor_vars[vrid].zstandard_level = refvar.zstandard_level; + cmor_vars[vrid].quantize_mode = refvar.quantize_mode; + cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; cmor_vars[vrid].first_bound = startimebnds; cmor_vars[vrid].last_bound = endtimebnds; cmor_vars[vrid].first_time = startime; diff --git a/Src/cmor_cfortran_interface.c b/Src/cmor_cfortran_interface.c index c150729f..7e40f839 100644 --- a/Src/cmor_cfortran_interface.c +++ b/Src/cmor_cfortran_interface.c @@ -92,6 +92,23 @@ int cmor_set_deflate_cff_(int *var_id, int *shuffle, return (cmor_set_deflate(*var_id, *shuffle, *deflate, *deflate_level)); } +/************************************************************************/ +/* cmor_set_zstandard_cff_() */ +/************************************************************************/ +int cmor_set_zstandard_cff_(int *var_id, int *zstandard_level) +{ + return (cmor_set_zstandard(*var_id, *zstandard_level)); +} + +/************************************************************************/ +/* cmor_set_quantize_cff_() */ +/************************************************************************/ +int cmor_set_quantize_cff_(int *var_id, int *quantize_mode, + int *quantize_level) +{ + return (cmor_set_quantize(*var_id, *quantize_mode, *quantize_level)); +} + /************************************************************************/ /* cmor_get_variable_attribute_cff_() */ /************************************************************************/ diff --git a/Src/cmor_fortran_interface.f90 b/Src/cmor_fortran_interface.f90 index 31eb9ffb..b4812548 100644 --- a/Src/cmor_fortran_interface.f90 +++ b/Src/cmor_fortran_interface.f90 @@ -85,6 +85,25 @@ function cmor_set_deflate_cff(var_id, shuffle, deflate, deflate_level )result (i end function cmor_set_deflate_cff end interface + interface + function cmor_set_zstandard_cff(var_id, zstandard_level )result (ierr) + + integer, intent(in) :: var_id + integer, intent(in) :: zstandard_level + integer :: ierr + end function cmor_set_zstandard_cff + end interface + + interface + function cmor_set_quantize_cff(var_id, quantize_mode, quantize_nsd )result (ierr) + + integer, intent(in) :: var_id + integer, intent(in) :: quantize_mode + integer, intent(in) :: quantize_nsd + integer :: ierr + end function cmor_set_quantize_cff + end interface + interface function cmor_setup_cff_nolog(path,ncmode,verbosity,mode,crsub) result (j) integer ncmode,verbosity,mode, j, crsub @@ -7168,6 +7187,24 @@ function cmor_set_deflate(var_id, shuffle, deflate, deflate_level) result (ierr) integer ierr ierr = cmor_set_deflate_cff(var_id, shuffle, deflate, deflate_level) end function cmor_set_deflate + + function cmor_set_zstandard(var_id, zstandard_level) result (ierr) + implicit none + integer, intent (in) :: var_id + integer, intent (in) :: zstandard_level + integer ierr + ierr = cmor_set_zstandard_cff(var_id, zstandard_level) + end function cmor_set_zstandard + + function cmor_set_quantize(var_id, quantize_mode, quantize_nsd) result (ierr) + implicit none + integer, intent (in) :: var_id + integer, intent (in) :: quantize_mode + integer, intent (in) :: quantize_nsd + integer ierr + ierr = cmor_set_quantize_cff(var_id, quantize_mode, quantize_nsd) + end function cmor_set_quantize + function cmor_setup_ints(inpath,netcdf_file_action, set_verbosity,& exit_control, logfile, create_subdirectories) result(ierr) implicit none diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index caf277f5..f69b9476 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1118,6 +1118,8 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, strcpy(cmor_vars[vrid].base_path, ""); strcpy(cmor_vars[vrid].current_path, ""); strcpy(cmor_vars[vrid].frequency, refvar.frequency); + printf("refvar.frequency = %s\n", refvar.frequency); + printf("strlen(refvar.frequency) = %u\n", strlen(refvar.frequency)); cmor_vars[vrid].suffix_has_date = 0; /* -------------------------------------------------------------------- */ @@ -1148,7 +1150,12 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; + cmor_vars[vrid].zstandard_level = refvar.zstandard_level; + cmor_vars[vrid].quantize_mode = refvar.quantize_mode; + cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; strcpy(cmor_vars[vrid].chunking_dimensions, refvar.chunking_dimensions); + printf("refvar.chunking_dimensions = %s\n", refvar.chunking_dimensions); + printf("strlen(refvar.chunking_dimensions) = %u\n", strlen(refvar.chunking_dimensions)); if (refvar.out_name[0] == '\0') { strncpy(cmor_vars[vrid].id, name, CMOR_MAX_STRING); @@ -1679,6 +1686,9 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, cmor_axes[laxes_ids[i]].id); cmor_update_history(vrid, msg); cmor_vars[vrid].singleton_ids[i - k] = laxes_ids[i]; + printf("singleton dim: cmor_vars[%d].singleton_ids[%d] = laxes_ids[%d]\n", vrid, i - k, i); + printf("singleton dim: laxes_ids[%d] = %d\n", i, laxes_ids[i]); + printf("singleton dim: cmor_axes[%d].id = %s\n", laxes_ids[i], cmor_axes[laxes_ids[i]].id); if (cmor_has_variable_attribute(vrid, "coordinates") == 0) { cmor_get_variable_attribute(vrid, "coordinates", &msg[0]); } else { @@ -1902,6 +1912,9 @@ void cmor_init_var_def(cmor_var_def_t * var, int table_id) var->shuffle = 0; var->deflate = 1; var->deflate_level = 1; + var->zstandard_level = 3; + var->quantize_mode = 0; + var->quantize_nsd = 1; var->generic_level_name[0] = '\0'; } @@ -2134,6 +2147,18 @@ int cmor_set_var_def_att(cmor_var_def_t * var, char att[CMOR_MAX_STRING], var->deflate_level = atoi(val); + } else if (strcmp(att, VARIABLE_ATT_ZSTANDARDLEVEL) == 0) { + + var->zstandard_level = atoi(val); + + } else if (strcmp(att, VARIABLE_ATT_QUANTIZEMODE) == 0) { + + var->quantize_mode = atoi(val); + + } else if (strcmp(att, VARIABLE_ATT_QUANTIZENSD) == 0) { + + var->quantize_nsd = atoi(val); + } else if (strcmp(att, VARIABLE_ATT_MODELINGREALM) == 0) { strncpy(var->realm, val, CMOR_MAX_STRING); @@ -2279,6 +2304,57 @@ int cmor_set_deflate(int var_id, int shuffle, int deflate, int deflate_level) return (0); } +/************************************************************************/ +/* cmor_set_zstandard() */ +/************************************************************************/ +int cmor_set_zstandard(int var_id, int zstandard_level) +{ + char msg[CMOR_MAX_STRING]; + + cmor_add_traceback("cmor_set_zstandard"); + cmor_is_setup(); + + if (cmor_vars[var_id].self != var_id) { + snprintf(msg, CMOR_MAX_STRING, + "You attempted to set the zstandard level of " + "variable id(%d) which was not initialized", var_id); + cmor_handle_error_var(msg, CMOR_CRITICAL, var_id); + cmor_pop_traceback(); + + return (-1); + } + + cmor_vars[var_id].zstandard_level = zstandard_level; + cmor_pop_traceback(); + return (0); +} + +/************************************************************************/ +/* cmor_set_quantize() */ +/************************************************************************/ +int cmor_set_quantize(int var_id, int quantize_mode, int quantize_nsd) +{ + char msg[CMOR_MAX_STRING]; + + cmor_add_traceback("cmor_set_quantize"); + cmor_is_setup(); + + if (cmor_vars[var_id].self != var_id) { + snprintf(msg, CMOR_MAX_STRING, + "You attempted to set the quantize mode of " + "variable id(%d) which was not initialized", var_id); + cmor_handle_error_var(msg, CMOR_CRITICAL, var_id); + cmor_pop_traceback(); + + return (-1); + } + + cmor_vars[var_id].quantize_mode = quantize_mode; + cmor_vars[var_id].quantize_nsd = quantize_nsd; + cmor_pop_traceback(); + return (0); +} + /************************************************************************/ /* cmor_get_variable_time_length() */ /************************************************************************/ diff --git a/Test/test_cmor_zstandard_and_quantize.py b/Test/test_cmor_zstandard_and_quantize.py new file mode 100644 index 00000000..0abe8d8a --- /dev/null +++ b/Test/test_cmor_zstandard_and_quantize.py @@ -0,0 +1,154 @@ +import cmor +import numpy +import unittest +import os +import shutil +import tempfile +from netCDF4 import Dataset + + +def run(): + unittest.main() + + +class TestCase(unittest.TestCase): + + def gen_file(self, seed, ntimes, zstd_level, + deflate, deflate_level, shuffle, + quantize_mode, quantize_nsd, out_dir): + + numpy.random.seed(seed) + + cmor.setup(inpath='Test', netcdf_file_action=cmor.CMOR_REPLACE) + + cmor.dataset_json("Test/CMOR_input_example.json") + + # creates 1 degree grid + nlat = 18 + nlon = 36 + alats = numpy.arange(180) - 89.5 + bnds_lat = numpy.arange(181) - 90 + alons = numpy.arange(360) + .5 + bnds_lon = numpy.arange(361) + cmor.load_table("Tables/CMIP6_Amon.json") + ilat = cmor.axis( + table_entry='latitude', + units='degrees_north', + length=nlat, + coord_vals=alats, + cell_bounds=bnds_lat) + + ilon = cmor.axis( + table_entry='longitude', + length=nlon, + units='degrees_east', + coord_vals=alons, + cell_bounds=bnds_lon) + + plevs = numpy.array([100000., 92500, 85000, 70000, 60000, 50000, 40000, + 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, + 2000, 1000, 999, 998, 997, 996, 995, 994, 500, 100]) + + itim = cmor.axis( + table_entry='time', + units='months since 2030-1-1', + length=ntimes, + interval='1 month') + + ilev = cmor.axis( + table_entry='plev19', + units='Pa', + coord_vals=plevs, + cell_bounds=None) + + var3d_ids = cmor.variable( + table_entry='ta', + units='K', + axis_ids=numpy.array((ilev, ilon, ilat, itim)), + missing_value=numpy.array([1.0e28, ], dtype=numpy.float32)[0], + original_name='cloud') + + cmor.set_quantize(var3d_ids, quantize_mode, quantize_nsd) + + use_deflate = 1 if deflate > 0 else 0 + use_shuffle = 1 if shuffle else 0 + cmor.set_deflate(var3d_ids, use_shuffle, use_deflate, + deflate_level=deflate_level) + cmor.set_zstandard(var3d_ids, zstd_level) + + for it in range(ntimes): + + time = numpy.array((it)) + bnds_time = numpy.array((it, it + 1)) + data3d = numpy.random.random((len(plevs), nlon, nlat)) * 30. + 265. + data3d = data3d.astype('f') + cmor.write( + var_id=var3d_ids, + data=data3d, + ntimes_passed=1, + time_vals=time, + time_bnds=bnds_time) + + nc_path = cmor.close(var3d_ids, file_name=True) + + if deflate: + dst_file = f'deflate_level_{str(deflate_level)}' + else: + dst_file = f'zstd_level_{str(zstd_level)}' + if shuffle: + dst_file += '_shuffle' + dst_file += f'qmode_{str(quantize_mode)}_nsd_{str(quantize_nsd)}.nc' + + dst_path = os.path.join(out_dir, dst_file) + + shutil.copy(nc_path, dst_path) + + return dst_path + + def testZstandardCompression(self): + + seed = 123 + ntimes = 100 + + with tempfile.TemporaryDirectory() as tmp_dir: + no_compression = self.gen_file(seed, ntimes, 0, True, 0, False, 0, 0, tmp_dir) + zstd_shuffle = self.gen_file(seed, ntimes, 3, False, 0, True, 0, 0, tmp_dir) + + no_comp_size = os.path.getsize(no_compression) + zstd_shuffle_size = os.path.getsize(zstd_shuffle) + print(f'File size without compression: {no_comp_size} bytes') + print(f'File size with zstandard compression and shuffle: {zstd_shuffle_size} bytes') + self.assertTrue(zstd_shuffle_size < no_comp_size) + + no_comp_nc = Dataset(no_compression, "r", format="NETCDF4") + zstd_shuffle_nc = Dataset(zstd_shuffle, "r", format="NETCDF4") + + no_comp_ta = no_comp_nc.variables['ta'][:] + zstd_shuffle_ta = zstd_shuffle_nc.variables['ta'][:] + self.assertIsNone(numpy.testing.assert_array_equal(no_comp_ta, zstd_shuffle_ta)) + + def testQuantize(self): + + seed = 123 + ntimes = 100 + + with tempfile.TemporaryDirectory() as tmp_dir: + default = self.gen_file(seed, ntimes, 0, True, 1, False, 0, 0, tmp_dir) + quantized = self.gen_file(seed, ntimes, 0, True, 1, False, 1, 4, tmp_dir) + + default_size = os.path.getsize(default) + quantized_size = os.path.getsize(quantized) + print(f'File size without quantization: {default_size} bytes') + print(f'File size with quantization: {quantized_size} bytes') + self.assertTrue(quantized_size < default_size) + + default_nc = Dataset(default, "r", format="NETCDF4") + quantized_nc = Dataset(quantized, "r", format="NETCDF4") + + default_ta = default_nc.variables['ta'][:] + quantized_ta = quantized_nc.variables['ta'][:] + self.assertIsNone(numpy.testing.assert_allclose(default_ta, quantized_ta, rtol=1e-4)) + + +if __name__ == '__main__': + run() diff --git a/Test/test_singletons.c b/Test/test_singletons.c index f11af7e0..acb02d81 100644 --- a/Test/test_singletons.c +++ b/Test/test_singletons.c @@ -33,7 +33,7 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base int var_id; float values[8], miss = 1.e20f; char positive = '\0'; - int i; + int i, singleton_id; int scatangle_found = 0; int wavelength_found = 0; @@ -43,10 +43,15 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base fail("cmor_variable(bs550aer)"); // Find singleton dimension for bs550aer - for(i = 0; i < cmor_vars[var_id].ndims; ++i){ - if(strcmp(cmor_axes[cmor_vars[var_id].singleton_ids[i]].id, "wavelength") == 0) - wavelength_found++; + for(i = 0; i < num_axes; ++i){ + singleton_id = cmor_vars[var_id].singleton_ids[i]; + if(singleton_id != -1) { + if(strcmp(cmor_axes[singleton_id].id, "wavelength") == 0) + wavelength_found++; + } } + printf("wavelength_found = %d\n", wavelength_found); + if(wavelength_found != 1) fail("error in singleton dimension"); diff --git a/include/cmor.h b/include/cmor.h index 54fbadf4..24202918 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -134,6 +134,9 @@ #define VARIABLE_ATT_SHUFFLE "shuffle" #define VARIABLE_ATT_DEFLATE "deflate" #define VARIABLE_ATT_DEFLATELEVEL "deflate_level" +#define VARIABLE_ATT_ZSTANDARDLEVEL "zstandard_level" +#define VARIABLE_ATT_QUANTIZEMODE "quantize_mode" +#define VARIABLE_ATT_QUANTIZENSD "quantize_nsd" #define VARIABLE_ATT_MODELINGREALM "modeling_realm" #define VARIALBE_ATT_FREQUENCY "frequency" #define VARIABLE_ATT_FLAGVALUES "flag_values" @@ -422,6 +425,9 @@ typedef struct cmor_variable_def_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; char required[CMOR_MAX_STRING]; char realm[CMOR_MAX_STRING]; char frequency[CMOR_MAX_STRING]; @@ -430,6 +436,17 @@ typedef struct cmor_variable_def_ { } cmor_var_def_t; typedef struct cmor_var_ { + double last_time_written; + double last_time_bounds_written[2]; + double attributes_values_num[CMOR_MAX_ATTRIBUTES]; + double missing; + double omissing; + double tolerance; + double *values; + double first_time; + double last_time; + double first_bound; + double last_bound; int self; int grid_id; int sign; @@ -443,51 +460,43 @@ typedef struct cmor_var_ { int nc_zfactors[CMOR_MAX_VARIABLES]; int nzfactor; int ntimes_written; - double last_time_written; - double last_time_bounds_written[2]; int ntimes_written_coords[10]; int associated_ids[10]; int ntimes_written_associated[10]; int time_nc_id; int time_bnds_nc_id; - char id[CMOR_MAX_STRING]; int ndims; int singleton_ids[CMOR_MAX_DIMENSIONS]; int axes_ids[CMOR_MAX_DIMENSIONS]; int original_order[CMOR_MAX_DIMENSIONS]; - char attributes_values_char[CMOR_MAX_ATTRIBUTES][CMOR_MAX_STRING]; - double attributes_values_num[CMOR_MAX_ATTRIBUTES]; - char attributes_type[CMOR_MAX_ATTRIBUTES]; /*stores attributes type */ - char attributes[CMOR_MAX_ATTRIBUTES][CMOR_MAX_STRING]; /*stores attributes names */ int nattributes; /* number of attributes */ - char type; - char itype; - double missing; - double omissing; - double tolerance; - float valid_min; - float valid_max; - float ok_min_mean_abs; - float ok_max_mean_abs; - char chunking_dimensions[CMOR_MAX_STRING]; int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; int nomissing; - char iunits[CMOR_MAX_STRING]; - char ounits[CMOR_MAX_STRING]; int isbounds; int needsinit; /* need to be init or associated to file */ int zaxis; /* for z vars, associated axis stored here */ - double *values; - double first_time; - double last_time; - double first_bound; - double last_bound; + int suffix_has_date; + float valid_min; + float valid_max; + float ok_min_mean_abs; + float ok_max_mean_abs; + char id[CMOR_MAX_STRING]; + char attributes_values_char[CMOR_MAX_ATTRIBUTES][CMOR_MAX_STRING]; + char attributes_type[CMOR_MAX_ATTRIBUTES]; /*stores attributes type */ + char attributes[CMOR_MAX_ATTRIBUTES][CMOR_MAX_STRING]; /*stores attributes names */ + char type; + char itype; + char chunking_dimensions[CMOR_MAX_STRING]; + char iunits[CMOR_MAX_STRING]; + char ounits[CMOR_MAX_STRING]; char base_path[CMOR_MAX_STRING]; char current_path[CMOR_MAX_STRING]; char suffix[CMOR_MAX_STRING]; - int suffix_has_date; char frequency[CMOR_MAX_STRING]; } cmor_var_t; diff --git a/include/cmor_func_def.h b/include/cmor_func_def.h index 7081568a..c452392e 100644 --- a/include/cmor_func_def.h +++ b/include/cmor_func_def.h @@ -222,6 +222,9 @@ extern int cmor_variable( int *var_id, char *name, char *units, int ndims, char *comment ); extern int cmor_set_deflate( int var_id, int shuffle, int deflate, int deflate_level ); +extern int cmor_set_zstandard( int var_id, int zstandard_level ); +extern int cmor_set_quantize( int var_id, int quantize_mode, + int quantize_nsd ); extern int cmor_set_chunking( int var_id, int nTableID, size_t nc_dim_chunking[]);