From af44acb88c82de10e062759b572537bd3b71080a Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 4 Jun 2024 18:23:26 -0700 Subject: [PATCH 01/33] Access NetCDF zstandard compression and quantization functions --- Lib/pywrapper.py | 35 +++++++++++++++++++++ Src/_cmormodule.c | 48 ++++++++++++++++++++++++++++ Src/cmor.c | 68 ++++++++++++++++++++++++++++++++++------ Src/cmor_CV.c | 3 ++ Src/cmor_variables.c | 69 +++++++++++++++++++++++++++++++++++++++++ include/cmor.h | 9 ++++++ include/cmor_func_def.h | 3 ++ 7 files changed, 225 insertions(+), 10 deletions(-) diff --git a/Lib/pywrapper.py b/Lib/pywrapper.py index 9a9b8c8d..9ff636c8 100644 --- a/Lib/pywrapper.py +++ b/Lib/pywrapper.py @@ -1032,6 +1032,41 @@ def set_deflate(var_id, shuffle, deflate, deflate_level): return _cmor.set_deflate(var_id, shuffle, deflate, deflate_level) +def set_zstandard(var_id, zstandar_level): + """Sets Zstandard compression on a cmor variable + Usage: + cmor.set_zstandard(var_id, szstandar_level) + Where: + var_id: is cmor variable id + zstandar_level: Compression level. Must be set from -131072 to 22 + + """ + + return _cmor.set_zstandard(var_id, zstandar_level) + + +def set_quantize(var_id, quantize_mode, quantize_nsd): + """Sets quantization on a cmor variable + Usage: + cmor.set_quantize(var_id, quantize_mode, quantize_nsd) + Where: + var_id: is cmor variable id + quantize_mode: Quantization mode. Can be set to the following. + 0: No quantization mode + 1: BitGroom + 2: Granular BitRound + 3: BitRound + quantize_nsd: Number of significant digits. If quantize_mode is set to + 1 or 2, then the value can be set from 1 to 7 for floats + and 1 to 23 for doubles. If quantize_mode is set to 3, then + the value can be set from 1 to 15 for floats and 1 to 52 + for doubles. The value is ignore if quantize_mode is 0. + + """ + + return _cmor.set_quantize(var_id, quantize_mode, quantize_nsd) + + def has_variable_attribute(var_id, name): """determines if the a cmor variable has an attribute Usage: diff --git a/Src/_cmormodule.c b/Src/_cmormodule.c index 744bb468..26c7115e 100644 --- a/Src/_cmormodule.c +++ b/Src/_cmormodule.c @@ -150,6 +150,52 @@ static PyObject *PyCMOR_set_deflate(PyObject * self, PyObject * args) return (Py_BuildValue("i", ierr)); } +/************************************************************************/ +/* PyCMOR_set_zstandard() */ +/************************************************************************/ +static PyObject *PyCMOR_set_zstandard(PyObject * self, PyObject * args) +{ + signal(signal_to_catch, signal_handler); + int ierr, var_id, zstandard_level; + + if (!PyArg_ParseTuple + (args, "iiii", &var_id, &zstandard_level)) + return NULL; + + ierr = cmor_set_zstandard(var_id, zstandard_level); + + if (ierr != 0 || raise_exception) { + raise_exception = 0; + PyErr_Format(CMORError, exception_message, "set_zstandard"); + return NULL; + } + + return (Py_BuildValue("i", ierr)); +} + +/************************************************************************/ +/* PyCMOR_set_quantize() */ +/************************************************************************/ +static PyObject *PyCMOR_set_quantize(PyObject * self, PyObject * args) +{ + signal(signal_to_catch, signal_handler); + int ierr, var_id, quantize_mode, quantize_nsd; + + if (!PyArg_ParseTuple + (args, "iiii", &var_id, &quantize_mode, &quantize_nsd)) + return NULL; + + ierr = cmor_set_quantize(var_id, quantize_mode, quantize_nsd); + + if (ierr != 0 || raise_exception) { + raise_exception = 0; + PyErr_Format(CMORError, exception_message, "set_quantize"); + return NULL; + } + + return (Py_BuildValue("i", ierr)); +} + /************************************************************************/ /* PyCMOR_set_variable_attribute() */ /************************************************************************/ @@ -1145,6 +1191,8 @@ static PyMethodDef MyExtractMethods[] = { {"set_furtherinfourl", PyCMOR_set_furtherinfourl, METH_VARARGS}, {"get_final_filename", PyCMOR_getFinalFilename, METH_VARARGS}, {"set_deflate", PyCMOR_set_deflate, METH_VARARGS}, + {"set_zstandard", PyCMOR_set_zstandard, METH_VARARGS}, + {"set_quantize", PyCMOR_set_quantize, METH_VARARGS}, {"set_terminate_signal", PyCMOR_set_terminate_signal, METH_VARARGS}, {"get_terminate_signal", PyCMOR_get_terminate_signal, METH_VARARGS}, {NULL, NULL} /*sentinel */ diff --git a/Src/cmor.c b/Src/cmor.c index abd6bdf5..831b361b 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -9,6 +9,7 @@ #include "cmor.h" #include "cmor_locale.h" #include +#include #include #include #include @@ -35,6 +36,16 @@ int nc_def_var_deflate(int i, int j, int k, int l, int m) return (0); }; +int nc_def_var_zstandard(int i, int j, int k) +{ + return (0); +}; + +int nc_def_var_quantize(int i, int j, int k, int l) +{ + return (0); +}; + int nc_def_var_chunking(int i, int j, int k, size_t * l) { return (0); @@ -720,6 +731,9 @@ void cmor_reset_variable(int var_id) cmor_vars[var_id].shuffle = 0; cmor_vars[var_id].deflate = 1; cmor_vars[var_id].deflate_level = 1; + cmor_vars[var_id].zstandard_level = 1; + cmor_vars[var_id].quantize_mode = 0; + cmor_vars[var_id].quantize_nsd = 1; cmor_vars[var_id].nomissing = 1; cmor_vars[var_id].iunits[0] = '\0'; cmor_vars[var_id].ounits[0] = '\0'; @@ -1869,7 +1883,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, ia; + int ics, icd, icdl, icz, icqm, icqn, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2064,10 +2078,17 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int nTableID = cmor_vars[l].ref_table_id; ics = cmor_tables[nTableID].vars[nTableID].shuffle; icd = cmor_tables[nTableID].vars[nTableID].deflate; + icz = cmor_tables[nTableID].vars[nTableID].zstandard_level; + icqm = cmor_tables[nTableID].vars[nTableID].quantize_mode; + icqn = cmor_tables[nTableID].vars[nTableID].quantize_nsd; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], + icqm, icqn); + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], ics, icd, icdl); + ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], + icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3432,7 +3453,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3784,9 +3805,15 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; - ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, + ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, + icqn); + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, icdl); + ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4039,7 +4066,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4306,9 +4333,25 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - - ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], - ics, icd, icdl); + icz = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + zstandard_level; + icqm = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_mode; + icqn = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_nsd; + + ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], + icqm, icqn); + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, icd, icdl); + ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], + icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " @@ -5093,7 +5136,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, int nVarRefTblID = cmor_vars[var_id].ref_table_id; int nelts; int *int_list = NULL; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int bChunk; cmor_add_traceback("cmor_create_var_attributes"); /* -------------------------------------------------------------------- */ @@ -5167,7 +5210,12 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - ierr = nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; + ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn); + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, diff --git a/Src/cmor_CV.c b/Src/cmor_CV.c index fa9da49e..ac8f46c9 100644 --- a/Src/cmor_CV.c +++ b/Src/cmor_CV.c @@ -2539,6 +2539,9 @@ int cmor_CV_variable(int *var_id, char *name, char *units, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; + cmor_vars[vrid].zstandard_level = refvar.zstandard_level; + cmor_vars[vrid].quantize_mode = refvar.quantize_mode; + cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; cmor_vars[vrid].first_bound = startimebnds; cmor_vars[vrid].last_bound = endtimebnds; cmor_vars[vrid].first_time = startime; diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index caf277f5..82282852 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1148,6 +1148,9 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; + cmor_vars[vrid].zstandard_level = refvar.zstandard_level; + cmor_vars[vrid].quantize_mode = refvar.quantize_mode; + cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; strcpy(cmor_vars[vrid].chunking_dimensions, refvar.chunking_dimensions); if (refvar.out_name[0] == '\0') { @@ -1902,6 +1905,9 @@ void cmor_init_var_def(cmor_var_def_t * var, int table_id) var->shuffle = 0; var->deflate = 1; var->deflate_level = 1; + var->zstandard_level = 1; + var->quantize_mode = 0; + var->quantize_nsd = 1; var->generic_level_name[0] = '\0'; } @@ -2134,6 +2140,18 @@ int cmor_set_var_def_att(cmor_var_def_t * var, char att[CMOR_MAX_STRING], var->deflate_level = atoi(val); + } else if (strcmp(att, VARIABLE_ATT_ZSTANDARDLEVEL) == 0) { + + var->zstandard_level = atoi(val); + + } else if (strcmp(att, VARIABLE_ATT_QUANTIZEMODE) == 0) { + + var->quantize_mode = atoi(val); + + } else if (strcmp(att, VARIABLE_ATT_QUANTIZENSD) == 0) { + + var->quantize_nsd = atoi(val); + } else if (strcmp(att, VARIABLE_ATT_MODELINGREALM) == 0) { strncpy(var->realm, val, CMOR_MAX_STRING); @@ -2279,6 +2297,57 @@ int cmor_set_deflate(int var_id, int shuffle, int deflate, int deflate_level) return (0); } +/************************************************************************/ +/* cmor_set_zstandard() */ +/************************************************************************/ +int cmor_set_zstandard(int var_id, int zstandard_level) +{ + char msg[CMOR_MAX_STRING]; + + cmor_add_traceback("cmor_set_zstandard"); + cmor_is_setup(); + + if (cmor_vars[var_id].self != var_id) { + snprintf(msg, CMOR_MAX_STRING, + "You attempted to set the zstandard level of " + "variable id(%d) which was not initialized", var_id); + cmor_handle_error_var(msg, CMOR_CRITICAL, var_id); + cmor_pop_traceback(); + + return (-1); + } + + cmor_vars[var_id].zstandard_level = zstandard_level; + cmor_pop_traceback(); + return (0); +} + +/************************************************************************/ +/* cmor_set_quantize() */ +/************************************************************************/ +int cmor_set_quantize(int var_id, int quantize_mode, int quantize_nsd) +{ + char msg[CMOR_MAX_STRING]; + + cmor_add_traceback("cmor_set_quantize"); + cmor_is_setup(); + + if (cmor_vars[var_id].self != var_id) { + snprintf(msg, CMOR_MAX_STRING, + "You attempted to set the quantize mode of " + "variable id(%d) which was not initialized", var_id); + cmor_handle_error_var(msg, CMOR_CRITICAL, var_id); + cmor_pop_traceback(); + + return (-1); + } + + cmor_vars[var_id].quantize_mode = quantize_mode; + cmor_vars[var_id].quantize_nsd = quantize_nsd; + cmor_pop_traceback(); + return (0); +} + /************************************************************************/ /* cmor_get_variable_time_length() */ /************************************************************************/ diff --git a/include/cmor.h b/include/cmor.h index 54fbadf4..278dfe64 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -134,6 +134,9 @@ #define VARIABLE_ATT_SHUFFLE "shuffle" #define VARIABLE_ATT_DEFLATE "deflate" #define VARIABLE_ATT_DEFLATELEVEL "deflate_level" +#define VARIABLE_ATT_ZSTANDARDLEVEL "zstandard_level" +#define VARIABLE_ATT_QUANTIZEMODE "quantize_mode" +#define VARIABLE_ATT_QUANTIZENSD "quantize_nsd" #define VARIABLE_ATT_MODELINGREALM "modeling_realm" #define VARIALBE_ATT_FREQUENCY "frequency" #define VARIABLE_ATT_FLAGVALUES "flag_values" @@ -422,6 +425,9 @@ typedef struct cmor_variable_def_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; char required[CMOR_MAX_STRING]; char realm[CMOR_MAX_STRING]; char frequency[CMOR_MAX_STRING]; @@ -473,6 +479,9 @@ typedef struct cmor_var_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; int nomissing; char iunits[CMOR_MAX_STRING]; char ounits[CMOR_MAX_STRING]; diff --git a/include/cmor_func_def.h b/include/cmor_func_def.h index 7081568a..c452392e 100644 --- a/include/cmor_func_def.h +++ b/include/cmor_func_def.h @@ -222,6 +222,9 @@ extern int cmor_variable( int *var_id, char *name, char *units, int ndims, char *comment ); extern int cmor_set_deflate( int var_id, int shuffle, int deflate, int deflate_level ); +extern int cmor_set_zstandard( int var_id, int zstandard_level ); +extern int cmor_set_quantize( int var_id, int quantize_mode, + int quantize_nsd ); extern int cmor_set_chunking( int var_id, int nTableID, size_t nc_dim_chunking[]); From 5620960952e9796e62352b6ca2d6d6f005a3908e Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 10 Jun 2024 18:54:14 -0700 Subject: [PATCH 02/33] Add Python functions for set_zstandard and set_quantize --- Lib/__init__.py | 5 +++-- Src/_cmormodule.c | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/__init__.py b/Lib/__init__.py index a6ef98d7..bc42ef04 100644 --- a/Lib/__init__.py +++ b/Lib/__init__.py @@ -11,8 +11,9 @@ close, grid, set_grid_mapping, time_varying_grid_coordinate, dataset_json, set_cur_dataset_attribute, get_cur_dataset_attribute, has_cur_dataset_attribute, set_variable_attribute, get_variable_attribute, - has_variable_attribute, get_final_filename, set_deflate, set_furtherinfourl, - set_climatology, get_climatology, set_terminate_signal, get_terminate_signal) + has_variable_attribute, get_final_filename, set_deflate, set_zstandard, + set_quantize, set_furtherinfourl, set_climatology, get_climatology, + set_terminate_signal, get_terminate_signal) try: from check_CMOR_compliant import checkCMOR diff --git a/Src/_cmormodule.c b/Src/_cmormodule.c index 26c7115e..3b9844df 100644 --- a/Src/_cmormodule.c +++ b/Src/_cmormodule.c @@ -159,7 +159,7 @@ static PyObject *PyCMOR_set_zstandard(PyObject * self, PyObject * args) int ierr, var_id, zstandard_level; if (!PyArg_ParseTuple - (args, "iiii", &var_id, &zstandard_level)) + (args, "ii", &var_id, &zstandard_level)) return NULL; ierr = cmor_set_zstandard(var_id, zstandard_level); @@ -182,7 +182,7 @@ static PyObject *PyCMOR_set_quantize(PyObject * self, PyObject * args) int ierr, var_id, quantize_mode, quantize_nsd; if (!PyArg_ParseTuple - (args, "iiii", &var_id, &quantize_mode, &quantize_nsd)) + (args, "iii", &var_id, &quantize_mode, &quantize_nsd)) return NULL; ierr = cmor_set_quantize(var_id, quantize_mode, quantize_nsd); From 0ac855b49123f5494072b3e31c022afbbdf8eaa2 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 26 Jun 2024 09:28:56 -0700 Subject: [PATCH 03/33] Only apply quantization and zstandard compression to the data --- Src/cmor.c | 42 ++++++------------------------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index 831b361b..4bed62f3 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -1883,7 +1883,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, icz, icqm, icqn, ia; + int ics, icd, icdl, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2078,17 +2078,10 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int nTableID = cmor_vars[l].ref_table_id; ics = cmor_tables[nTableID].vars[nTableID].shuffle; icd = cmor_tables[nTableID].vars[nTableID].deflate; - icz = cmor_tables[nTableID].vars[nTableID].zstandard_level; - icqm = cmor_tables[nTableID].vars[nTableID].quantize_mode; - icqn = cmor_tables[nTableID].vars[nTableID].quantize_nsd; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], - icqm, icqn); - ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], ics, icd, icdl); - ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], - icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3453,7 +3446,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl, icz, icqm, icqn; + int ics, icd, icdl; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3805,15 +3798,9 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - icz = pVar->zstandard_level; - icqm = pVar->quantize_mode; - icqn = pVar->quantize_nsd; - ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, - icqn); - ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, + ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, icdl); - ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4066,7 +4053,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl, icz, icqm, icqn; + int ics, icd, icdl; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4333,25 +4320,8 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - icz = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - zstandard_level; - icqm = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - quantize_mode; - icqn = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - quantize_nsd; - - ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], - icqm, icqn); - ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], ics, icd, icdl); - ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], - icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " From 3f40a052e3ce7ec0800505f1c0d915f0f2773c58 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 29 Jul 2024 13:15:08 -0700 Subject: [PATCH 04/33] Have zstandard compression only be applied if deflate is disabled, and set default zstandard level to 3 --- Src/cmor.c | 12 +++++++++--- Src/cmor_variables.c | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index 4bed62f3..4440b375 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -731,7 +731,7 @@ void cmor_reset_variable(int var_id) cmor_vars[var_id].shuffle = 0; cmor_vars[var_id].deflate = 1; cmor_vars[var_id].deflate_level = 1; - cmor_vars[var_id].zstandard_level = 1; + cmor_vars[var_id].zstandard_level = 3; cmor_vars[var_id].quantize_mode = 0; cmor_vars[var_id].quantize_nsd = 1; cmor_vars[var_id].nomissing = 1; @@ -5184,8 +5184,14 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, icqm = pVar->quantize_mode; icqn = pVar->quantize_nsd; ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn); - ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); - ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); + + // Only use zstandard compression if deflate is disabled + if (icd != 0) { + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, 0, 0); + ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index 82282852..275cedd2 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1905,7 +1905,7 @@ void cmor_init_var_def(cmor_var_def_t * var, int table_id) var->shuffle = 0; var->deflate = 1; var->deflate_level = 1; - var->zstandard_level = 1; + var->zstandard_level = 3; var->quantize_mode = 0; var->quantize_nsd = 1; var->generic_level_name[0] = '\0'; From c8a52896c5d32d6b6b06ccbfbe451b666d452f82 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 29 Jul 2024 13:15:47 -0700 Subject: [PATCH 05/33] Add zstandard compression test --- Makefile.in | 1 + Test/test_cmor_zstandard.py | 128 ++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 Test/test_cmor_zstandard.py diff --git a/Makefile.in b/Makefile.in index 86382f0b..48fd32fc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -225,6 +225,7 @@ test_python: python env TEST_NAME=Test/test_cmor_python_not_enough_times_written.py make test_a_python env TEST_NAME=Test/test_python_forecast_coordinates.py make test_a_python env TEST_NAME=Test/test_cmor_CMIP6Plus.py make test_a_python + env TEST_NAME=Test/test_cmor_zstandard.py make test_a_python test_cmip6_cv: python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentnotset.py make test_a_python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentbad.py make test_a_python diff --git a/Test/test_cmor_zstandard.py b/Test/test_cmor_zstandard.py new file mode 100644 index 00000000..ec3be3fd --- /dev/null +++ b/Test/test_cmor_zstandard.py @@ -0,0 +1,128 @@ +import cmor +import numpy +import unittest +import os +import shutil +import tempfile +from netCDF4 import Dataset + + +def run(): + unittest.main() + + +class TestCase(unittest.TestCase): + + def gen_file(self, seed, ntimes, zstd_level, deflate, deflate_level, shuffle, out_dir): + + numpy.random.seed(seed) + + cmor.setup(inpath='Test', netcdf_file_action=cmor.CMOR_REPLACE) + + cmor.dataset_json("Test/CMOR_input_example.json") + + # creates 1 degree grid + nlat = 18 + nlon = 36 + alats = numpy.arange(180) - 89.5 + bnds_lat = numpy.arange(181) - 90 + alons = numpy.arange(360) + .5 + bnds_lon = numpy.arange(361) + cmor.load_table("Tables/CMIP6_Amon.json") + ilat = cmor.axis( + table_entry='latitude', + units='degrees_north', + length=nlat, + coord_vals=alats, + cell_bounds=bnds_lat) + + ilon = cmor.axis( + table_entry='longitude', + length=nlon, + units='degrees_east', + coord_vals=alons, + cell_bounds=bnds_lon) + + plevs = numpy.array([100000., 92500, 85000, 70000, 60000, 50000, 40000, + 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, + 2000, 1000, 999, 998, 997, 996, 995, 994, 500, 100]) + + itim = cmor.axis( + table_entry='time', + units='months since 2030-1-1', + length=ntimes, + interval='1 month') + + ilev = cmor.axis( + table_entry='plev19', + units='Pa', + coord_vals=plevs, + cell_bounds=None) + + var3d_ids = cmor.variable( + table_entry='ta', + units='K', + axis_ids=numpy.array((ilev, ilon, ilat, itim)), + missing_value=numpy.array([1.0e28, ], dtype=numpy.float32)[0], + original_name='cloud') + + use_deflate = 1 if deflate > 0 else 0 + use_shuffle = 1 if shuffle else 0 + cmor.set_deflate(var3d_ids, use_shuffle, use_deflate, + deflate_level=deflate_level) + cmor.set_zstandard(var3d_ids, zstd_level) + + for it in range(ntimes): + + time = numpy.array((it)) + bnds_time = numpy.array((it, it + 1)) + data3d = numpy.random.random((len(plevs), nlon, nlat)) * 30. + 265. + data3d = data3d.astype('f') + cmor.write( + var_id=var3d_ids, + data=data3d, + ntimes_passed=1, + time_vals=time, + time_bnds=bnds_time) + + nc_path = cmor.close(var3d_ids, file_name=True) + + if deflate: + dst_file = f'deflate_level_{str(deflate_level)}' + else: + dst_file = f'zstd_level_{str(zstd_level)}' + if shuffle: + dst_file += '_shuffle' + dst_file += '.nc' + + dst_path = os.path.join(out_dir, dst_file) + + shutil.copy(nc_path, dst_path) + + return dst_path + + def testZstandardCompression(self): + + seed = 123 + ntimes = 100 + + with tempfile.TemporaryDirectory() as tmp_dir: + no_compression = self.gen_file(seed, ntimes, 0, True, 0, False, tmp_dir) + zstd_shuffle = self.gen_file(seed, ntimes, 3, False, 0, True, tmp_dir) + + no_comp_size = os.path.getsize(no_compression) + zstd_shuffle_size = os.path.getsize(zstd_shuffle) + print(f'File size without compression: {no_comp_size} bytes') + print(f'File size with zstandard compression and shuffle: {zstd_shuffle_size} bytes') + self.assertTrue(zstd_shuffle_size < no_comp_size) + + no_comp_nc = Dataset(no_compression, "r", format="NETCDF4") + zstd_shuffle_nc = Dataset(zstd_shuffle, "r", format="NETCDF4") + + no_comp_ta = no_comp_nc.variables['ta'][:] + zstd_shuffle_ta = zstd_shuffle_nc.variables['ta'][:] + self.assertIsNone(numpy.testing.assert_array_equal(no_comp_ta, zstd_shuffle_ta)) + + +if __name__ == '__main__': + run() From f1550d7db8fbf480750f1585041a7c155755f4d3 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 30 Jul 2024 10:05:39 -0700 Subject: [PATCH 06/33] Add quantization test --- Makefile.in | 2 +- ...py => test_cmor_zstandard_and_quantize.py} | 34 ++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) rename Test/{test_cmor_zstandard.py => test_cmor_zstandard_and_quantize.py} (74%) diff --git a/Makefile.in b/Makefile.in index 48fd32fc..a0b046cb 100644 --- a/Makefile.in +++ b/Makefile.in @@ -225,7 +225,7 @@ test_python: python env TEST_NAME=Test/test_cmor_python_not_enough_times_written.py make test_a_python env TEST_NAME=Test/test_python_forecast_coordinates.py make test_a_python env TEST_NAME=Test/test_cmor_CMIP6Plus.py make test_a_python - env TEST_NAME=Test/test_cmor_zstandard.py make test_a_python + env TEST_NAME=Test/test_cmor_zstandard_and_quantize.py make test_a_python test_cmip6_cv: python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentnotset.py make test_a_python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentbad.py make test_a_python diff --git a/Test/test_cmor_zstandard.py b/Test/test_cmor_zstandard_and_quantize.py similarity index 74% rename from Test/test_cmor_zstandard.py rename to Test/test_cmor_zstandard_and_quantize.py index ec3be3fd..0abe8d8a 100644 --- a/Test/test_cmor_zstandard.py +++ b/Test/test_cmor_zstandard_and_quantize.py @@ -13,7 +13,9 @@ def run(): class TestCase(unittest.TestCase): - def gen_file(self, seed, ntimes, zstd_level, deflate, deflate_level, shuffle, out_dir): + def gen_file(self, seed, ntimes, zstd_level, + deflate, deflate_level, shuffle, + quantize_mode, quantize_nsd, out_dir): numpy.random.seed(seed) @@ -66,6 +68,8 @@ def gen_file(self, seed, ntimes, zstd_level, deflate, deflate_level, shuffle, ou missing_value=numpy.array([1.0e28, ], dtype=numpy.float32)[0], original_name='cloud') + cmor.set_quantize(var3d_ids, quantize_mode, quantize_nsd) + use_deflate = 1 if deflate > 0 else 0 use_shuffle = 1 if shuffle else 0 cmor.set_deflate(var3d_ids, use_shuffle, use_deflate, @@ -93,7 +97,7 @@ def gen_file(self, seed, ntimes, zstd_level, deflate, deflate_level, shuffle, ou dst_file = f'zstd_level_{str(zstd_level)}' if shuffle: dst_file += '_shuffle' - dst_file += '.nc' + dst_file += f'qmode_{str(quantize_mode)}_nsd_{str(quantize_nsd)}.nc' dst_path = os.path.join(out_dir, dst_file) @@ -107,8 +111,8 @@ def testZstandardCompression(self): ntimes = 100 with tempfile.TemporaryDirectory() as tmp_dir: - no_compression = self.gen_file(seed, ntimes, 0, True, 0, False, tmp_dir) - zstd_shuffle = self.gen_file(seed, ntimes, 3, False, 0, True, tmp_dir) + no_compression = self.gen_file(seed, ntimes, 0, True, 0, False, 0, 0, tmp_dir) + zstd_shuffle = self.gen_file(seed, ntimes, 3, False, 0, True, 0, 0, tmp_dir) no_comp_size = os.path.getsize(no_compression) zstd_shuffle_size = os.path.getsize(zstd_shuffle) @@ -123,6 +127,28 @@ def testZstandardCompression(self): zstd_shuffle_ta = zstd_shuffle_nc.variables['ta'][:] self.assertIsNone(numpy.testing.assert_array_equal(no_comp_ta, zstd_shuffle_ta)) + def testQuantize(self): + + seed = 123 + ntimes = 100 + + with tempfile.TemporaryDirectory() as tmp_dir: + default = self.gen_file(seed, ntimes, 0, True, 1, False, 0, 0, tmp_dir) + quantized = self.gen_file(seed, ntimes, 0, True, 1, False, 1, 4, tmp_dir) + + default_size = os.path.getsize(default) + quantized_size = os.path.getsize(quantized) + print(f'File size without quantization: {default_size} bytes') + print(f'File size with quantization: {quantized_size} bytes') + self.assertTrue(quantized_size < default_size) + + default_nc = Dataset(default, "r", format="NETCDF4") + quantized_nc = Dataset(quantized, "r", format="NETCDF4") + + default_ta = default_nc.variables['ta'][:] + quantized_ta = quantized_nc.variables['ta'][:] + self.assertIsNone(numpy.testing.assert_allclose(default_ta, quantized_ta, rtol=1e-4)) + if __name__ == '__main__': run() From 9dfbb965284020bd6f8ecbe5bc11890fd5be5d48 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 30 Jul 2024 11:53:47 -0700 Subject: [PATCH 07/33] Add zstandard and quantize functions to Fortran interface --- Src/cmor_cfortran_interface.c | 17 ++++++++++++++++ Src/cmor_fortran_interface.f90 | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/Src/cmor_cfortran_interface.c b/Src/cmor_cfortran_interface.c index c150729f..7e40f839 100644 --- a/Src/cmor_cfortran_interface.c +++ b/Src/cmor_cfortran_interface.c @@ -92,6 +92,23 @@ int cmor_set_deflate_cff_(int *var_id, int *shuffle, return (cmor_set_deflate(*var_id, *shuffle, *deflate, *deflate_level)); } +/************************************************************************/ +/* cmor_set_zstandard_cff_() */ +/************************************************************************/ +int cmor_set_zstandard_cff_(int *var_id, int *zstandard_level) +{ + return (cmor_set_zstandard(*var_id, *zstandard_level)); +} + +/************************************************************************/ +/* cmor_set_quantize_cff_() */ +/************************************************************************/ +int cmor_set_quantize_cff_(int *var_id, int *quantize_mode, + int *quantize_level) +{ + return (cmor_set_quantize(*var_id, *quantize_mode, *quantize_level)); +} + /************************************************************************/ /* cmor_get_variable_attribute_cff_() */ /************************************************************************/ diff --git a/Src/cmor_fortran_interface.f90 b/Src/cmor_fortran_interface.f90 index 31eb9ffb..b4812548 100644 --- a/Src/cmor_fortran_interface.f90 +++ b/Src/cmor_fortran_interface.f90 @@ -85,6 +85,25 @@ function cmor_set_deflate_cff(var_id, shuffle, deflate, deflate_level )result (i end function cmor_set_deflate_cff end interface + interface + function cmor_set_zstandard_cff(var_id, zstandard_level )result (ierr) + + integer, intent(in) :: var_id + integer, intent(in) :: zstandard_level + integer :: ierr + end function cmor_set_zstandard_cff + end interface + + interface + function cmor_set_quantize_cff(var_id, quantize_mode, quantize_nsd )result (ierr) + + integer, intent(in) :: var_id + integer, intent(in) :: quantize_mode + integer, intent(in) :: quantize_nsd + integer :: ierr + end function cmor_set_quantize_cff + end interface + interface function cmor_setup_cff_nolog(path,ncmode,verbosity,mode,crsub) result (j) integer ncmode,verbosity,mode, j, crsub @@ -7168,6 +7187,24 @@ function cmor_set_deflate(var_id, shuffle, deflate, deflate_level) result (ierr) integer ierr ierr = cmor_set_deflate_cff(var_id, shuffle, deflate, deflate_level) end function cmor_set_deflate + + function cmor_set_zstandard(var_id, zstandard_level) result (ierr) + implicit none + integer, intent (in) :: var_id + integer, intent (in) :: zstandard_level + integer ierr + ierr = cmor_set_zstandard_cff(var_id, zstandard_level) + end function cmor_set_zstandard + + function cmor_set_quantize(var_id, quantize_mode, quantize_nsd) result (ierr) + implicit none + integer, intent (in) :: var_id + integer, intent (in) :: quantize_mode + integer, intent (in) :: quantize_nsd + integer ierr + ierr = cmor_set_quantize_cff(var_id, quantize_mode, quantize_nsd) + end function cmor_set_quantize + function cmor_setup_ints(inpath,netcdf_file_action, set_verbosity,& exit_control, logfile, create_subdirectories) result(ierr) implicit none From f0d7c5962b073ba82e1674dd564350415db75acb Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 09:27:03 -0700 Subject: [PATCH 08/33] Check if zstandard and quantize functions are supported in the NetCDF C API --- Src/cmor.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index 4440b375..baf1f083 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -36,17 +36,21 @@ int nc_def_var_deflate(int i, int j, int k, int l, int m) return (0); }; -int nc_def_var_zstandard(int i, int j, int k) +int nc_def_var_chunking(int i, int j, int k, size_t * l) { return (0); }; +#endif -int nc_def_var_quantize(int i, int j, int k, int l) +#ifndef H5Z_FILTER_ZSTD +int nc_def_var_zstandard(int i, int j, int k) { return (0); }; +#endif -int nc_def_var_chunking(int i, int j, int k, size_t * l) +#ifndef NC_QUANTIZE_BITGROOM +int nc_def_var_quantize(int i, int j, int k, int l) { return (0); }; From 739c4b0f38889d7c4eed13744388e06b182d9cfa Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 11:01:01 -0700 Subject: [PATCH 09/33] Apply zstandard and quantize to dimensions, grids, and zfactors --- Src/cmor.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index baf1f083..a8e60f6d 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -1887,7 +1887,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, ia; + int ics, icd, icdl, icz, icqm, icqn, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2084,8 +2084,24 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, icd = cmor_tables[nTableID].vars[nTableID].deflate; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], - ics, icd, icdl); + icz = + cmor_tables[nTableID].vars[nTableID].zstandard_level; + icqm = + cmor_tables[nTableID].vars[nTableID].quantize_mode; + icqn = + cmor_tables[nTableID].vars[nTableID].quantize_nsd; + + ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], + icqm, icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3450,7 +3466,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3802,9 +3818,21 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - - ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, - icdl); + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; + + ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, + icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4057,7 +4085,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4324,8 +4352,29 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], - ics, icd, icdl); + icz = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + zstandard_level; + icqm = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_mode; + icqn = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_nsd; + ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], + icqm, icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " From 17a784fca60e71ab16c4bde48deeed222e2e59d5 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 16:29:04 -0700 Subject: [PATCH 10/33] Just test python 3.10 --- .circleci/config.yml | 2 +- .github/workflows/nightly-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 597bb333..c23341d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -197,7 +197,7 @@ workflows: matrix: parameters: os: [ linux ] - python_version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + python_version: [ "3.10" ] name: build-<< matrix.os >>-<< matrix.python_version >> - upload: diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 44f04844..1f1af0d4 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -28,7 +28,7 @@ jobs: C_COMPILER: clang_osx-64 FORTRAN_COMPILER: gfortran_osx-64 PROJECT_DIR: workdir/macos_64 - python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_version: ['3.10'] runs-on: ${{ matrix.runner.RUNNER_OS }} env: PACKAGE_NAME: cmor From f1a335f00f9284edd381a5c82416018b78bac95a Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 16:52:28 -0700 Subject: [PATCH 11/33] Remove zstandard and quantize from variables other than data --- Src/cmor.c | 70 +++++++++--------------------------------------------- 1 file changed, 11 insertions(+), 59 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index a8e60f6d..d90203bf 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -1887,7 +1887,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, icz, icqm, icqn, ia; + int ics, icd, icdl, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2084,24 +2084,8 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, icd = cmor_tables[nTableID].vars[nTableID].deflate; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - icz = - cmor_tables[nTableID].vars[nTableID].zstandard_level; - icqm = - cmor_tables[nTableID].vars[nTableID].quantize_mode; - icqn = - cmor_tables[nTableID].vars[nTableID].quantize_nsd; - - ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], - icqm, icqn); - if (icd != 0) { - ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], - ics, icd, icdl); - } else { - ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], - ics, 0, 0); - ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], - icz); - } + ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, icd, icdl); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3466,7 +3450,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl, icz, icqm, icqn; + int ics, icd, icdl; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3818,21 +3802,9 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - icz = pVar->zstandard_level; - icqm = pVar->quantize_mode; - icqn = pVar->quantize_nsd; - - ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, - icqn); - if (icd != 0) { - ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], - ics, icd, icdl); - } else { - ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], - ics, 0, 0); - ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], - icz); - } + + ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, + icdl); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4085,7 +4057,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl, icz, icqm, icqn; + int ics, icd, icdl; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4352,29 +4324,9 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - icz = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - zstandard_level; - icqm = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - quantize_mode; - icqn = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - quantize_nsd; - ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], - icqm, icqn); - if (icd != 0) { - ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], - ics, icd, icdl); - } else { - ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], - ics, 0, 0); - ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], - icz); - } + + ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, icd, icdl); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " From 07034bf14137c3aa396e09c9b6f4a559a0bb6ad2 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 17:04:29 -0700 Subject: [PATCH 12/33] Remove zstandard and quantize function checks --- Src/cmor.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index d90203bf..3f86aca7 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -42,20 +42,6 @@ int nc_def_var_chunking(int i, int j, int k, size_t * l) }; #endif -#ifndef H5Z_FILTER_ZSTD -int nc_def_var_zstandard(int i, int j, int k) -{ - return (0); -}; -#endif - -#ifndef NC_QUANTIZE_BITGROOM -int nc_def_var_quantize(int i, int j, int k, int l) -{ - return (0); -}; -#endif - /* -------------------------------------------------------------------- */ /* function declaration */ /* -------------------------------------------------------------------- */ From 4114845f5ff4c3a63f0146c5bd78cad93e579ff4 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 17:16:40 -0700 Subject: [PATCH 13/33] Remove quantize and zstandard functions from cmor.c --- Src/cmor.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index 3f86aca7..3c717820 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -9,7 +9,6 @@ #include "cmor.h" #include "cmor_locale.h" #include -#include #include #include #include @@ -5097,7 +5096,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, int nVarRefTblID = cmor_vars[var_id].ref_table_id; int nelts; int *int_list = NULL; - int ics, icd, icdl, icz, icqm, icqn; + int ics, icd, icdl; int bChunk; cmor_add_traceback("cmor_create_var_attributes"); /* -------------------------------------------------------------------- */ @@ -5171,18 +5170,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - icz = pVar->zstandard_level; - icqm = pVar->quantize_mode; - icqn = pVar->quantize_nsd; - ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn); - - // Only use zstandard compression if deflate is disabled - if (icd != 0) { - ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); - } else { - ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, 0, 0); - ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); - } + ierr = nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, From 256375b199b349b8caa3bb60d0c27774963d6dd7 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 17:28:21 -0700 Subject: [PATCH 14/33] Remove setting of zstandard and quantize values in variables --- Src/cmor.c | 3 --- Src/cmor_CV.c | 3 --- Src/cmor_variables.c | 3 --- 3 files changed, 9 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index 3c717820..abd6bdf5 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -720,9 +720,6 @@ void cmor_reset_variable(int var_id) cmor_vars[var_id].shuffle = 0; cmor_vars[var_id].deflate = 1; cmor_vars[var_id].deflate_level = 1; - cmor_vars[var_id].zstandard_level = 3; - cmor_vars[var_id].quantize_mode = 0; - cmor_vars[var_id].quantize_nsd = 1; cmor_vars[var_id].nomissing = 1; cmor_vars[var_id].iunits[0] = '\0'; cmor_vars[var_id].ounits[0] = '\0'; diff --git a/Src/cmor_CV.c b/Src/cmor_CV.c index ac8f46c9..fa9da49e 100644 --- a/Src/cmor_CV.c +++ b/Src/cmor_CV.c @@ -2539,9 +2539,6 @@ int cmor_CV_variable(int *var_id, char *name, char *units, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; - cmor_vars[vrid].zstandard_level = refvar.zstandard_level; - cmor_vars[vrid].quantize_mode = refvar.quantize_mode; - cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; cmor_vars[vrid].first_bound = startimebnds; cmor_vars[vrid].last_bound = endtimebnds; cmor_vars[vrid].first_time = startime; diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index 275cedd2..5680d0e6 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1148,9 +1148,6 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; - cmor_vars[vrid].zstandard_level = refvar.zstandard_level; - cmor_vars[vrid].quantize_mode = refvar.quantize_mode; - cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; strcpy(cmor_vars[vrid].chunking_dimensions, refvar.chunking_dimensions); if (refvar.out_name[0] == '\0') { From 3463ac0834431d048e052700e3c8cadbbb14bcde Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 12 Aug 2024 09:54:32 -0700 Subject: [PATCH 15/33] Remove setting of zstandard and quantize values in cmor_variables.c --- Src/cmor_variables.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index 5680d0e6..563eb928 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1902,9 +1902,6 @@ void cmor_init_var_def(cmor_var_def_t * var, int table_id) var->shuffle = 0; var->deflate = 1; var->deflate_level = 1; - var->zstandard_level = 3; - var->quantize_mode = 0; - var->quantize_nsd = 1; var->generic_level_name[0] = '\0'; } @@ -2137,18 +2134,6 @@ int cmor_set_var_def_att(cmor_var_def_t * var, char att[CMOR_MAX_STRING], var->deflate_level = atoi(val); - } else if (strcmp(att, VARIABLE_ATT_ZSTANDARDLEVEL) == 0) { - - var->zstandard_level = atoi(val); - - } else if (strcmp(att, VARIABLE_ATT_QUANTIZEMODE) == 0) { - - var->quantize_mode = atoi(val); - - } else if (strcmp(att, VARIABLE_ATT_QUANTIZENSD) == 0) { - - var->quantize_nsd = atoi(val); - } else if (strcmp(att, VARIABLE_ATT_MODELINGREALM) == 0) { strncpy(var->realm, val, CMOR_MAX_STRING); From c6ad1267744f06649a975737a287975fce3c99fd Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 12 Aug 2024 10:08:27 -0700 Subject: [PATCH 16/33] Remove zstandard and quantize values from variable structs --- Src/cmor_variables.c | 3 --- include/cmor.h | 9 --------- 2 files changed, 12 deletions(-) diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index 563eb928..6db389f1 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -2299,7 +2299,6 @@ int cmor_set_zstandard(int var_id, int zstandard_level) return (-1); } - cmor_vars[var_id].zstandard_level = zstandard_level; cmor_pop_traceback(); return (0); } @@ -2324,8 +2323,6 @@ int cmor_set_quantize(int var_id, int quantize_mode, int quantize_nsd) return (-1); } - cmor_vars[var_id].quantize_mode = quantize_mode; - cmor_vars[var_id].quantize_nsd = quantize_nsd; cmor_pop_traceback(); return (0); } diff --git a/include/cmor.h b/include/cmor.h index 278dfe64..54fbadf4 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -134,9 +134,6 @@ #define VARIABLE_ATT_SHUFFLE "shuffle" #define VARIABLE_ATT_DEFLATE "deflate" #define VARIABLE_ATT_DEFLATELEVEL "deflate_level" -#define VARIABLE_ATT_ZSTANDARDLEVEL "zstandard_level" -#define VARIABLE_ATT_QUANTIZEMODE "quantize_mode" -#define VARIABLE_ATT_QUANTIZENSD "quantize_nsd" #define VARIABLE_ATT_MODELINGREALM "modeling_realm" #define VARIALBE_ATT_FREQUENCY "frequency" #define VARIABLE_ATT_FLAGVALUES "flag_values" @@ -425,9 +422,6 @@ typedef struct cmor_variable_def_ { int shuffle; int deflate; int deflate_level; - int zstandard_level; - int quantize_mode; - int quantize_nsd; char required[CMOR_MAX_STRING]; char realm[CMOR_MAX_STRING]; char frequency[CMOR_MAX_STRING]; @@ -479,9 +473,6 @@ typedef struct cmor_var_ { int shuffle; int deflate; int deflate_level; - int zstandard_level; - int quantize_mode; - int quantize_nsd; int nomissing; char iunits[CMOR_MAX_STRING]; char ounits[CMOR_MAX_STRING]; From 2ac1b135253e0c838ce73aef265a43f9210986fc Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 12 Aug 2024 11:18:21 -0700 Subject: [PATCH 17/33] Add zstandard and quantize back to cmor_variable_def_ --- include/cmor.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/cmor.h b/include/cmor.h index 54fbadf4..ef7dc953 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -422,6 +422,9 @@ typedef struct cmor_variable_def_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; char required[CMOR_MAX_STRING]; char realm[CMOR_MAX_STRING]; char frequency[CMOR_MAX_STRING]; From b126cd603962f640d47da2a2235b6b7434449a0c Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 12 Aug 2024 11:33:41 -0700 Subject: [PATCH 18/33] Add zstandard and quantize back to cmor_var_ --- include/cmor.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/cmor.h b/include/cmor.h index ef7dc953..f8375e25 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -476,6 +476,9 @@ typedef struct cmor_var_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; int nomissing; char iunits[CMOR_MAX_STRING]; char ounits[CMOR_MAX_STRING]; From 7785b0bf4fe24930f1ea5c8dabf4d8c4ae5f59b4 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 12 Aug 2024 11:43:59 -0700 Subject: [PATCH 19/33] Remove zstandard and quantize from cmor_variable_def_ again --- include/cmor.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/cmor.h b/include/cmor.h index f8375e25..cf5d5a2f 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -422,9 +422,6 @@ typedef struct cmor_variable_def_ { int shuffle; int deflate; int deflate_level; - int zstandard_level; - int quantize_mode; - int quantize_nsd; char required[CMOR_MAX_STRING]; char realm[CMOR_MAX_STRING]; char frequency[CMOR_MAX_STRING]; From 6bacd6f57759148827e11497ff1512064389dce6 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 12 Aug 2024 12:27:05 -0700 Subject: [PATCH 20/33] Disable singleton test --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index a0b046cb..63c59f61 100644 --- a/Makefile.in +++ b/Makefile.in @@ -129,7 +129,7 @@ test: cmor test_C @TEST_FORTRAN@ test_python test_cmip6_cv test_C: cmor mkdir -p test_bin rm -f test_bin/ipcc_test_code ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/ipcc_test_code.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ @LDFLAGS@ -o test_bin/ipcc_test_code @VERB@; ./test_bin/ipcc_test_code @VERB@ - rm -f test_bin/test_singletons ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_singletons.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ @LDFLAGS@ -o test_bin/test_singletons @VERB@; ./test_bin/test_singletons @VERB@ +# rm -f test_bin/test_singletons ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_singletons.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ @LDFLAGS@ -o test_bin/test_singletons @VERB@; ./test_bin/test_singletons @VERB@ rm -f test_bin/test_grid ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_grid.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ -o test_bin/test_grid @LDFLAGS@ @VERB@; ./test_bin/test_grid @VERB@; rm -f test_bin/test_grid_unstructured ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_cmor_grid_unstructured_fabi.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ -o test_bin/test_grid_unstructured @LDFLAGS@ @VERB@; ./test_bin/test_grid_unstructured @VERB@; rm -f test_bin/test_grid_unstructured_3d ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_cmor_grid_unstructured_3d_fabi.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ -o test_bin/test_grid_unstructured_3d @LDFLAGS@ @VERB@; ./test_bin/test_grid_unstructured_3d @VERB@; From d14fb94b2caf381cc21821bbb0d44d06b322f06b Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 14 Aug 2024 08:43:39 -0700 Subject: [PATCH 21/33] Check values of singleton id check --- Test/test_singletons.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Test/test_singletons.c b/Test/test_singletons.c index f11af7e0..51741525 100644 --- a/Test/test_singletons.c +++ b/Test/test_singletons.c @@ -33,7 +33,7 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base int var_id; float values[8], miss = 1.e20f; char positive = '\0'; - int i; + int i, singleton_id; int scatangle_found = 0; int wavelength_found = 0; @@ -44,9 +44,15 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base // Find singleton dimension for bs550aer for(i = 0; i < cmor_vars[var_id].ndims; ++i){ - if(strcmp(cmor_axes[cmor_vars[var_id].singleton_ids[i]].id, "wavelength") == 0) - wavelength_found++; + singleton_id = cmor_vars[var_id].singleton_ids[i]; + if(singleton_id != -1) { + printf("cmor_axes[%d].id = %s\n", singleton_id, cmor_axes[singleton_id].id); + if(strcmp(cmor_axes[singleton_id].id, "wavelength") == 0) + wavelength_found++; + } } + printf("wavelength_found = %d\n", wavelength_found); + if(wavelength_found != 1) fail("error in singleton dimension"); From 381a43d6e5fc5937fede84df99f26d06c03dbec8 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 14 Aug 2024 08:55:12 -0700 Subject: [PATCH 22/33] Enable singleton test --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index 63c59f61..a0b046cb 100644 --- a/Makefile.in +++ b/Makefile.in @@ -129,7 +129,7 @@ test: cmor test_C @TEST_FORTRAN@ test_python test_cmip6_cv test_C: cmor mkdir -p test_bin rm -f test_bin/ipcc_test_code ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/ipcc_test_code.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ @LDFLAGS@ -o test_bin/ipcc_test_code @VERB@; ./test_bin/ipcc_test_code @VERB@ -# rm -f test_bin/test_singletons ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_singletons.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ @LDFLAGS@ -o test_bin/test_singletons @VERB@; ./test_bin/test_singletons @VERB@ + rm -f test_bin/test_singletons ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_singletons.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ @LDFLAGS@ -o test_bin/test_singletons @VERB@; ./test_bin/test_singletons @VERB@ rm -f test_bin/test_grid ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_grid.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ -o test_bin/test_grid @LDFLAGS@ @VERB@; ./test_bin/test_grid @VERB@; rm -f test_bin/test_grid_unstructured ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_cmor_grid_unstructured_fabi.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ -o test_bin/test_grid_unstructured @LDFLAGS@ @VERB@; ./test_bin/test_grid_unstructured @VERB@; rm -f test_bin/test_grid_unstructured_3d ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@ Test/test_cmor_grid_unstructured_3d_fabi.c -L@prefix@/lib -I@prefix@/include -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ -o test_bin/test_grid_unstructured_3d @LDFLAGS@ @VERB@; ./test_bin/test_grid_unstructured_3d @VERB@; From 62d9c3fe2553aab56e9fd92406daea206e040489 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 14 Aug 2024 09:16:33 -0700 Subject: [PATCH 23/33] Check all axis ids --- Test/test_singletons.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Test/test_singletons.c b/Test/test_singletons.c index 51741525..46a1e0c3 100644 --- a/Test/test_singletons.c +++ b/Test/test_singletons.c @@ -44,6 +44,7 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base // Find singleton dimension for bs550aer for(i = 0; i < cmor_vars[var_id].ndims; ++i){ + printf("cmor_axes[%d].id = %s\n", i, cmor_axes[i].id); singleton_id = cmor_vars[var_id].singleton_ids[i]; if(singleton_id != -1) { printf("cmor_axes[%d].id = %s\n", singleton_id, cmor_axes[singleton_id].id); From 6e7e8b68e09570558e663147e6f71ac05c68bfc0 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 14 Aug 2024 11:24:16 -0700 Subject: [PATCH 24/33] Uses num_axes instead of ndims --- Test/test_singletons.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Test/test_singletons.c b/Test/test_singletons.c index 46a1e0c3..1ca015be 100644 --- a/Test/test_singletons.c +++ b/Test/test_singletons.c @@ -43,7 +43,8 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base fail("cmor_variable(bs550aer)"); // Find singleton dimension for bs550aer - for(i = 0; i < cmor_vars[var_id].ndims; ++i){ + printf("cmor_vars[%d].ndims = %d\n", var_id, cmor_vars[var_id].ndims); + for(i = 0; i < num_axes; ++i){ printf("cmor_axes[%d].id = %s\n", i, cmor_axes[i].id); singleton_id = cmor_vars[var_id].singleton_ids[i]; if(singleton_id != -1) { From 46656e9cda2831c7c74a127e4f1f9dc183ee57a8 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 14 Aug 2024 11:56:27 -0700 Subject: [PATCH 25/33] Remove quantize value --- include/cmor.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/cmor.h b/include/cmor.h index cf5d5a2f..25d50369 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -475,7 +475,6 @@ typedef struct cmor_var_ { int deflate_level; int zstandard_level; int quantize_mode; - int quantize_nsd; int nomissing; char iunits[CMOR_MAX_STRING]; char ounits[CMOR_MAX_STRING]; From 784dd68dca8bb4f10b8fcd48b5011a716edf59bc Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 14 Aug 2024 12:21:07 -0700 Subject: [PATCH 26/33] Remove zstandard value --- include/cmor.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/cmor.h b/include/cmor.h index 25d50369..228ceaa3 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -473,7 +473,6 @@ typedef struct cmor_var_ { int shuffle; int deflate; int deflate_level; - int zstandard_level; int quantize_mode; int nomissing; char iunits[CMOR_MAX_STRING]; From 22b004f2aef59d09f435e85dfe11fd45499c8b19 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 15 Aug 2024 10:53:07 -0700 Subject: [PATCH 27/33] move zstandard and quantize values to be above ndims and singleton_ids in cmor_var_t --- include/cmor.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/cmor.h b/include/cmor.h index 228ceaa3..33b49961 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -451,6 +451,9 @@ typedef struct cmor_var_ { int time_nc_id; int time_bnds_nc_id; char id[CMOR_MAX_STRING]; + int zstandard_level; + int quantize_mode; + int quantize_nsd; int ndims; int singleton_ids[CMOR_MAX_DIMENSIONS]; int axes_ids[CMOR_MAX_DIMENSIONS]; @@ -473,7 +476,6 @@ typedef struct cmor_var_ { int shuffle; int deflate; int deflate_level; - int quantize_mode; int nomissing; char iunits[CMOR_MAX_STRING]; char ounits[CMOR_MAX_STRING]; From 68db91731a9c704bec728439f9caf10dfc843a9c Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 19 Aug 2024 09:04:53 -0700 Subject: [PATCH 28/33] Add osx-arm64 build to circleci --- .circleci/config.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c23341d1..34dde007 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,8 +22,8 @@ aliases: name: setup_miniconda command: | source $BASH_ENV - if [[ $OS == 'osx-64' ]]; then - curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o miniconda.sh + if [[ $OS == 'osx-arm64' ]]; then + curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o miniconda.sh else curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh fi @@ -75,8 +75,8 @@ aliases: export ARTIFACT_DIR=`pwd`/artifacts/$OS mkdir -p $ARTIFACT_DIR - if [[ $OS == 'osx-64' ]]; then - export OS_NAME="osx_64" + if [[ $OS == 'osx-arm64' ]]; then + export OS_NAME="osx_arm64" else export OS_NAME="linux_64" fi @@ -123,12 +123,12 @@ executors: CONDA_COMPILERS: "gcc_linux-64 gfortran_linux-64" macos: macos: - xcode: "13.4.1" - resource_class: macos.x86.medium.gen2 + xcode: "15.4.0" + resource_class: macos.m1.medium.gen1 environment: - OS: "osx-64" + OS: "osx-arm64" PROJECT_DIR: "workdir/macos" - CONDA_COMPILERS: "clang_osx-64 gfortran_osx-64" + CONDA_COMPILERS: "clang_osx-arm64 gfortran_osx-arm64" jobs: build: @@ -196,7 +196,7 @@ workflows: - build: matrix: parameters: - os: [ linux ] + os: [ linux, macos ] python_version: [ "3.10" ] name: build-<< matrix.os >>-<< matrix.python_version >> From 300eb6f8de67b7e509e6d12107fadd0ef5b38706 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 19 Aug 2024 09:37:54 -0700 Subject: [PATCH 29/33] Move zstandard and quantize values in cmor_var_t struct --- include/cmor.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/cmor.h b/include/cmor.h index 33b49961..cf5d5a2f 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -451,9 +451,6 @@ typedef struct cmor_var_ { int time_nc_id; int time_bnds_nc_id; char id[CMOR_MAX_STRING]; - int zstandard_level; - int quantize_mode; - int quantize_nsd; int ndims; int singleton_ids[CMOR_MAX_DIMENSIONS]; int axes_ids[CMOR_MAX_DIMENSIONS]; @@ -476,6 +473,9 @@ typedef struct cmor_var_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; int nomissing; char iunits[CMOR_MAX_STRING]; char ounits[CMOR_MAX_STRING]; From 483d969b9f9eb760d086d1b3b8c71cf9b825f955 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 19 Aug 2024 10:28:14 -0700 Subject: [PATCH 30/33] Add back zstandard and quantize functionality --- Src/cmor.c | 103 +++++++++++++++++++++++++++++++++++++------ Src/cmor_variables.c | 21 +++++++++ include/cmor.h | 3 ++ 3 files changed, 114 insertions(+), 13 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index abd6bdf5..a8e60f6d 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -9,6 +9,7 @@ #include "cmor.h" #include "cmor_locale.h" #include +#include #include #include #include @@ -41,6 +42,20 @@ int nc_def_var_chunking(int i, int j, int k, size_t * l) }; #endif +#ifndef H5Z_FILTER_ZSTD +int nc_def_var_zstandard(int i, int j, int k) +{ + return (0); +}; +#endif + +#ifndef NC_QUANTIZE_BITGROOM +int nc_def_var_quantize(int i, int j, int k, int l) +{ + return (0); +}; +#endif + /* -------------------------------------------------------------------- */ /* function declaration */ /* -------------------------------------------------------------------- */ @@ -720,6 +735,9 @@ void cmor_reset_variable(int var_id) cmor_vars[var_id].shuffle = 0; cmor_vars[var_id].deflate = 1; cmor_vars[var_id].deflate_level = 1; + cmor_vars[var_id].zstandard_level = 3; + cmor_vars[var_id].quantize_mode = 0; + cmor_vars[var_id].quantize_nsd = 1; cmor_vars[var_id].nomissing = 1; cmor_vars[var_id].iunits[0] = '\0'; cmor_vars[var_id].ounits[0] = '\0'; @@ -1869,7 +1887,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, ia; + int ics, icd, icdl, icz, icqm, icqn, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2066,8 +2084,24 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, icd = cmor_tables[nTableID].vars[nTableID].deflate; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], - ics, icd, icdl); + icz = + cmor_tables[nTableID].vars[nTableID].zstandard_level; + icqm = + cmor_tables[nTableID].vars[nTableID].quantize_mode; + icqn = + cmor_tables[nTableID].vars[nTableID].quantize_nsd; + + ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], + icqm, icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3432,7 +3466,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3784,9 +3818,21 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - - ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, - icdl); + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; + + ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, + icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4039,7 +4085,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4306,9 +4352,29 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - - ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], - ics, icd, icdl); + icz = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + zstandard_level; + icqm = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_mode; + icqn = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_nsd; + ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], + icqm, icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " @@ -5093,7 +5159,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, int nVarRefTblID = cmor_vars[var_id].ref_table_id; int nelts; int *int_list = NULL; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int bChunk; cmor_add_traceback("cmor_create_var_attributes"); /* -------------------------------------------------------------------- */ @@ -5167,7 +5233,18 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - ierr = nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; + ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn); + + // Only use zstandard compression if deflate is disabled + if (icd != 0) { + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, 0, 0); + ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index 6db389f1..275cedd2 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1148,6 +1148,9 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; + cmor_vars[vrid].zstandard_level = refvar.zstandard_level; + cmor_vars[vrid].quantize_mode = refvar.quantize_mode; + cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; strcpy(cmor_vars[vrid].chunking_dimensions, refvar.chunking_dimensions); if (refvar.out_name[0] == '\0') { @@ -1902,6 +1905,9 @@ void cmor_init_var_def(cmor_var_def_t * var, int table_id) var->shuffle = 0; var->deflate = 1; var->deflate_level = 1; + var->zstandard_level = 3; + var->quantize_mode = 0; + var->quantize_nsd = 1; var->generic_level_name[0] = '\0'; } @@ -2134,6 +2140,18 @@ int cmor_set_var_def_att(cmor_var_def_t * var, char att[CMOR_MAX_STRING], var->deflate_level = atoi(val); + } else if (strcmp(att, VARIABLE_ATT_ZSTANDARDLEVEL) == 0) { + + var->zstandard_level = atoi(val); + + } else if (strcmp(att, VARIABLE_ATT_QUANTIZEMODE) == 0) { + + var->quantize_mode = atoi(val); + + } else if (strcmp(att, VARIABLE_ATT_QUANTIZENSD) == 0) { + + var->quantize_nsd = atoi(val); + } else if (strcmp(att, VARIABLE_ATT_MODELINGREALM) == 0) { strncpy(var->realm, val, CMOR_MAX_STRING); @@ -2299,6 +2317,7 @@ int cmor_set_zstandard(int var_id, int zstandard_level) return (-1); } + cmor_vars[var_id].zstandard_level = zstandard_level; cmor_pop_traceback(); return (0); } @@ -2323,6 +2342,8 @@ int cmor_set_quantize(int var_id, int quantize_mode, int quantize_nsd) return (-1); } + cmor_vars[var_id].quantize_mode = quantize_mode; + cmor_vars[var_id].quantize_nsd = quantize_nsd; cmor_pop_traceback(); return (0); } diff --git a/include/cmor.h b/include/cmor.h index cf5d5a2f..f8375e25 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -422,6 +422,9 @@ typedef struct cmor_variable_def_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; char required[CMOR_MAX_STRING]; char realm[CMOR_MAX_STRING]; char frequency[CMOR_MAX_STRING]; From 1516b28d4df6bd096480eb44844a13eb075d4e3f Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 19 Aug 2024 10:34:17 -0700 Subject: [PATCH 31/33] Add back zstandard and quantize code --- Src/cmor_CV.c | 3 +++ include/cmor.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Src/cmor_CV.c b/Src/cmor_CV.c index fa9da49e..ac8f46c9 100644 --- a/Src/cmor_CV.c +++ b/Src/cmor_CV.c @@ -2539,6 +2539,9 @@ int cmor_CV_variable(int *var_id, char *name, char *units, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; + cmor_vars[vrid].zstandard_level = refvar.zstandard_level; + cmor_vars[vrid].quantize_mode = refvar.quantize_mode; + cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; cmor_vars[vrid].first_bound = startimebnds; cmor_vars[vrid].last_bound = endtimebnds; cmor_vars[vrid].first_time = startime; diff --git a/include/cmor.h b/include/cmor.h index f8375e25..278dfe64 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -134,6 +134,9 @@ #define VARIABLE_ATT_SHUFFLE "shuffle" #define VARIABLE_ATT_DEFLATE "deflate" #define VARIABLE_ATT_DEFLATELEVEL "deflate_level" +#define VARIABLE_ATT_ZSTANDARDLEVEL "zstandard_level" +#define VARIABLE_ATT_QUANTIZEMODE "quantize_mode" +#define VARIABLE_ATT_QUANTIZENSD "quantize_nsd" #define VARIABLE_ATT_MODELINGREALM "modeling_realm" #define VARIALBE_ATT_FREQUENCY "frequency" #define VARIABLE_ATT_FLAGVALUES "flag_values" From 619f0a781c9b79a4c4730d918d1c6958cf0b932c Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 21 Aug 2024 08:51:33 -0700 Subject: [PATCH 32/33] Reorganize cmor_var_t in descending order of data type size to improve alignment --- include/cmor.h | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/include/cmor.h b/include/cmor.h index 278dfe64..24202918 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -436,6 +436,17 @@ typedef struct cmor_variable_def_ { } cmor_var_def_t; typedef struct cmor_var_ { + double last_time_written; + double last_time_bounds_written[2]; + double attributes_values_num[CMOR_MAX_ATTRIBUTES]; + double missing; + double omissing; + double tolerance; + double *values; + double first_time; + double last_time; + double first_bound; + double last_bound; int self; int grid_id; int sign; @@ -449,33 +460,16 @@ typedef struct cmor_var_ { int nc_zfactors[CMOR_MAX_VARIABLES]; int nzfactor; int ntimes_written; - double last_time_written; - double last_time_bounds_written[2]; int ntimes_written_coords[10]; int associated_ids[10]; int ntimes_written_associated[10]; int time_nc_id; int time_bnds_nc_id; - char id[CMOR_MAX_STRING]; int ndims; int singleton_ids[CMOR_MAX_DIMENSIONS]; int axes_ids[CMOR_MAX_DIMENSIONS]; int original_order[CMOR_MAX_DIMENSIONS]; - char attributes_values_char[CMOR_MAX_ATTRIBUTES][CMOR_MAX_STRING]; - double attributes_values_num[CMOR_MAX_ATTRIBUTES]; - char attributes_type[CMOR_MAX_ATTRIBUTES]; /*stores attributes type */ - char attributes[CMOR_MAX_ATTRIBUTES][CMOR_MAX_STRING]; /*stores attributes names */ int nattributes; /* number of attributes */ - char type; - char itype; - double missing; - double omissing; - double tolerance; - float valid_min; - float valid_max; - float ok_min_mean_abs; - float ok_max_mean_abs; - char chunking_dimensions[CMOR_MAX_STRING]; int shuffle; int deflate; int deflate_level; @@ -483,20 +477,26 @@ typedef struct cmor_var_ { int quantize_mode; int quantize_nsd; int nomissing; - char iunits[CMOR_MAX_STRING]; - char ounits[CMOR_MAX_STRING]; int isbounds; int needsinit; /* need to be init or associated to file */ int zaxis; /* for z vars, associated axis stored here */ - double *values; - double first_time; - double last_time; - double first_bound; - double last_bound; + int suffix_has_date; + float valid_min; + float valid_max; + float ok_min_mean_abs; + float ok_max_mean_abs; + char id[CMOR_MAX_STRING]; + char attributes_values_char[CMOR_MAX_ATTRIBUTES][CMOR_MAX_STRING]; + char attributes_type[CMOR_MAX_ATTRIBUTES]; /*stores attributes type */ + char attributes[CMOR_MAX_ATTRIBUTES][CMOR_MAX_STRING]; /*stores attributes names */ + char type; + char itype; + char chunking_dimensions[CMOR_MAX_STRING]; + char iunits[CMOR_MAX_STRING]; + char ounits[CMOR_MAX_STRING]; char base_path[CMOR_MAX_STRING]; char current_path[CMOR_MAX_STRING]; char suffix[CMOR_MAX_STRING]; - int suffix_has_date; char frequency[CMOR_MAX_STRING]; } cmor_var_t; From 060d664cbfa682cea8c911fec599e0a0290c0b95 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 22 Aug 2024 09:02:09 -0700 Subject: [PATCH 33/33] See how axes are set in cmor_variable --- Src/cmor_variables.c | 7 +++++++ Test/test_singletons.c | 3 --- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index 275cedd2..f69b9476 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1118,6 +1118,8 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, strcpy(cmor_vars[vrid].base_path, ""); strcpy(cmor_vars[vrid].current_path, ""); strcpy(cmor_vars[vrid].frequency, refvar.frequency); + printf("refvar.frequency = %s\n", refvar.frequency); + printf("strlen(refvar.frequency) = %u\n", strlen(refvar.frequency)); cmor_vars[vrid].suffix_has_date = 0; /* -------------------------------------------------------------------- */ @@ -1152,6 +1154,8 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, cmor_vars[vrid].quantize_mode = refvar.quantize_mode; cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; strcpy(cmor_vars[vrid].chunking_dimensions, refvar.chunking_dimensions); + printf("refvar.chunking_dimensions = %s\n", refvar.chunking_dimensions); + printf("strlen(refvar.chunking_dimensions) = %u\n", strlen(refvar.chunking_dimensions)); if (refvar.out_name[0] == '\0') { strncpy(cmor_vars[vrid].id, name, CMOR_MAX_STRING); @@ -1682,6 +1686,9 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, cmor_axes[laxes_ids[i]].id); cmor_update_history(vrid, msg); cmor_vars[vrid].singleton_ids[i - k] = laxes_ids[i]; + printf("singleton dim: cmor_vars[%d].singleton_ids[%d] = laxes_ids[%d]\n", vrid, i - k, i); + printf("singleton dim: laxes_ids[%d] = %d\n", i, laxes_ids[i]); + printf("singleton dim: cmor_axes[%d].id = %s\n", laxes_ids[i], cmor_axes[laxes_ids[i]].id); if (cmor_has_variable_attribute(vrid, "coordinates") == 0) { cmor_get_variable_attribute(vrid, "coordinates", &msg[0]); } else { diff --git a/Test/test_singletons.c b/Test/test_singletons.c index 1ca015be..acb02d81 100644 --- a/Test/test_singletons.c +++ b/Test/test_singletons.c @@ -43,12 +43,9 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base fail("cmor_variable(bs550aer)"); // Find singleton dimension for bs550aer - printf("cmor_vars[%d].ndims = %d\n", var_id, cmor_vars[var_id].ndims); for(i = 0; i < num_axes; ++i){ - printf("cmor_axes[%d].id = %s\n", i, cmor_axes[i].id); singleton_id = cmor_vars[var_id].singleton_ids[i]; if(singleton_id != -1) { - printf("cmor_axes[%d].id = %s\n", singleton_id, cmor_axes[singleton_id].id); if(strcmp(cmor_axes[singleton_id].id, "wavelength") == 0) wavelength_found++; }