From af44acb88c82de10e062759b572537bd3b71080a Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 4 Jun 2024 18:23:26 -0700 Subject: [PATCH 01/14] Access NetCDF zstandard compression and quantization functions --- Lib/pywrapper.py | 35 +++++++++++++++++++++ Src/_cmormodule.c | 48 ++++++++++++++++++++++++++++ Src/cmor.c | 68 ++++++++++++++++++++++++++++++++++------ Src/cmor_CV.c | 3 ++ Src/cmor_variables.c | 69 +++++++++++++++++++++++++++++++++++++++++ include/cmor.h | 9 ++++++ include/cmor_func_def.h | 3 ++ 7 files changed, 225 insertions(+), 10 deletions(-) diff --git a/Lib/pywrapper.py b/Lib/pywrapper.py index 9a9b8c8d..9ff636c8 100644 --- a/Lib/pywrapper.py +++ b/Lib/pywrapper.py @@ -1032,6 +1032,41 @@ def set_deflate(var_id, shuffle, deflate, deflate_level): return _cmor.set_deflate(var_id, shuffle, deflate, deflate_level) +def set_zstandard(var_id, zstandar_level): + """Sets Zstandard compression on a cmor variable + Usage: + cmor.set_zstandard(var_id, szstandar_level) + Where: + var_id: is cmor variable id + zstandar_level: Compression level. Must be set from -131072 to 22 + + """ + + return _cmor.set_zstandard(var_id, zstandar_level) + + +def set_quantize(var_id, quantize_mode, quantize_nsd): + """Sets quantization on a cmor variable + Usage: + cmor.set_quantize(var_id, quantize_mode, quantize_nsd) + Where: + var_id: is cmor variable id + quantize_mode: Quantization mode. Can be set to the following. + 0: No quantization mode + 1: BitGroom + 2: Granular BitRound + 3: BitRound + quantize_nsd: Number of significant digits. If quantize_mode is set to + 1 or 2, then the value can be set from 1 to 7 for floats + and 1 to 23 for doubles. If quantize_mode is set to 3, then + the value can be set from 1 to 15 for floats and 1 to 52 + for doubles. The value is ignore if quantize_mode is 0. + + """ + + return _cmor.set_quantize(var_id, quantize_mode, quantize_nsd) + + def has_variable_attribute(var_id, name): """determines if the a cmor variable has an attribute Usage: diff --git a/Src/_cmormodule.c b/Src/_cmormodule.c index 744bb468..26c7115e 100644 --- a/Src/_cmormodule.c +++ b/Src/_cmormodule.c @@ -150,6 +150,52 @@ static PyObject *PyCMOR_set_deflate(PyObject * self, PyObject * args) return (Py_BuildValue("i", ierr)); } +/************************************************************************/ +/* PyCMOR_set_zstandard() */ +/************************************************************************/ +static PyObject *PyCMOR_set_zstandard(PyObject * self, PyObject * args) +{ + signal(signal_to_catch, signal_handler); + int ierr, var_id, zstandard_level; + + if (!PyArg_ParseTuple + (args, "iiii", &var_id, &zstandard_level)) + return NULL; + + ierr = cmor_set_zstandard(var_id, zstandard_level); + + if (ierr != 0 || raise_exception) { + raise_exception = 0; + PyErr_Format(CMORError, exception_message, "set_zstandard"); + return NULL; + } + + return (Py_BuildValue("i", ierr)); +} + +/************************************************************************/ +/* PyCMOR_set_quantize() */ +/************************************************************************/ +static PyObject *PyCMOR_set_quantize(PyObject * self, PyObject * args) +{ + signal(signal_to_catch, signal_handler); + int ierr, var_id, quantize_mode, quantize_nsd; + + if (!PyArg_ParseTuple + (args, "iiii", &var_id, &quantize_mode, &quantize_nsd)) + return NULL; + + ierr = cmor_set_quantize(var_id, quantize_mode, quantize_nsd); + + if (ierr != 0 || raise_exception) { + raise_exception = 0; + PyErr_Format(CMORError, exception_message, "set_quantize"); + return NULL; + } + + return (Py_BuildValue("i", ierr)); +} + /************************************************************************/ /* PyCMOR_set_variable_attribute() */ /************************************************************************/ @@ -1145,6 +1191,8 @@ static PyMethodDef MyExtractMethods[] = { {"set_furtherinfourl", PyCMOR_set_furtherinfourl, METH_VARARGS}, {"get_final_filename", PyCMOR_getFinalFilename, METH_VARARGS}, {"set_deflate", PyCMOR_set_deflate, METH_VARARGS}, + {"set_zstandard", PyCMOR_set_zstandard, METH_VARARGS}, + {"set_quantize", PyCMOR_set_quantize, METH_VARARGS}, {"set_terminate_signal", PyCMOR_set_terminate_signal, METH_VARARGS}, {"get_terminate_signal", PyCMOR_get_terminate_signal, METH_VARARGS}, {NULL, NULL} /*sentinel */ diff --git a/Src/cmor.c b/Src/cmor.c index abd6bdf5..831b361b 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -9,6 +9,7 @@ #include "cmor.h" #include "cmor_locale.h" #include +#include #include #include #include @@ -35,6 +36,16 @@ int nc_def_var_deflate(int i, int j, int k, int l, int m) return (0); }; +int nc_def_var_zstandard(int i, int j, int k) +{ + return (0); +}; + +int nc_def_var_quantize(int i, int j, int k, int l) +{ + return (0); +}; + int nc_def_var_chunking(int i, int j, int k, size_t * l) { return (0); @@ -720,6 +731,9 @@ void cmor_reset_variable(int var_id) cmor_vars[var_id].shuffle = 0; cmor_vars[var_id].deflate = 1; cmor_vars[var_id].deflate_level = 1; + cmor_vars[var_id].zstandard_level = 1; + cmor_vars[var_id].quantize_mode = 0; + cmor_vars[var_id].quantize_nsd = 1; cmor_vars[var_id].nomissing = 1; cmor_vars[var_id].iunits[0] = '\0'; cmor_vars[var_id].ounits[0] = '\0'; @@ -1869,7 +1883,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, ia; + int ics, icd, icdl, icz, icqm, icqn, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2064,10 +2078,17 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int nTableID = cmor_vars[l].ref_table_id; ics = cmor_tables[nTableID].vars[nTableID].shuffle; icd = cmor_tables[nTableID].vars[nTableID].deflate; + icz = cmor_tables[nTableID].vars[nTableID].zstandard_level; + icqm = cmor_tables[nTableID].vars[nTableID].quantize_mode; + icqn = cmor_tables[nTableID].vars[nTableID].quantize_nsd; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], + icqm, icqn); + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], ics, icd, icdl); + ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], + icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3432,7 +3453,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3784,9 +3805,15 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; - ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, + ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, + icqn); + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, icdl); + ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4039,7 +4066,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4306,9 +4333,25 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - - ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], - ics, icd, icdl); + icz = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + zstandard_level; + icqm = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_mode; + icqn = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_nsd; + + ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], + icqm, icqn); + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, icd, icdl); + ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], + icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " @@ -5093,7 +5136,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, int nVarRefTblID = cmor_vars[var_id].ref_table_id; int nelts; int *int_list = NULL; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int bChunk; cmor_add_traceback("cmor_create_var_attributes"); /* -------------------------------------------------------------------- */ @@ -5167,7 +5210,12 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - ierr = nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; + ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn); + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, diff --git a/Src/cmor_CV.c b/Src/cmor_CV.c index fa9da49e..ac8f46c9 100644 --- a/Src/cmor_CV.c +++ b/Src/cmor_CV.c @@ -2539,6 +2539,9 @@ int cmor_CV_variable(int *var_id, char *name, char *units, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; + cmor_vars[vrid].zstandard_level = refvar.zstandard_level; + cmor_vars[vrid].quantize_mode = refvar.quantize_mode; + cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; cmor_vars[vrid].first_bound = startimebnds; cmor_vars[vrid].last_bound = endtimebnds; cmor_vars[vrid].first_time = startime; diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index caf277f5..82282852 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1148,6 +1148,9 @@ int cmor_variable(int *var_id, char *name, char *units, int ndims, cmor_vars[vrid].shuffle = refvar.shuffle; cmor_vars[vrid].deflate = refvar.deflate; cmor_vars[vrid].deflate_level = refvar.deflate_level; + cmor_vars[vrid].zstandard_level = refvar.zstandard_level; + cmor_vars[vrid].quantize_mode = refvar.quantize_mode; + cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd; strcpy(cmor_vars[vrid].chunking_dimensions, refvar.chunking_dimensions); if (refvar.out_name[0] == '\0') { @@ -1902,6 +1905,9 @@ void cmor_init_var_def(cmor_var_def_t * var, int table_id) var->shuffle = 0; var->deflate = 1; var->deflate_level = 1; + var->zstandard_level = 1; + var->quantize_mode = 0; + var->quantize_nsd = 1; var->generic_level_name[0] = '\0'; } @@ -2134,6 +2140,18 @@ int cmor_set_var_def_att(cmor_var_def_t * var, char att[CMOR_MAX_STRING], var->deflate_level = atoi(val); + } else if (strcmp(att, VARIABLE_ATT_ZSTANDARDLEVEL) == 0) { + + var->zstandard_level = atoi(val); + + } else if (strcmp(att, VARIABLE_ATT_QUANTIZEMODE) == 0) { + + var->quantize_mode = atoi(val); + + } else if (strcmp(att, VARIABLE_ATT_QUANTIZENSD) == 0) { + + var->quantize_nsd = atoi(val); + } else if (strcmp(att, VARIABLE_ATT_MODELINGREALM) == 0) { strncpy(var->realm, val, CMOR_MAX_STRING); @@ -2279,6 +2297,57 @@ int cmor_set_deflate(int var_id, int shuffle, int deflate, int deflate_level) return (0); } +/************************************************************************/ +/* cmor_set_zstandard() */ +/************************************************************************/ +int cmor_set_zstandard(int var_id, int zstandard_level) +{ + char msg[CMOR_MAX_STRING]; + + cmor_add_traceback("cmor_set_zstandard"); + cmor_is_setup(); + + if (cmor_vars[var_id].self != var_id) { + snprintf(msg, CMOR_MAX_STRING, + "You attempted to set the zstandard level of " + "variable id(%d) which was not initialized", var_id); + cmor_handle_error_var(msg, CMOR_CRITICAL, var_id); + cmor_pop_traceback(); + + return (-1); + } + + cmor_vars[var_id].zstandard_level = zstandard_level; + cmor_pop_traceback(); + return (0); +} + +/************************************************************************/ +/* cmor_set_quantize() */ +/************************************************************************/ +int cmor_set_quantize(int var_id, int quantize_mode, int quantize_nsd) +{ + char msg[CMOR_MAX_STRING]; + + cmor_add_traceback("cmor_set_quantize"); + cmor_is_setup(); + + if (cmor_vars[var_id].self != var_id) { + snprintf(msg, CMOR_MAX_STRING, + "You attempted to set the quantize mode of " + "variable id(%d) which was not initialized", var_id); + cmor_handle_error_var(msg, CMOR_CRITICAL, var_id); + cmor_pop_traceback(); + + return (-1); + } + + cmor_vars[var_id].quantize_mode = quantize_mode; + cmor_vars[var_id].quantize_nsd = quantize_nsd; + cmor_pop_traceback(); + return (0); +} + /************************************************************************/ /* cmor_get_variable_time_length() */ /************************************************************************/ diff --git a/include/cmor.h b/include/cmor.h index 54fbadf4..278dfe64 100644 --- a/include/cmor.h +++ b/include/cmor.h @@ -134,6 +134,9 @@ #define VARIABLE_ATT_SHUFFLE "shuffle" #define VARIABLE_ATT_DEFLATE "deflate" #define VARIABLE_ATT_DEFLATELEVEL "deflate_level" +#define VARIABLE_ATT_ZSTANDARDLEVEL "zstandard_level" +#define VARIABLE_ATT_QUANTIZEMODE "quantize_mode" +#define VARIABLE_ATT_QUANTIZENSD "quantize_nsd" #define VARIABLE_ATT_MODELINGREALM "modeling_realm" #define VARIALBE_ATT_FREQUENCY "frequency" #define VARIABLE_ATT_FLAGVALUES "flag_values" @@ -422,6 +425,9 @@ typedef struct cmor_variable_def_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; char required[CMOR_MAX_STRING]; char realm[CMOR_MAX_STRING]; char frequency[CMOR_MAX_STRING]; @@ -473,6 +479,9 @@ typedef struct cmor_var_ { int shuffle; int deflate; int deflate_level; + int zstandard_level; + int quantize_mode; + int quantize_nsd; int nomissing; char iunits[CMOR_MAX_STRING]; char ounits[CMOR_MAX_STRING]; diff --git a/include/cmor_func_def.h b/include/cmor_func_def.h index 7081568a..c452392e 100644 --- a/include/cmor_func_def.h +++ b/include/cmor_func_def.h @@ -222,6 +222,9 @@ extern int cmor_variable( int *var_id, char *name, char *units, int ndims, char *comment ); extern int cmor_set_deflate( int var_id, int shuffle, int deflate, int deflate_level ); +extern int cmor_set_zstandard( int var_id, int zstandard_level ); +extern int cmor_set_quantize( int var_id, int quantize_mode, + int quantize_nsd ); extern int cmor_set_chunking( int var_id, int nTableID, size_t nc_dim_chunking[]); From 5620960952e9796e62352b6ca2d6d6f005a3908e Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 10 Jun 2024 18:54:14 -0700 Subject: [PATCH 02/14] Add Python functions for set_zstandard and set_quantize --- Lib/__init__.py | 5 +++-- Src/_cmormodule.c | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/__init__.py b/Lib/__init__.py index a6ef98d7..bc42ef04 100644 --- a/Lib/__init__.py +++ b/Lib/__init__.py @@ -11,8 +11,9 @@ close, grid, set_grid_mapping, time_varying_grid_coordinate, dataset_json, set_cur_dataset_attribute, get_cur_dataset_attribute, has_cur_dataset_attribute, set_variable_attribute, get_variable_attribute, - has_variable_attribute, get_final_filename, set_deflate, set_furtherinfourl, - set_climatology, get_climatology, set_terminate_signal, get_terminate_signal) + has_variable_attribute, get_final_filename, set_deflate, set_zstandard, + set_quantize, set_furtherinfourl, set_climatology, get_climatology, + set_terminate_signal, get_terminate_signal) try: from check_CMOR_compliant import checkCMOR diff --git a/Src/_cmormodule.c b/Src/_cmormodule.c index 26c7115e..3b9844df 100644 --- a/Src/_cmormodule.c +++ b/Src/_cmormodule.c @@ -159,7 +159,7 @@ static PyObject *PyCMOR_set_zstandard(PyObject * self, PyObject * args) int ierr, var_id, zstandard_level; if (!PyArg_ParseTuple - (args, "iiii", &var_id, &zstandard_level)) + (args, "ii", &var_id, &zstandard_level)) return NULL; ierr = cmor_set_zstandard(var_id, zstandard_level); @@ -182,7 +182,7 @@ static PyObject *PyCMOR_set_quantize(PyObject * self, PyObject * args) int ierr, var_id, quantize_mode, quantize_nsd; if (!PyArg_ParseTuple - (args, "iiii", &var_id, &quantize_mode, &quantize_nsd)) + (args, "iii", &var_id, &quantize_mode, &quantize_nsd)) return NULL; ierr = cmor_set_quantize(var_id, quantize_mode, quantize_nsd); From 0ac855b49123f5494072b3e31c022afbbdf8eaa2 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 26 Jun 2024 09:28:56 -0700 Subject: [PATCH 03/14] Only apply quantization and zstandard compression to the data --- Src/cmor.c | 42 ++++++------------------------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index 831b361b..4bed62f3 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -1883,7 +1883,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, icz, icqm, icqn, ia; + int ics, icd, icdl, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2078,17 +2078,10 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int nTableID = cmor_vars[l].ref_table_id; ics = cmor_tables[nTableID].vars[nTableID].shuffle; icd = cmor_tables[nTableID].vars[nTableID].deflate; - icz = cmor_tables[nTableID].vars[nTableID].zstandard_level; - icqm = cmor_tables[nTableID].vars[nTableID].quantize_mode; - icqn = cmor_tables[nTableID].vars[nTableID].quantize_nsd; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], - icqm, icqn); - ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], ics, icd, icdl); - ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], - icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3453,7 +3446,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl, icz, icqm, icqn; + int ics, icd, icdl; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3805,15 +3798,9 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - icz = pVar->zstandard_level; - icqm = pVar->quantize_mode; - icqn = pVar->quantize_nsd; - ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, - icqn); - ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, + ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, icdl); - ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4066,7 +4053,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl, icz, icqm, icqn; + int ics, icd, icdl; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4333,25 +4320,8 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - icz = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - zstandard_level; - icqm = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - quantize_mode; - icqn = - cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. - ref_var_id]. - quantize_nsd; - - ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], - icqm, icqn); - ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], ics, icd, icdl); - ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], - icz); if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " From 3f40a052e3ce7ec0800505f1c0d915f0f2773c58 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 29 Jul 2024 13:15:08 -0700 Subject: [PATCH 04/14] Have zstandard compression only be applied if deflate is disabled, and set default zstandard level to 3 --- Src/cmor.c | 12 +++++++++--- Src/cmor_variables.c | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index 4bed62f3..4440b375 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -731,7 +731,7 @@ void cmor_reset_variable(int var_id) cmor_vars[var_id].shuffle = 0; cmor_vars[var_id].deflate = 1; cmor_vars[var_id].deflate_level = 1; - cmor_vars[var_id].zstandard_level = 1; + cmor_vars[var_id].zstandard_level = 3; cmor_vars[var_id].quantize_mode = 0; cmor_vars[var_id].quantize_nsd = 1; cmor_vars[var_id].nomissing = 1; @@ -5184,8 +5184,14 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid, icqm = pVar->quantize_mode; icqn = pVar->quantize_nsd; ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn); - ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); - ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); + + // Only use zstandard compression if deflate is disabled + if (icd != 0) { + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, 0, 0); + ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, diff --git a/Src/cmor_variables.c b/Src/cmor_variables.c index 82282852..275cedd2 100644 --- a/Src/cmor_variables.c +++ b/Src/cmor_variables.c @@ -1905,7 +1905,7 @@ void cmor_init_var_def(cmor_var_def_t * var, int table_id) var->shuffle = 0; var->deflate = 1; var->deflate_level = 1; - var->zstandard_level = 1; + var->zstandard_level = 3; var->quantize_mode = 0; var->quantize_nsd = 1; var->generic_level_name[0] = '\0'; From c8a52896c5d32d6b6b06ccbfbe451b666d452f82 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 29 Jul 2024 13:15:47 -0700 Subject: [PATCH 05/14] Add zstandard compression test --- Makefile.in | 1 + Test/test_cmor_zstandard.py | 128 ++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 Test/test_cmor_zstandard.py diff --git a/Makefile.in b/Makefile.in index 86382f0b..48fd32fc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -225,6 +225,7 @@ test_python: python env TEST_NAME=Test/test_cmor_python_not_enough_times_written.py make test_a_python env TEST_NAME=Test/test_python_forecast_coordinates.py make test_a_python env TEST_NAME=Test/test_cmor_CMIP6Plus.py make test_a_python + env TEST_NAME=Test/test_cmor_zstandard.py make test_a_python test_cmip6_cv: python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentnotset.py make test_a_python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentbad.py make test_a_python diff --git a/Test/test_cmor_zstandard.py b/Test/test_cmor_zstandard.py new file mode 100644 index 00000000..ec3be3fd --- /dev/null +++ b/Test/test_cmor_zstandard.py @@ -0,0 +1,128 @@ +import cmor +import numpy +import unittest +import os +import shutil +import tempfile +from netCDF4 import Dataset + + +def run(): + unittest.main() + + +class TestCase(unittest.TestCase): + + def gen_file(self, seed, ntimes, zstd_level, deflate, deflate_level, shuffle, out_dir): + + numpy.random.seed(seed) + + cmor.setup(inpath='Test', netcdf_file_action=cmor.CMOR_REPLACE) + + cmor.dataset_json("Test/CMOR_input_example.json") + + # creates 1 degree grid + nlat = 18 + nlon = 36 + alats = numpy.arange(180) - 89.5 + bnds_lat = numpy.arange(181) - 90 + alons = numpy.arange(360) + .5 + bnds_lon = numpy.arange(361) + cmor.load_table("Tables/CMIP6_Amon.json") + ilat = cmor.axis( + table_entry='latitude', + units='degrees_north', + length=nlat, + coord_vals=alats, + cell_bounds=bnds_lat) + + ilon = cmor.axis( + table_entry='longitude', + length=nlon, + units='degrees_east', + coord_vals=alons, + cell_bounds=bnds_lon) + + plevs = numpy.array([100000., 92500, 85000, 70000, 60000, 50000, 40000, + 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, + 2000, 1000, 999, 998, 997, 996, 995, 994, 500, 100]) + + itim = cmor.axis( + table_entry='time', + units='months since 2030-1-1', + length=ntimes, + interval='1 month') + + ilev = cmor.axis( + table_entry='plev19', + units='Pa', + coord_vals=plevs, + cell_bounds=None) + + var3d_ids = cmor.variable( + table_entry='ta', + units='K', + axis_ids=numpy.array((ilev, ilon, ilat, itim)), + missing_value=numpy.array([1.0e28, ], dtype=numpy.float32)[0], + original_name='cloud') + + use_deflate = 1 if deflate > 0 else 0 + use_shuffle = 1 if shuffle else 0 + cmor.set_deflate(var3d_ids, use_shuffle, use_deflate, + deflate_level=deflate_level) + cmor.set_zstandard(var3d_ids, zstd_level) + + for it in range(ntimes): + + time = numpy.array((it)) + bnds_time = numpy.array((it, it + 1)) + data3d = numpy.random.random((len(plevs), nlon, nlat)) * 30. + 265. + data3d = data3d.astype('f') + cmor.write( + var_id=var3d_ids, + data=data3d, + ntimes_passed=1, + time_vals=time, + time_bnds=bnds_time) + + nc_path = cmor.close(var3d_ids, file_name=True) + + if deflate: + dst_file = f'deflate_level_{str(deflate_level)}' + else: + dst_file = f'zstd_level_{str(zstd_level)}' + if shuffle: + dst_file += '_shuffle' + dst_file += '.nc' + + dst_path = os.path.join(out_dir, dst_file) + + shutil.copy(nc_path, dst_path) + + return dst_path + + def testZstandardCompression(self): + + seed = 123 + ntimes = 100 + + with tempfile.TemporaryDirectory() as tmp_dir: + no_compression = self.gen_file(seed, ntimes, 0, True, 0, False, tmp_dir) + zstd_shuffle = self.gen_file(seed, ntimes, 3, False, 0, True, tmp_dir) + + no_comp_size = os.path.getsize(no_compression) + zstd_shuffle_size = os.path.getsize(zstd_shuffle) + print(f'File size without compression: {no_comp_size} bytes') + print(f'File size with zstandard compression and shuffle: {zstd_shuffle_size} bytes') + self.assertTrue(zstd_shuffle_size < no_comp_size) + + no_comp_nc = Dataset(no_compression, "r", format="NETCDF4") + zstd_shuffle_nc = Dataset(zstd_shuffle, "r", format="NETCDF4") + + no_comp_ta = no_comp_nc.variables['ta'][:] + zstd_shuffle_ta = zstd_shuffle_nc.variables['ta'][:] + self.assertIsNone(numpy.testing.assert_array_equal(no_comp_ta, zstd_shuffle_ta)) + + +if __name__ == '__main__': + run() From f1550d7db8fbf480750f1585041a7c155755f4d3 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 30 Jul 2024 10:05:39 -0700 Subject: [PATCH 06/14] Add quantization test --- Makefile.in | 2 +- ...py => test_cmor_zstandard_and_quantize.py} | 34 ++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) rename Test/{test_cmor_zstandard.py => test_cmor_zstandard_and_quantize.py} (74%) diff --git a/Makefile.in b/Makefile.in index 48fd32fc..a0b046cb 100644 --- a/Makefile.in +++ b/Makefile.in @@ -225,7 +225,7 @@ test_python: python env TEST_NAME=Test/test_cmor_python_not_enough_times_written.py make test_a_python env TEST_NAME=Test/test_python_forecast_coordinates.py make test_a_python env TEST_NAME=Test/test_cmor_CMIP6Plus.py make test_a_python - env TEST_NAME=Test/test_cmor_zstandard.py make test_a_python + env TEST_NAME=Test/test_cmor_zstandard_and_quantize.py make test_a_python test_cmip6_cv: python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentnotset.py make test_a_python env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentbad.py make test_a_python diff --git a/Test/test_cmor_zstandard.py b/Test/test_cmor_zstandard_and_quantize.py similarity index 74% rename from Test/test_cmor_zstandard.py rename to Test/test_cmor_zstandard_and_quantize.py index ec3be3fd..0abe8d8a 100644 --- a/Test/test_cmor_zstandard.py +++ b/Test/test_cmor_zstandard_and_quantize.py @@ -13,7 +13,9 @@ def run(): class TestCase(unittest.TestCase): - def gen_file(self, seed, ntimes, zstd_level, deflate, deflate_level, shuffle, out_dir): + def gen_file(self, seed, ntimes, zstd_level, + deflate, deflate_level, shuffle, + quantize_mode, quantize_nsd, out_dir): numpy.random.seed(seed) @@ -66,6 +68,8 @@ def gen_file(self, seed, ntimes, zstd_level, deflate, deflate_level, shuffle, ou missing_value=numpy.array([1.0e28, ], dtype=numpy.float32)[0], original_name='cloud') + cmor.set_quantize(var3d_ids, quantize_mode, quantize_nsd) + use_deflate = 1 if deflate > 0 else 0 use_shuffle = 1 if shuffle else 0 cmor.set_deflate(var3d_ids, use_shuffle, use_deflate, @@ -93,7 +97,7 @@ def gen_file(self, seed, ntimes, zstd_level, deflate, deflate_level, shuffle, ou dst_file = f'zstd_level_{str(zstd_level)}' if shuffle: dst_file += '_shuffle' - dst_file += '.nc' + dst_file += f'qmode_{str(quantize_mode)}_nsd_{str(quantize_nsd)}.nc' dst_path = os.path.join(out_dir, dst_file) @@ -107,8 +111,8 @@ def testZstandardCompression(self): ntimes = 100 with tempfile.TemporaryDirectory() as tmp_dir: - no_compression = self.gen_file(seed, ntimes, 0, True, 0, False, tmp_dir) - zstd_shuffle = self.gen_file(seed, ntimes, 3, False, 0, True, tmp_dir) + no_compression = self.gen_file(seed, ntimes, 0, True, 0, False, 0, 0, tmp_dir) + zstd_shuffle = self.gen_file(seed, ntimes, 3, False, 0, True, 0, 0, tmp_dir) no_comp_size = os.path.getsize(no_compression) zstd_shuffle_size = os.path.getsize(zstd_shuffle) @@ -123,6 +127,28 @@ def testZstandardCompression(self): zstd_shuffle_ta = zstd_shuffle_nc.variables['ta'][:] self.assertIsNone(numpy.testing.assert_array_equal(no_comp_ta, zstd_shuffle_ta)) + def testQuantize(self): + + seed = 123 + ntimes = 100 + + with tempfile.TemporaryDirectory() as tmp_dir: + default = self.gen_file(seed, ntimes, 0, True, 1, False, 0, 0, tmp_dir) + quantized = self.gen_file(seed, ntimes, 0, True, 1, False, 1, 4, tmp_dir) + + default_size = os.path.getsize(default) + quantized_size = os.path.getsize(quantized) + print(f'File size without quantization: {default_size} bytes') + print(f'File size with quantization: {quantized_size} bytes') + self.assertTrue(quantized_size < default_size) + + default_nc = Dataset(default, "r", format="NETCDF4") + quantized_nc = Dataset(quantized, "r", format="NETCDF4") + + default_ta = default_nc.variables['ta'][:] + quantized_ta = quantized_nc.variables['ta'][:] + self.assertIsNone(numpy.testing.assert_allclose(default_ta, quantized_ta, rtol=1e-4)) + if __name__ == '__main__': run() From 9dfbb965284020bd6f8ecbe5bc11890fd5be5d48 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 30 Jul 2024 11:53:47 -0700 Subject: [PATCH 07/14] Add zstandard and quantize functions to Fortran interface --- Src/cmor_cfortran_interface.c | 17 ++++++++++++++++ Src/cmor_fortran_interface.f90 | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/Src/cmor_cfortran_interface.c b/Src/cmor_cfortran_interface.c index c150729f..7e40f839 100644 --- a/Src/cmor_cfortran_interface.c +++ b/Src/cmor_cfortran_interface.c @@ -92,6 +92,23 @@ int cmor_set_deflate_cff_(int *var_id, int *shuffle, return (cmor_set_deflate(*var_id, *shuffle, *deflate, *deflate_level)); } +/************************************************************************/ +/* cmor_set_zstandard_cff_() */ +/************************************************************************/ +int cmor_set_zstandard_cff_(int *var_id, int *zstandard_level) +{ + return (cmor_set_zstandard(*var_id, *zstandard_level)); +} + +/************************************************************************/ +/* cmor_set_quantize_cff_() */ +/************************************************************************/ +int cmor_set_quantize_cff_(int *var_id, int *quantize_mode, + int *quantize_level) +{ + return (cmor_set_quantize(*var_id, *quantize_mode, *quantize_level)); +} + /************************************************************************/ /* cmor_get_variable_attribute_cff_() */ /************************************************************************/ diff --git a/Src/cmor_fortran_interface.f90 b/Src/cmor_fortran_interface.f90 index 31eb9ffb..b4812548 100644 --- a/Src/cmor_fortran_interface.f90 +++ b/Src/cmor_fortran_interface.f90 @@ -85,6 +85,25 @@ function cmor_set_deflate_cff(var_id, shuffle, deflate, deflate_level )result (i end function cmor_set_deflate_cff end interface + interface + function cmor_set_zstandard_cff(var_id, zstandard_level )result (ierr) + + integer, intent(in) :: var_id + integer, intent(in) :: zstandard_level + integer :: ierr + end function cmor_set_zstandard_cff + end interface + + interface + function cmor_set_quantize_cff(var_id, quantize_mode, quantize_nsd )result (ierr) + + integer, intent(in) :: var_id + integer, intent(in) :: quantize_mode + integer, intent(in) :: quantize_nsd + integer :: ierr + end function cmor_set_quantize_cff + end interface + interface function cmor_setup_cff_nolog(path,ncmode,verbosity,mode,crsub) result (j) integer ncmode,verbosity,mode, j, crsub @@ -7168,6 +7187,24 @@ function cmor_set_deflate(var_id, shuffle, deflate, deflate_level) result (ierr) integer ierr ierr = cmor_set_deflate_cff(var_id, shuffle, deflate, deflate_level) end function cmor_set_deflate + + function cmor_set_zstandard(var_id, zstandard_level) result (ierr) + implicit none + integer, intent (in) :: var_id + integer, intent (in) :: zstandard_level + integer ierr + ierr = cmor_set_zstandard_cff(var_id, zstandard_level) + end function cmor_set_zstandard + + function cmor_set_quantize(var_id, quantize_mode, quantize_nsd) result (ierr) + implicit none + integer, intent (in) :: var_id + integer, intent (in) :: quantize_mode + integer, intent (in) :: quantize_nsd + integer ierr + ierr = cmor_set_quantize_cff(var_id, quantize_mode, quantize_nsd) + end function cmor_set_quantize + function cmor_setup_ints(inpath,netcdf_file_action, set_verbosity,& exit_control, logfile, create_subdirectories) result(ierr) implicit none From f0d7c5962b073ba82e1674dd564350415db75acb Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 09:27:03 -0700 Subject: [PATCH 08/14] Check if zstandard and quantize functions are supported in the NetCDF C API --- Src/cmor.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index 4440b375..baf1f083 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -36,17 +36,21 @@ int nc_def_var_deflate(int i, int j, int k, int l, int m) return (0); }; -int nc_def_var_zstandard(int i, int j, int k) +int nc_def_var_chunking(int i, int j, int k, size_t * l) { return (0); }; +#endif -int nc_def_var_quantize(int i, int j, int k, int l) +#ifndef H5Z_FILTER_ZSTD +int nc_def_var_zstandard(int i, int j, int k) { return (0); }; +#endif -int nc_def_var_chunking(int i, int j, int k, size_t * l) +#ifndef NC_QUANTIZE_BITGROOM +int nc_def_var_quantize(int i, int j, int k, int l) { return (0); }; From 739c4b0f38889d7c4eed13744388e06b182d9cfa Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 8 Aug 2024 11:01:01 -0700 Subject: [PATCH 09/14] Apply zstandard and quantize to dimensions, grids, and zfactors --- Src/cmor.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/Src/cmor.c b/Src/cmor.c index baf1f083..a8e60f6d 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -1887,7 +1887,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL; int dim_holder[CMOR_MAX_VARIABLES]; int lnzfactors; - int ics, icd, icdl, ia; + int ics, icd, icdl, icz, icqm, icqn, ia; cmor_add_traceback("cmor_define_zfactors_vars"); cmor_is_setup(); lnzfactors = *nzfactors; @@ -2084,8 +2084,24 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim, icd = cmor_tables[nTableID].vars[nTableID].deflate; icdl = cmor_tables[nTableID].vars[nTableID].deflate_level; - ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], - ics, icd, icdl); + icz = + cmor_tables[nTableID].vars[nTableID].zstandard_level; + icqm = + cmor_tables[nTableID].vars[nTableID].quantize_mode; + icqn = + cmor_tables[nTableID].vars[nTableID].quantize_nsd; + + ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors], + icqm, icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, @@ -3450,7 +3466,7 @@ void cmor_define_dimensions(int var_id, int ncid, int tmp_dims[2]; int dims_bnds_ids[2]; int nVarRefTblID = cmor_vars[var_id].ref_table_id; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; int itmpmsg, itmp2, itmp3; int maxStrLen; @@ -3802,9 +3818,21 @@ void cmor_define_dimensions(int var_id, int ncid, ics = pVar->shuffle; icd = pVar->deflate; icdl = pVar->deflate_level; - - ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd, - icdl); + icz = pVar->zstandard_level; + icqm = pVar->quantize_mode; + icqn = pVar->quantize_nsd; + + ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm, + icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NCError (%i: %s) defining compression\n! " @@ -4057,7 +4085,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, int *int_list = NULL; char mtype; int nelts; - int ics, icd, icdl; + int ics, icd, icdl, icz, icqm, icqn; cmor_add_traceback("cmor_grids_def"); /* -------------------------------------------------------------------- */ @@ -4324,8 +4352,29 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af, cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. ref_var_id]. deflate_level; - ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i], - ics, icd, icdl); + icz = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + zstandard_level; + icqm = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_mode; + icqn = + cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j]. + ref_var_id]. + quantize_nsd; + ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i], + icqm, icqn); + if (icd != 0) { + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, icd, icdl); + } else { + ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i], + ics, 0, 0); + ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i], + icz); + } if (ierr != NC_NOERR) { snprintf(msg, CMOR_MAX_STRING, "NetCDF Error (%i: %s) defining\n! " From 1a551f87c4ffce467c7f9add7eb60f61c7324d53 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 14 Aug 2024 08:08:18 -0700 Subject: [PATCH 10/14] Check if singleton id is -1 before using it as an axis index --- Test/test_singletons.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Test/test_singletons.c b/Test/test_singletons.c index f11af7e0..dbb164ba 100644 --- a/Test/test_singletons.c +++ b/Test/test_singletons.c @@ -33,7 +33,7 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base int var_id; float values[8], miss = 1.e20f; char positive = '\0'; - int i; + int i, singleton_id; int scatangle_found = 0; int wavelength_found = 0; @@ -44,7 +44,8 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base // Find singleton dimension for bs550aer for(i = 0; i < cmor_vars[var_id].ndims; ++i){ - if(strcmp(cmor_axes[cmor_vars[var_id].singleton_ids[i]].id, "wavelength") == 0) + singleton_id = cmor_vars[var_id].singleton_ids[i]; + if(singleton_id != -1 && strcmp(cmor_axes[singleton_id].id, "wavelength") == 0) wavelength_found++; } if(wavelength_found != 1) From 1e32b83c77c31342e35ac443c80e7ed9da52dc2f Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 14 Aug 2024 08:22:53 -0700 Subject: [PATCH 11/14] Separate check of singleton id from check that accesses the axis array --- Test/test_singletons.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Test/test_singletons.c b/Test/test_singletons.c index dbb164ba..e90e670e 100644 --- a/Test/test_singletons.c +++ b/Test/test_singletons.c @@ -45,8 +45,10 @@ int test_bs550aer(const int *axes_ids, int num_axes, int zfactor_id, double base // Find singleton dimension for bs550aer for(i = 0; i < cmor_vars[var_id].ndims; ++i){ singleton_id = cmor_vars[var_id].singleton_ids[i]; - if(singleton_id != -1 && strcmp(cmor_axes[singleton_id].id, "wavelength") == 0) - wavelength_found++; + if(singleton_id != -1) { + if(strcmp(cmor_axes[singleton_id].id, "wavelength") == 0) + wavelength_found++; + } } if(wavelength_found != 1) fail("error in singleton dimension"); From 61970906137345ed09e009a96003d090cd9bb434 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 26 Aug 2024 14:41:09 -0700 Subject: [PATCH 12/14] Test branch identical to PR but with less builds --- .circleci/config.yml | 20 ++++++++++---------- .github/workflows/nightly-build.yml | 17 ++--------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 597bb333..34dde007 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,8 +22,8 @@ aliases: name: setup_miniconda command: | source $BASH_ENV - if [[ $OS == 'osx-64' ]]; then - curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o miniconda.sh + if [[ $OS == 'osx-arm64' ]]; then + curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o miniconda.sh else curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh fi @@ -75,8 +75,8 @@ aliases: export ARTIFACT_DIR=`pwd`/artifacts/$OS mkdir -p $ARTIFACT_DIR - if [[ $OS == 'osx-64' ]]; then - export OS_NAME="osx_64" + if [[ $OS == 'osx-arm64' ]]; then + export OS_NAME="osx_arm64" else export OS_NAME="linux_64" fi @@ -123,12 +123,12 @@ executors: CONDA_COMPILERS: "gcc_linux-64 gfortran_linux-64" macos: macos: - xcode: "13.4.1" - resource_class: macos.x86.medium.gen2 + xcode: "15.4.0" + resource_class: macos.m1.medium.gen1 environment: - OS: "osx-64" + OS: "osx-arm64" PROJECT_DIR: "workdir/macos" - CONDA_COMPILERS: "clang_osx-64 gfortran_osx-64" + CONDA_COMPILERS: "clang_osx-arm64 gfortran_osx-arm64" jobs: build: @@ -196,8 +196,8 @@ workflows: - build: matrix: parameters: - os: [ linux ] - python_version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + os: [ linux, macos ] + python_version: [ "3.10" ] name: build-<< matrix.os >>-<< matrix.python_version >> - upload: diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 44f04844..945ce76d 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -1,12 +1,6 @@ name: CMOR Nightly Build run-name: CMOR Nightly Build -on: - push: - branches: - - "main" - pull_request: - branches: - - "main" +on: push jobs: nightly: @@ -21,14 +15,7 @@ jobs: C_COMPILER: clang_osx-arm64 FORTRAN_COMPILER: gfortran_osx-arm64 PROJECT_DIR: workdir/macos_arm64 - - RUNNER_OS: 'macos-13' - OS: osx-64 - OS_NAME: osx_64 - MINICONDA_INSTALLER_URL: https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - C_COMPILER: clang_osx-64 - FORTRAN_COMPILER: gfortran_osx-64 - PROJECT_DIR: workdir/macos_64 - python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_version: ['3.10'] runs-on: ${{ matrix.runner.RUNNER_OS }} env: PACKAGE_NAME: cmor From aa84d0c2c101471de97b2e9c2b9fbb6f1560e380 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 26 Aug 2024 14:57:44 -0700 Subject: [PATCH 13/14] Add macos-13 --- .github/workflows/nightly-build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 945ce76d..f94d700b 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -15,6 +15,13 @@ jobs: C_COMPILER: clang_osx-arm64 FORTRAN_COMPILER: gfortran_osx-arm64 PROJECT_DIR: workdir/macos_arm64 + - RUNNER_OS: 'macos-13' + OS: osx-64 + OS_NAME: osx_64 + MINICONDA_INSTALLER_URL: https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + C_COMPILER: clang_osx-64 + FORTRAN_COMPILER: gfortran_osx-64 + PROJECT_DIR: workdir/macos_64 python_version: ['3.10'] runs-on: ${{ matrix.runner.RUNNER_OS }} env: From 087e1d313487682a106c23199e0d8d277bbf6df5 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 26 Aug 2024 15:32:28 -0700 Subject: [PATCH 14/14] Launch github actions build only on PR --- .github/workflows/nightly-build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index f94d700b..1f1af0d4 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -1,6 +1,12 @@ name: CMOR Nightly Build run-name: CMOR Nightly Build -on: push +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" jobs: nightly: