diff --git a/CCDFDataModel/CProj4ToCF.cpp b/CCDFDataModel/CProj4ToCF.cpp index 8141ff6f..6eaa53f4 100644 --- a/CCDFDataModel/CProj4ToCF.cpp +++ b/CCDFDataModel/CProj4ToCF.cpp @@ -390,7 +390,6 @@ CProj4ToCF::~CProj4ToCF() {} int CProj4ToCF::convertProjToCF(CDF::Variable *projectionVariable, const char *proj4String) { // Create a list with key value pairs of projection options - std::vector projKVPList; CT::string proj4CTString; proj4CTString.copy(proj4String); @@ -401,11 +400,13 @@ int CProj4ToCF::convertProjToCF(CDF::Variable *projectionVariable, const char *p return 1; } for (size_t j = 0; j < projElements->count; j++) { - KVP *option = new KVP(); CT::string *element = projElements[j].splitToArray("="); - option->name.copy(&element[0]); - option->value.copy(&element[1]); - projKVPList.push_back(option); + if (element != nullptr && element->count == 2) { + KVP *option = new KVP(); + option->name.copy(&element[0]); + option->value.copy(&element[1]); + projKVPList.push_back(option); + } delete[] element; } delete[] projElements; diff --git a/Dockerfile b/Dockerfile index 1c0755b9..a1e4b416 100755 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ USER root LABEL maintainer="adaguc@knmi.nl" # Version should be same as in Definitions.h -LABEL version="2.26.0" +LABEL version="2.27.0" # Try to update image packages RUN apt-get -q -y update \ diff --git a/NEWS.md b/NEWS.md index 607fd293..d19b1e94 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +**Version 2.27.0 2024-08-28** +- Support PNG files without projection and time information +- A time dimension can now be set based on the date in the filename in the Layer configuration +- Un-georeferenced data can now be georeferenced by setting the projection and extent in the Layer configuration + **Version 2.26.0 2024-07-12** - Added EDR cube call for gridded datsets to ADAGUC. diff --git a/adagucserverEC/CDFObjectStore.cpp b/adagucserverEC/CDFObjectStore.cpp index b3ea5c00..596759c1 100644 --- a/adagucserverEC/CDFObjectStore.cpp +++ b/adagucserverEC/CDFObjectStore.cpp @@ -22,7 +22,6 @@ * limitations under the License. * ******************************************************************************/ - #include "CDFObjectStore.h" const char *CDFObjectStore::className = "CDFObjectStore"; #include "CConvertASCAT.h" @@ -39,6 +38,7 @@ const char *CDFObjectStore::className = "CDFObjectStore"; #include "CConvertLatLonGrid.h" #include "CDataReader.h" #include "CCDFCSVReader.h" +#include "utils/CDFObjectStoreUtils.h" // #define CDFOBJECTSTORE_DEBUG #define MAX_OPEN_FILES 500 extern CDFObjectStore cdfObjectStore; @@ -301,6 +301,7 @@ CDFObject *CDFObjectStore::getCDFObject(CDataSource *dataSource, CServerParams * cdfObject->applyNCMLFile(ncmlFileName.c_str()); } } + // Set metadata into the variable based on configuration settings if (dataSource->cfgLayer->Variable.size() > 0) { // Shorthand to variable configuration in the layer. auto *cfgVar = dataSource->cfgLayer->Variable[0]; @@ -331,6 +332,7 @@ CDFObject *CDFObjectStore::getCDFObject(CDataSource *dataSource, CServerParams * var->setAttributeText("standard_name", cfgVar->attr.standard_name); } } + handleAddDimension(dataSource, cdfObject, fileLocationToOpen); } } diff --git a/adagucserverEC/CDataReader.cpp b/adagucserverEC/CDataReader.cpp index 7189287c..b358f449 100755 --- a/adagucserverEC/CDataReader.cpp +++ b/adagucserverEC/CDataReader.cpp @@ -208,26 +208,54 @@ bool CDataReader::copyCRSFromConfigToDataSource(CDataSource *dataSource) const { return false; } - CREPORT_INFO_NODOC("Projection is obtained from ADAGUC config file", CReportMessage::Categories::GENERAL); + auto *projection = dataSource->cfgLayer->Projection[0]; + // CREPORT_INFO_NODOC("Projection is obtained from ADAGUC config file", CReportMessage::Categories::GENERAL); // Read the EPSG-code from configuration. - if (dataSource->cfgLayer->Projection[0]->attr.id.empty() == false) { - dataSource->nativeEPSG.copy(dataSource->cfgLayer->Projection[0]->attr.id.c_str()); + if (projection->attr.id.empty() == false) { + dataSource->nativeEPSG.copy(projection->attr.id.c_str()); } else { CT::string defaultEPSGCode = "EPSG:4326"; - CREPORT_WARN_NODOC(CT::string("Projection id not in config, using default value ") + defaultEPSGCode, CReportMessage::Categories::GENERAL); + // CREPORT_WARN_NODOC(CT::string("Projection id not in config, using default value ") + defaultEPSGCode, CReportMessage::Categories::GENERAL); dataSource->nativeEPSG.copy(defaultEPSGCode); } // Read proj4 string from configuration. - if (dataSource->cfgLayer->Projection[0]->attr.proj4.empty() == false) { - dataSource->nativeProj4.copy(dataSource->cfgLayer->Projection[0]->attr.proj4.c_str()); + if (projection->attr.proj4.empty() == false) { + dataSource->nativeProj4.copy(projection->attr.proj4.c_str()); } else { CT::string defaultProj4String = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"; - CREPORT_WARN_NODOC(CT::string("Proj4 string not in config, using default value ") + defaultProj4String, CReportMessage::Categories::GENERAL); + // CREPORT_WARN_NODOC(CT::string("Proj4 string not in config, using default value ") + defaultProj4String, CReportMessage::Categories::GENERAL); dataSource->nativeProj4.copy(defaultProj4String); } + // Set extent based on configuration setting in Layer element + if (projection->attr.minx != projection->attr.maxx) { + if (dataSource->varX->getType() != CDF_DOUBLE) { + CDBWarning("Overriding extent is only possible on coordinate variables with CDF_DOUBLE as datatype"); + } else { + // Casting to shorthand pointers + double *varXData = (double *)dataSource->varX->data; + double *varYData = (double *)dataSource->varY->data; + size_t dWidth = dataSource->varX->dimensionlinks[0]->getSize(); + size_t dHeight = dataSource->varY->dimensionlinks[0]->getSize(); + + // Set width, height and cellsize + dataSource->dWidth = dWidth; + dataSource->dHeight = dHeight; + dataSource->dfCellSizeX = (projection->attr.maxx - projection->attr.minx) / dWidth; + dataSource->dfCellSizeY = (projection->attr.maxy - projection->attr.miny) / dHeight; + + // Fill in the coordinate variables + for (size_t j = 0; j < dWidth; j += 1) { + varXData[j] = projection->attr.minx + j * dataSource->dfCellSizeX + dataSource->dfCellSizeX / 2; + } + for (size_t j = 0; j < dHeight; j += 1) { + varYData[j] = projection->attr.miny + j * dataSource->dfCellSizeY + dataSource->dfCellSizeY / 2; + } + } + } + return true; } @@ -815,9 +843,6 @@ bool CDataReader::calculateCellSizeAndBBox(CDataSource *dataSource, const CDF::V dataSource->dfBBOX[3] = dfdim_Y[0] - dataSource->dfCellSizeY / 2.0f; } - dataSource->origBBOXLeft = dataSource->dfBBOX[0]; - dataSource->origBBOXRight = dataSource->dfBBOX[2]; - return true; } diff --git a/adagucserverEC/CDataSource.cpp b/adagucserverEC/CDataSource.cpp index cb61b1d7..fb7a0022 100644 --- a/adagucserverEC/CDataSource.cpp +++ b/adagucserverEC/CDataSource.cpp @@ -1317,8 +1317,6 @@ CDataSource *CDataSource::clone() { d->dimYIndex = dimYIndex; d->stride2DMap = stride2DMap; d->useLonTransformation = useLonTransformation; - d->origBBOXLeft = origBBOXLeft; - d->origBBOXRight = origBBOXRight; d->dOrigWidth = dOrigWidth; d->lonTransformDone = lonTransformDone; d->swapXYDimensions = swapXYDimensions; diff --git a/adagucserverEC/CDataSource.h b/adagucserverEC/CDataSource.h index c25a4a56..f23cecea 100644 --- a/adagucserverEC/CDataSource.h +++ b/adagucserverEC/CDataSource.h @@ -264,7 +264,6 @@ class CDataSource { // Lon transformation is used to swap datasets from 0-360 degrees to -180 till 180 degrees // Swap data from >180 degrees to domain of -180 till 180 in case of lat lon source data int useLonTransformation; - double origBBOXLeft, origBBOXRight; int dOrigWidth; bool lonTransformDone; diff --git a/adagucserverEC/CMakeLists.txt b/adagucserverEC/CMakeLists.txt index 942d0bc4..c9beab6b 100644 --- a/adagucserverEC/CMakeLists.txt +++ b/adagucserverEC/CMakeLists.txt @@ -68,6 +68,8 @@ add_library( CDBFileScannerCleanFiles.cpp CDFObjectStore.cpp CDFObjectStore.h + utils/CDFObjectStoreUtils.h + utils/CDFObjectStoreUtils.cpp CDataPostProcessors/CDataPostProcessor_AddFeatures.cpp CDataPostProcessors/CDataPostProcessor_AddFeatures.h CDataPostProcessors/CDataPostProcessor_CDPDBZtoRR.cpp diff --git a/adagucserverEC/CServerConfig_CPPXSD.h b/adagucserverEC/CServerConfig_CPPXSD.h index 96b3224c..ed7717ba 100644 --- a/adagucserverEC/CServerConfig_CPPXSD.h +++ b/adagucserverEC/CServerConfig_CPPXSD.h @@ -1211,6 +1211,42 @@ class CServerConfig : public CXMLSerializerInterface { } }; + class XMLE_AddDimension : public CXMLObjectInterface { + public: + class Cattr { + public: + CT::string name, type, year, month, day, hour, minute, second; + } attr; + void addAttribute(const char *attrname, const char *attrvalue) { + if (equals("name", attrname)) { + attr.name.copy(attrvalue); + return; + } else if (equals("type", attrname)) { + attr.type.copy(attrvalue); + return; + } else if (equals("year", attrname)) { + attr.year.copy(attrvalue); + return; + } else if (equals("month", attrname)) { + attr.month.copy(attrvalue); + return; + } else if (equals("day", attrname)) { + attr.day.copy(attrvalue); + return; + } else if (equals("hour", attrname)) { + attr.hour.copy(attrvalue); + return; + } else if (equals("minute", attrname)) { + attr.minute.copy(attrvalue); + return; + } else if (equals("second", attrname)) { + attr.second.copy(attrvalue); + return; + } + return; + } + }; + class XMLE_Dimension : public CXMLObjectInterface { public: class Cattr { @@ -1355,17 +1391,28 @@ class CServerConfig : public CXMLSerializerInterface { class Cattr { public: CT::string id, proj4; + double minx = 0, miny = 0, maxx = 0, maxy = 0; } attr; std::vector LatLonBox; ~XMLE_Projection() { XMLE_DELOBJ(LatLonBox); } void addAttribute(const char *attrname, const char *attrvalue) { if (equals("id", attrname)) { attr.id.copy(attrvalue); - return; - } - if (equals("proj4", attrname)) { + } else if (equals("proj4", attrname)) { attr.proj4.copy(attrvalue); return; + } else if (equals("minx", attrname)) { + attr.minx = parseDouble(attrvalue); + return; + } else if (equals("miny", attrname)) { + attr.miny = parseDouble(attrvalue); + return; + } else if (equals("maxx", attrname)) { + attr.maxx = parseDouble(attrvalue); + return; + } else if (equals("maxy", attrname)) { + attr.maxy = parseDouble(attrvalue); + return; } } void addElement(CXMLObjectInterface *baseClass, int rc, const char *name, const char *value) { @@ -1770,6 +1817,7 @@ class CServerConfig : public CXMLSerializerInterface { std::vector TileSettings; std::vector DataReader; std::vector Dimension; + std::vector AddDimension; std::vector Legend; std::vector Scale; std::vector Offset; @@ -1808,6 +1856,7 @@ class CServerConfig : public CXMLSerializerInterface { XMLE_DELOBJ(TileSettings) XMLE_DELOBJ(DataReader); XMLE_DELOBJ(Dimension); + XMLE_DELOBJ(AddDimension); XMLE_DELOBJ(Legend); XMLE_DELOBJ(Scale); XMLE_DELOBJ(Offset); @@ -1863,6 +1912,8 @@ class CServerConfig : public CXMLSerializerInterface { XMLE_ADDOBJ(DataReader); } else if (equals("Dimension", name)) { XMLE_ADDOBJ(Dimension); + } else if (equals("AddDimension", name)) { + XMLE_ADDOBJ(AddDimension); } else if (equals("Legend", name)) { XMLE_ADDOBJ(Legend); } else if (equals("Scale", name)) { diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h index 0958c919..74e33a56 100755 --- a/adagucserverEC/Definitions.h +++ b/adagucserverEC/Definitions.h @@ -28,7 +28,7 @@ #ifndef Definitions_H #define Definitions_H -#define ADAGUCSERVER_VERSION "2.26.0" // Please also update in the Dockerfile to the same version +#define ADAGUCSERVER_VERSION "2.27.0" // Please also update in the Dockerfile to the same version // CConfigReaderLayerType #define CConfigReaderLayerTypeUnknown 0 diff --git a/adagucserverEC/utils/CDFObjectStoreUtils.cpp b/adagucserverEC/utils/CDFObjectStoreUtils.cpp new file mode 100644 index 00000000..bcf388c2 --- /dev/null +++ b/adagucserverEC/utils/CDFObjectStoreUtils.cpp @@ -0,0 +1,104 @@ +/****************************************************************************** + * + * Project: ADAGUC Server + * Purpose: ADAGUC OGC Server + * Author: Maarten Plieger, plieger "at" knmi.nl + * Date: 2024-08-28 + * + ****************************************************************************** + * + * Copyright 2013, Royal Netherlands Meteorological Institute (KNMI) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ +#include "CDFObjectStoreUtils.h" +#include "CServerConfig_CPPXSD.h" +#include +#include +#include +#include "CDataSource.h" +#include + +int regex_match_item_return_int(std::string baseName, CT::string itemToTest) { + std::regex rgx(itemToTest); + std::smatch matches; + if (std::regex_search(baseName, matches, rgx)) { + if (matches.size() == 2) { + return atoi(matches[1].str().c_str()); + } + } + return 0; +} + +CDF::Variable *add_dimension_variable(CDF::Variable *sourceVariable, CT::string dimensionNameToAdd) { + CDFObject *cdfObject = (CDFObject *)sourceVariable->getParentCDFObject(); + + // Create the new dimension if not available + CDF::Dimension *cdfDimensionToAdd = cdfObject->getDimensionNE(dimensionNameToAdd); + if (cdfDimensionToAdd == nullptr) { + cdfDimensionToAdd = cdfObject->addDimension(new CDF::Dimension("time", 1)); + } + + // Create the new variable if not available + CDF::Variable *newDimensionVariable = cdfObject->getVariableNE(dimensionNameToAdd); + if (newDimensionVariable == nullptr) { + newDimensionVariable = cdfObject->addVariable(new CDF::Variable(dimensionNameToAdd, CDF_DOUBLE, &cdfDimensionToAdd, 1, true)); + newDimensionVariable->allocateData(1); + newDimensionVariable->setAttributeText("units", "seconds since 1970-01-01 0:0:0"); + newDimensionVariable->setAttributeText("standard_name", dimensionNameToAdd.c_str()); + newDimensionVariable->setAttributeText("info", "added by adaguc-server add_dimension_variable routines"); + } + + // Make sure the new dimensionvariable is part of the dimensionlist of the source variable, but do not add it twice. + if (sourceVariable->getDimensionNE(dimensionNameToAdd) == nullptr) { + sourceVariable->dimensionlinks.insert(sourceVariable->dimensionlinks.begin(), cdfDimensionToAdd); + } + + return newDimensionVariable; +} + +void filename_regexp_extracttime(CDataSource *dataSource, CDFObject *cdfObject, CT::string fileLocationToOpen, CServerConfig::XMLE_AddDimension *addDimensionElement) { + + auto dimensionNameToAdd = addDimensionElement->attr.name; + auto *cfgVariable = dataSource->cfgLayer->Variable[0]; + auto *layerVariable = cdfObject->getVariable(cfgVariable->value); + auto *timeVariable = add_dimension_variable(layerVariable, dimensionNameToAdd); + auto *ctime = CTime::GetCTimeInstance(timeVariable); + if (ctime == nullptr) { + throw(CTIME_GETINSTANCE_ERROR_MESSAGE); + } + + std::string baseName = fileLocationToOpen.basename().c_str(); + int year = regex_match_item_return_int(baseName, addDimensionElement->attr.year); + int month = regex_match_item_return_int(baseName, addDimensionElement->attr.month); + int day = regex_match_item_return_int(baseName, addDimensionElement->attr.day); + int hour = regex_match_item_return_int(baseName, addDimensionElement->attr.hour); + int minute = regex_match_item_return_int(baseName, addDimensionElement->attr.minute); + int second = regex_match_item_return_int(baseName, addDimensionElement->attr.second); + + CT::string dateString; + dateString.print("%04d-%02d-%02dT%02d:%02d:%02dZ", year, month, day, hour, minute, second); + timeVariable->setAttributeText("info_datestring", dateString.c_str()); + + // Convert the timestring into the double value matching the units. + ((double *)timeVariable->data)[0] = ctime->dateToOffset(ctime->freeDateStringToDate(dateString)); +} + +void handleAddDimension(CDataSource *dataSource, CDFObject *cdfObject, CT::string fileLocationToOpen) { + for (auto a : dataSource->cfgLayer->AddDimension) { + if (a->attr.type.equals(ADDDIMENSION_TYPE_REGEXP_EXTRACTTIME)) { + filename_regexp_extracttime(dataSource, cdfObject, fileLocationToOpen, a); + } + } +} diff --git a/adagucserverEC/utils/CDFObjectStoreUtils.h b/adagucserverEC/utils/CDFObjectStoreUtils.h new file mode 100644 index 00000000..321ec375 --- /dev/null +++ b/adagucserverEC/utils/CDFObjectStoreUtils.h @@ -0,0 +1,69 @@ +/****************************************************************************** + * + * Project: ADAGUC Server + * Purpose: ADAGUC OGC Server + * Author: Maarten Plieger, plieger "at" knmi.nl + * Date: 2024-08-28 + * + ****************************************************************************** + * + * Copyright 2013, Royal Netherlands Meteorological Institute (KNMI) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +#ifndef CDFOBJECTSTOREUTILS_H +#define CDFOBJECTSTOREUTILS_H + +#include "CDataSource.h" + +#define ADDDIMENSION_TYPE_REGEXP_EXTRACTTIME "filename_regexp_extracttime" + +/** + * Allows adding a custom dimension. This is useful for files which do not have a time dimension. + * The dimension can be added and populated with information from for example the filename using regular expressions. + * + * The following configuration for a layer will do this for the file /data/adaguc-data/MTG/day_highres_knmi_weur_20240731_72.png. + * The inforomation is extracted from the filename using regular expressions, see the AddDimension section. + * + + + day_highres_knmi_weur + day_highres_knmi_weur + /data/adaguc-data/MTG/ + + time + + pngdata + rgba + + +* +*/ +void handleAddDimension(CDataSource *dataSource, CDFObject *cdfObject, CT::string fileLocationToOpen); + +#endif \ No newline at end of file