Support data which has no georeference or time information in it

CCDFDataModel/CProj4ToCF.cpp

@@ -390,7 +390,6 @@ CProj4ToCF::~CProj4ToCF() {}
 
 int CProj4ToCF::convertProjToCF(CDF::Variable *projectionVariable, const char *proj4String) {
   // Create a list with key value pairs of projection options
-
   std::vector<CProj4ToCF::KVP *> projKVPList;
   CT::string proj4CTString;
   proj4CTString.copy(proj4String);
@@ -401,11 +400,13 @@ int CProj4ToCF::convertProjToCF(CDF::Variable *projectionVariable, const char *p
     return 1;
   }
   for (size_t j = 0; j < projElements->count; j++) {
-    KVP *option = new KVP();
     CT::string *element = projElements[j].splitToArray("=");
-    option->name.copy(&element[0]);
-    option->value.copy(&element[1]);
-    projKVPList.push_back(option);
+    if (element != nullptr && element->count == 2) {
+      KVP *option = new KVP();
+      option->name.copy(&element[0]);
+      option->value.copy(&element[1]);
+      projKVPList.push_back(option);
+    }
     delete[] element;
   }
   delete[] projElements;

Dockerfile

@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="[email protected]"
 
 # Version should be same as in Definitions.h
-LABEL version="2.26.0"
+LABEL version="2.27.0"
 
 # Try to update image packages
 RUN apt-get -q -y update \

NEWS.md

@@ -1,3 +1,8 @@
+**Version 2.27.0 2024-08-28**
+- Support PNG files without projection and time information
+- A time dimension can now be set based on the date in the filename in the Layer configuration
+- Un-georeferenced data can now be georeferenced by setting the projection and extent in the Layer configuration
+
 **Version 2.26.0 2024-07-12**
 - Added EDR cube call for gridded datsets to ADAGUC.
 

adagucserverEC/CDFObjectStore.cpp

@@ -22,7 +22,6 @@
  * limitations under the License.
  *
  ******************************************************************************/
-
 #include "CDFObjectStore.h"
 const char *CDFObjectStore::className = "CDFObjectStore";
 #include "CConvertASCAT.h"
@@ -39,6 +38,7 @@ const char *CDFObjectStore::className = "CDFObjectStore";
 #include "CConvertLatLonGrid.h"
 #include "CDataReader.h"
 #include "CCDFCSVReader.h"
+#include "utils/CDFObjectStoreUtils.h"
 // #define CDFOBJECTSTORE_DEBUG
 #define MAX_OPEN_FILES 500
 extern CDFObjectStore cdfObjectStore;
@@ -301,6 +301,7 @@ CDFObject *CDFObjectStore::getCDFObject(CDataSource *dataSource, CServerParams *
           cdfObject->applyNCMLFile(ncmlFileName.c_str());
         }
       }
+      // Set metadata into the variable based on configuration settings
       if (dataSource->cfgLayer->Variable.size() > 0) {
         // Shorthand to variable configuration in the layer.
         auto *cfgVar = dataSource->cfgLayer->Variable[0];
@@ -331,6 +332,7 @@ CDFObject *CDFObjectStore::getCDFObject(CDataSource *dataSource, CServerParams *
           var->setAttributeText("standard_name", cfgVar->attr.standard_name);
         }
       }
+      handleAddDimension(dataSource, cdfObject, fileLocationToOpen);
     }
   }
 

adagucserverEC/CDataReader.cpp

@@ -208,26 +208,54 @@ bool CDataReader::copyCRSFromConfigToDataSource(CDataSource *dataSource) const {
     return false;
   }
 
-  CREPORT_INFO_NODOC("Projection is obtained from ADAGUC config file", CReportMessage::Categories::GENERAL);
+  auto *projection = dataSource->cfgLayer->Projection[0];
+  // CREPORT_INFO_NODOC("Projection is obtained from ADAGUC config file", CReportMessage::Categories::GENERAL);
 
   // Read the EPSG-code from configuration.
-  if (dataSource->cfgLayer->Projection[0]->attr.id.empty() == false) {
-    dataSource->nativeEPSG.copy(dataSource->cfgLayer->Projection[0]->attr.id.c_str());
+  if (projection->attr.id.empty() == false) {
+    dataSource->nativeEPSG.copy(projection->attr.id.c_str());
   } else {
     CT::string defaultEPSGCode = "EPSG:4326";
-    CREPORT_WARN_NODOC(CT::string("Projection id not in config, using default value ") + defaultEPSGCode, CReportMessage::Categories::GENERAL);
+    // CREPORT_WARN_NODOC(CT::string("Projection id not in config, using default value ") + defaultEPSGCode, CReportMessage::Categories::GENERAL);
     dataSource->nativeEPSG.copy(defaultEPSGCode);
   }
 
   // Read proj4 string from configuration.
-  if (dataSource->cfgLayer->Projection[0]->attr.proj4.empty() == false) {
-    dataSource->nativeProj4.copy(dataSource->cfgLayer->Projection[0]->attr.proj4.c_str());
+  if (projection->attr.proj4.empty() == false) {
+    dataSource->nativeProj4.copy(projection->attr.proj4.c_str());
   } else {
     CT::string defaultProj4String = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs";
-    CREPORT_WARN_NODOC(CT::string("Proj4 string not in config, using default value ") + defaultProj4String, CReportMessage::Categories::GENERAL);
+    // CREPORT_WARN_NODOC(CT::string("Proj4 string not in config, using default value ") + defaultProj4String, CReportMessage::Categories::GENERAL);
     dataSource->nativeProj4.copy(defaultProj4String);
   }
 
+  // Set extent based on configuration setting in Layer element
+  if (projection->attr.minx != projection->attr.maxx) {
+    if (dataSource->varX->getType() != CDF_DOUBLE) {
+      CDBWarning("Overriding extent is only possible on coordinate variables with CDF_DOUBLE as datatype");
+    } else {
+      // Casting to shorthand pointers
+      double *varXData = (double *)dataSource->varX->data;
+      double *varYData = (double *)dataSource->varY->data;
+      size_t dWidth = dataSource->varX->dimensionlinks[0]->getSize();
+      size_t dHeight = dataSource->varY->dimensionlinks[0]->getSize();
+
+      // Set width, height and cellsize
+      dataSource->dWidth = dWidth;
+      dataSource->dHeight = dHeight;
+      dataSource->dfCellSizeX = (projection->attr.maxx - projection->attr.minx) / dWidth;
+      dataSource->dfCellSizeY = (projection->attr.maxy - projection->attr.miny) / dHeight;
+
+      // Fill in the coordinate variables
+      for (size_t j = 0; j < dWidth; j += 1) {
+        varXData[j] = projection->attr.minx + j * dataSource->dfCellSizeX + dataSource->dfCellSizeX / 2;
+      }
+      for (size_t j = 0; j < dHeight; j += 1) {
+        varYData[j] = projection->attr.miny + j * dataSource->dfCellSizeY + dataSource->dfCellSizeY / 2;
+      }
+    }
+  }
+
   return true;
 }
 
@@ -815,9 +843,6 @@ bool CDataReader::calculateCellSizeAndBBox(CDataSource *dataSource, const CDF::V
     dataSource->dfBBOX[3] = dfdim_Y[0] - dataSource->dfCellSizeY / 2.0f;
   }
 
-  dataSource->origBBOXLeft = dataSource->dfBBOX[0];
-  dataSource->origBBOXRight = dataSource->dfBBOX[2];
-
   return true;
 }
 

adagucserverEC/CDataSource.cpp

@@ -1317,8 +1317,6 @@ CDataSource *CDataSource::clone() {
   d->dimYIndex = dimYIndex;
   d->stride2DMap = stride2DMap;
   d->useLonTransformation = useLonTransformation;
-  d->origBBOXLeft = origBBOXLeft;
-  d->origBBOXRight = origBBOXRight;
   d->dOrigWidth = dOrigWidth;
   d->lonTransformDone = lonTransformDone;
   d->swapXYDimensions = swapXYDimensions;

adagucserverEC/CDataSource.h

@@ -264,7 +264,6 @@ class CDataSource {
   // Lon transformation is used to swap datasets from 0-360 degrees to -180 till 180 degrees
   // Swap data from >180 degrees to domain of -180 till 180 in case of lat lon source data
   int useLonTransformation;
-  double origBBOXLeft, origBBOXRight;
   int dOrigWidth;
   bool lonTransformDone;
 

adagucserverEC/CMakeLists.txt

@@ -68,6 +68,8 @@ add_library(
     CDBFileScannerCleanFiles.cpp
     CDFObjectStore.cpp
     CDFObjectStore.h
+    utils/CDFObjectStoreUtils.h
+    utils/CDFObjectStoreUtils.cpp
     CDataPostProcessors/CDataPostProcessor_AddFeatures.cpp
     CDataPostProcessors/CDataPostProcessor_AddFeatures.h
     CDataPostProcessors/CDataPostProcessor_CDPDBZtoRR.cpp

adagucserverEC/CServerConfig_CPPXSD.h

@@ -1211,6 +1211,42 @@ class CServerConfig : public CXMLSerializerInterface {
     }
   };
 
+  class XMLE_AddDimension : public CXMLObjectInterface {
+  public:
+    class Cattr {
+    public:
+      CT::string name, type, year, month, day, hour, minute, second;
+    } attr;
+    void addAttribute(const char *attrname, const char *attrvalue) {
+      if (equals("name", attrname)) {
+        attr.name.copy(attrvalue);
+        return;
+      } else if (equals("type", attrname)) {
+        attr.type.copy(attrvalue);
+        return;
+      } else if (equals("year", attrname)) {
+        attr.year.copy(attrvalue);
+        return;
+      } else if (equals("month", attrname)) {
+        attr.month.copy(attrvalue);
+        return;
+      } else if (equals("day", attrname)) {
+        attr.day.copy(attrvalue);
+        return;
+      } else if (equals("hour", attrname)) {
+        attr.hour.copy(attrvalue);
+        return;
+      } else if (equals("minute", attrname)) {
+        attr.minute.copy(attrvalue);
+        return;
+      } else if (equals("second", attrname)) {
+        attr.second.copy(attrvalue);
+        return;
+      }
+      return;
+    }
+  };
+
   class XMLE_Dimension : public CXMLObjectInterface {
   public:
     class Cattr {
@@ -1355,17 +1391,28 @@ class CServerConfig : public CXMLSerializerInterface {
     class Cattr {
     public:
       CT::string id, proj4;
+      double minx = 0, miny = 0, maxx = 0, maxy = 0;
     } attr;
     std::vector<XMLE_LatLonBox *> LatLonBox;
     ~XMLE_Projection() { XMLE_DELOBJ(LatLonBox); }
     void addAttribute(const char *attrname, const char *attrvalue) {
       if (equals("id", attrname)) {
         attr.id.copy(attrvalue);
-        return;
-      }
-      if (equals("proj4", attrname)) {
+      } else if (equals("proj4", attrname)) {
         attr.proj4.copy(attrvalue);
         return;
+      } else if (equals("minx", attrname)) {
+        attr.minx = parseDouble(attrvalue);
+        return;
+      } else if (equals("miny", attrname)) {
+        attr.miny = parseDouble(attrvalue);
+        return;
+      } else if (equals("maxx", attrname)) {
+        attr.maxx = parseDouble(attrvalue);
+        return;
+      } else if (equals("maxy", attrname)) {
+        attr.maxy = parseDouble(attrvalue);
+        return;
       }
     }
     void addElement(CXMLObjectInterface *baseClass, int rc, const char *name, const char *value) {
@@ -1770,6 +1817,7 @@ class CServerConfig : public CXMLSerializerInterface {
     std::vector<XMLE_TileSettings *> TileSettings;
     std::vector<XMLE_DataReader *> DataReader;
     std::vector<XMLE_Dimension *> Dimension;
+    std::vector<XMLE_AddDimension *> AddDimension;
     std::vector<XMLE_Legend *> Legend;
     std::vector<XMLE_Scale *> Scale;
     std::vector<XMLE_Offset *> Offset;
@@ -1808,6 +1856,7 @@ class CServerConfig : public CXMLSerializerInterface {
       XMLE_DELOBJ(TileSettings)
       XMLE_DELOBJ(DataReader);
       XMLE_DELOBJ(Dimension);
+      XMLE_DELOBJ(AddDimension);
       XMLE_DELOBJ(Legend);
       XMLE_DELOBJ(Scale);
       XMLE_DELOBJ(Offset);
@@ -1863,6 +1912,8 @@ class CServerConfig : public CXMLSerializerInterface {
           XMLE_ADDOBJ(DataReader);
         } else if (equals("Dimension", name)) {
           XMLE_ADDOBJ(Dimension);
+        } else if (equals("AddDimension", name)) {
+          XMLE_ADDOBJ(AddDimension);
         } else if (equals("Legend", name)) {
           XMLE_ADDOBJ(Legend);
         } else if (equals("Scale", name)) {

adagucserverEC/Definitions.h

@@ -28,7 +28,7 @@
 #ifndef Definitions_H
 #define Definitions_H
 
-#define ADAGUCSERVER_VERSION "2.26.0" // Please also update in the Dockerfile to the same version
+#define ADAGUCSERVER_VERSION "2.27.0" // Please also update in the Dockerfile to the same version
 
 // CConfigReaderLayerType
 #define CConfigReaderLayerTypeUnknown 0

adagucserverEC/utils/CDFObjectStoreUtils.cpp

@@ -0,0 +1,104 @@
+/******************************************************************************
+ *
+ * Project:  ADAGUC Server
+ * Purpose:  ADAGUC OGC Server
+ * Author:   Maarten Plieger, plieger "at" knmi.nl
+ * Date:     2024-08-28
+ *
+ ******************************************************************************
+ *
+ * Copyright 2013, Royal Netherlands Meteorological Institute (KNMI)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+#include "CDFObjectStoreUtils.h"
+#include "CServerConfig_CPPXSD.h"
+#include <iostream>
+#include <string>
+#include <regex>
+#include "CDataSource.h"
+#include <CTime.h>
+
+int regex_match_item_return_int(std::string baseName, CT::string itemToTest) {
+  std::regex rgx(itemToTest);
+  std::smatch matches;
+  if (std::regex_search(baseName, matches, rgx)) {
+    if (matches.size() == 2) {
+      return atoi(matches[1].str().c_str());
+    }
+  }
+  return 0;
+}
+
+CDF::Variable *add_dimension_variable(CDF::Variable *sourceVariable, CT::string dimensionNameToAdd) {
+  CDFObject *cdfObject = (CDFObject *)sourceVariable->getParentCDFObject();
+
+  // Create the new dimension if not available
+  CDF::Dimension *cdfDimensionToAdd = cdfObject->getDimensionNE(dimensionNameToAdd);
+  if (cdfDimensionToAdd == nullptr) {
+    cdfDimensionToAdd = cdfObject->addDimension(new CDF::Dimension("time", 1));
+  }
+
+  // Create the new variable if not available
+  CDF::Variable *newDimensionVariable = cdfObject->getVariableNE(dimensionNameToAdd);
+  if (newDimensionVariable == nullptr) {
+    newDimensionVariable = cdfObject->addVariable(new CDF::Variable(dimensionNameToAdd, CDF_DOUBLE, &cdfDimensionToAdd, 1, true));
+    newDimensionVariable->allocateData(1);
+    newDimensionVariable->setAttributeText("units", "seconds since 1970-01-01 0:0:0");
+    newDimensionVariable->setAttributeText("standard_name", dimensionNameToAdd.c_str());
+    newDimensionVariable->setAttributeText("info", "added by adaguc-server add_dimension_variable routines");
+  }
+
+  // Make sure the new dimensionvariable is part of the dimensionlist of the source variable, but do not add it twice.
+  if (sourceVariable->getDimensionNE(dimensionNameToAdd) == nullptr) {
+    sourceVariable->dimensionlinks.insert(sourceVariable->dimensionlinks.begin(), cdfDimensionToAdd);
+  }
+
+  return newDimensionVariable;
+}
+
+void filename_regexp_extracttime(CDataSource *dataSource, CDFObject *cdfObject, CT::string fileLocationToOpen, CServerConfig::XMLE_AddDimension *addDimensionElement) {
+
+  auto dimensionNameToAdd = addDimensionElement->attr.name;
+  auto *cfgVariable = dataSource->cfgLayer->Variable[0];
+  auto *layerVariable = cdfObject->getVariable(cfgVariable->value);
+  auto *timeVariable = add_dimension_variable(layerVariable, dimensionNameToAdd);
+  auto *ctime = CTime::GetCTimeInstance(timeVariable);
+  if (ctime == nullptr) {
+    throw(CTIME_GETINSTANCE_ERROR_MESSAGE);
+  }
+
+  std::string baseName = fileLocationToOpen.basename().c_str();
+  int year = regex_match_item_return_int(baseName, addDimensionElement->attr.year);
+  int month = regex_match_item_return_int(baseName, addDimensionElement->attr.month);
+  int day = regex_match_item_return_int(baseName, addDimensionElement->attr.day);
+  int hour = regex_match_item_return_int(baseName, addDimensionElement->attr.hour);
+  int minute = regex_match_item_return_int(baseName, addDimensionElement->attr.minute);
+  int second = regex_match_item_return_int(baseName, addDimensionElement->attr.second);
+
+  CT::string dateString;
+  dateString.print("%04d-%02d-%02dT%02d:%02d:%02dZ", year, month, day, hour, minute, second);
+  timeVariable->setAttributeText("info_datestring", dateString.c_str());
+
+  // Convert the timestring into the double value matching the units.
+  ((double *)timeVariable->data)[0] = ctime->dateToOffset(ctime->freeDateStringToDate(dateString));
+}
+
+void handleAddDimension(CDataSource *dataSource, CDFObject *cdfObject, CT::string fileLocationToOpen) {
+  for (auto a : dataSource->cfgLayer->AddDimension) {
+    if (a->attr.type.equals(ADDDIMENSION_TYPE_REGEXP_EXTRACTTIME)) {
+      filename_regexp_extracttime(dataSource, cdfObject, fileLocationToOpen, a);
+    }
+  }
+}