Merge pull request #102 from metno/origin/issue90_parent_child_relati…

…onship issue90 parent child relationship
metno · Oct 5, 2022 · 2c3a9c9 · 2c3a9c9
2 parents a9560d4 + f5d5b0c
commit 2c3a9c9
Show file tree

Hide file tree

Showing 14 changed files with 711 additions and 56 deletions.
diff --git a/dmci/api/app.py b/dmci/api/app.py
@@ -47,6 +47,10 @@ def __init__(self):
             logger.error("Parameter mmd_xsd_path in config is not set")
             sys.exit(1)
 
+        if self._conf.path_to_parent_list is None:
+            logger.error("Parameter path_to_parent_list in config is not set")
+            sys.exit(1)
+
         # Create the XML Validator Object
         try:
             self._xsd_obj = etree.XMLSchema(etree.parse(self._conf.mmd_xsd_path))
@@ -122,7 +126,8 @@ def _insert_update_method_post(self, cmd, request):
             return msg, code
 
         # Run the validator
-        worker = Worker(cmd, full_path, self._xsd_obj)
+        worker = Worker(cmd, full_path, self._xsd_obj,
+                        path_to_parent_list=self._conf.path_to_parent_list)
         valid, msg = worker.validate(data)
         if not valid:
             self._handle_persist_file(False, full_path, reject_path, msg)
@@ -155,7 +160,8 @@ def _validate_method_post(self, request):
             return msg, code
 
         # Run the validator
-        worker = Worker("none", full_path, self._xsd_obj)
+        worker = Worker("none", full_path, self._xsd_obj,
+                        path_to_parent_list=self._conf.path_to_parent_list)
         valid, msg = worker.validate(data)
         self._handle_persist_file(True, full_path)
         if valid:

diff --git a/dmci/api/worker.py b/dmci/api/worker.py
@@ -125,7 +125,8 @@ def distribute(self):
                 self._dist_cmd,
                 xml_file=self._dist_xml_file,
                 metadata_id=self._dist_metadata_id,
-                worker=self
+                worker=self,
+                path_to_parent_list=self._kwargs.get('path_to_parent_list', None)
             )
             valid &= obj.is_valid()
             if obj.is_valid():

diff --git a/dmci/config.py b/dmci/config.py
@@ -38,6 +38,7 @@ def __init__(self):
         self.max_permitted_size = 100000  # Size of files permitted through API
         self.mmd_xslt_path = None
         self.mmd_xsd_path = None
+        self.path_to_parent_list = None
 
         # PyCSW Distributor
         self.csw_service_url = None
@@ -93,6 +94,7 @@ def _read_core(self):
         self.max_permitted_size = conf.get("max_permitted_size", self.max_permitted_size)
         self.mmd_xslt_path = conf.get("mmd_xslt_path", self.mmd_xslt_path)
         self.mmd_xsd_path = conf.get("mmd_xsd_path", self.mmd_xsd_path)
+        self.path_to_parent_list = conf.get("path_to_parent_list", self.path_to_parent_list)
 
         return
 
@@ -122,6 +124,7 @@ def _validate_config(self):
         valid &= self._check_file_exists(self.mmd_xsd_path, "mmd_xsd_path")
         valid &= self._check_folder_exists(self.distributor_cache, "distributor_cache")
         valid &= self._check_folder_exists(self.rejected_jobs_path, "rejected_jobs_path")
+        valid &= self._check_file_exists(self.path_to_parent_list, "path_to_parent_list")
 
         if "pycsw" in self.call_distributors:
             valid &= self._check_file_exists(self.mmd_xslt_path, "mmd_xslt_path")

diff --git a/dmci/distributors/distributor.py b/dmci/distributors/distributor.py
@@ -46,6 +46,7 @@ def __init__(self, cmd, xml_file=None, metadata_id=None, worker=None, **kwargs):
 
         self._cmd = None
         self._xml_file = None
+        self._path_to_parent_list = None
         self._metadata_id = None
         self._worker = worker
         self._kwargs = kwargs
@@ -77,12 +78,21 @@ def __init__(self, cmd, xml_file=None, metadata_id=None, worker=None, **kwargs):
                     logger.error("Metadata identifier must be a valid UUID")
                     self._valid = False
                     return
-
         else:
             logger.error("Either xml_file or metadata_id must be specified, but not both")
             self._valid = False
             return
 
+        self._path_to_parent_list = kwargs.get('path_to_parent_list', None)
+        if self._path_to_parent_list is not None:
+            if os.path.isfile(kwargs['path_to_parent_list']):
+                self._path_to_parent_list = kwargs['path_to_parent_list']
+                self._valid = True
+            else:
+                logger.error("File does not exist: %s" % str(kwargs['path_to_parent_list']))
+                self._valid = False
+                return
+
         # Check consistency between command and data
         if self._cmd in (DistCmd.UPDATE, DistCmd.INSERT) and self._xml_file is None:
             logger.error("Command '%s' requires `xml_file` to be specified" % str(cmd))

diff --git a/dmci/distributors/pycsw_dist.py b/dmci/distributors/pycsw_dist.py
@@ -67,7 +67,10 @@ def _translate(self):
         try:
             xml_doc = etree.ElementTree(file=self._xml_file)
             transform = etree.XSLT(etree.parse(self._conf.mmd_xslt_path))
-            new_doc = transform(xml_doc)
+            # If the dataset is a parent dataset, the
+            # self._xml_file needs to contain the string "parent"
+            new_doc = transform(xml_doc, path_to_parent_list=etree.XSLT.strparam(
+                self._path_to_parent_list))
             result = etree.tostring(new_doc, pretty_print=False, encoding="utf-8")
         except Exception as e:
             logger.error("Failed to translate MMD to ISO19139")

diff --git a/example_config.yaml b/example_config.yaml
@@ -7,6 +7,7 @@ dmci:
   max_permitted_size: 100000
   mmd_xslt_path: null
   mmd_xsd_path: null
+  path_to_parent_list: null
 
 pycsw:
   csw_service_url: http://localhost

diff --git a/tests/files/api/aqua-modis-parent.xml b/tests/files/api/aqua-modis-parent.xml
@@ -0,0 +1,105 @@
+<mmd:mmd xmlns:mmd="http://www.met.no/schema/mmd" xmlns:gml="http://www.opengis.net/gml">
+  <mmd:metadata_identifier>64db6102-14ce-41e9-b93b-61dbb2cb8b4e</mmd:metadata_identifier>
+  <mmd:title xml:lang="en">Direct Broadcast data processed in satellite swath to L1C.</mmd:title>
+  <mmd:title xml:lang="no">Direktesendte satellittdata prosessert i satellittsveip til L1C.</mmd:title>
+  <mmd:abstract xml:lang="en">Direct Broadcast data received at MET NORWAY Oslo. Processed by standard processing software to geolocated and calibrated values in satellite swath in received instrument resolution.</mmd:abstract>
+  <mmd:abstract xml:lang="no">Direktesendte satellittdata mottatt ved Meteorologisk Institutt Oslo. Prosessert med standard prosesseringssoftware til geolokaliserte og kalibrerte verdier i satellitsveip i mottatt instrument oppløsning.</mmd:abstract>
+  <mmd:metadata_status>Active</mmd:metadata_status>
+  <mmd:dataset_production_status>Complete</mmd:dataset_production_status>
+  <mmd:collection>METNCS</mmd:collection>
+  <mmd:last_metadata_update>
+    <mmd:update>
+      <mmd:datetime>2020-05-07T16:18:23Z</mmd:datetime>
+      <mmd:type>Created</mmd:type>
+    </mmd:update>
+  </mmd:last_metadata_update>
+  <mmd:temporal_extent>
+    <mmd:start_date>2020-04-30T00:04:58.268708Z</mmd:start_date>
+  </mmd:temporal_extent>
+  <mmd:iso_topic_category>climatologyMeteorologyAtmosphere</mmd:iso_topic_category>
+  <mmd:iso_topic_category>environment</mmd:iso_topic_category>
+  <mmd:iso_topic_category>oceans</mmd:iso_topic_category>
+  <mmd:keywords vocabulary="GCMDSK">
+    <mmd:keyword>Earth Science &gt; Atmosphere &gt; Atmospheric radiation</mmd:keyword>
+    <mmd:resource>https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords</mmd:resource>
+    <mmd:separator></mmd:separator>
+  </mmd:keywords>
+  <mmd:keywords vocabulary="GCMDPROV">
+    <mmd:keyword> Government Agencies-non-US &gt; Norway &gt; NO/MET &gt; Norwegian Meteorological Institute</mmd:keyword>
+    <mmd:resource>https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/providers</mmd:resource>
+    <mmd:separator></mmd:separator>
+  </mmd:keywords>
+  <mmd:keywords vocabulary="GCMDLOC">
+    <mmd:keyword>Geographic Region &gt; Northern Hemisphere</mmd:keyword>
+    <mmd:resource>https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/locations</mmd:resource>
+    <mmd:separator></mmd:separator>
+  </mmd:keywords>
+  <mmd:keywords vocabulary="GEMET">
+    <mmd:keyword>Meteorological geographical features</mmd:keyword>
+    <mmd:keyword>Atmospheric conditions</mmd:keyword>
+    <mmd:keyword>Oceanographic geographical features</mmd:keyword>
+    <mmd:resource>http://inspire.ec.europa.eu/theme</mmd:resource>
+    <mmd:separator></mmd:separator>
+  </mmd:keywords>
+  <mmd:keywords vocabulary="NORTHEMES">
+    <mmd:keyword>Weather and climate</mmd:keyword>
+    <mmd:resource>https://register.geonorge.no/subregister/metadata-kodelister/kartverket/nasjonal-temainndeling</mmd:resource>
+    <mmd:separator></mmd:separator>
+  </mmd:keywords>
+  <mmd:geographic_extent>
+    <mmd:rectangle srsName="EPSG:4326">
+      <mmd:north>90.0</mmd:north>
+      <mmd:south>25.0</mmd:south>
+      <mmd:east>180.0</mmd:east>
+      <mmd:west>-180.0</mmd:west>
+    </mmd:rectangle>
+  </mmd:geographic_extent>
+  <mmd:dataset_language>en</mmd:dataset_language>
+  <mmd:operational_status>Operational</mmd:operational_status>
+  <mmd:access_constraint>Open</mmd:access_constraint>
+  <mmd:use_constraint>
+    <mmd:identifier>CC-BY-4.0</mmd:identifier>
+    <mmd:resource>https://spdx.org/licenses/CC-BY-4.0</mmd:resource>
+  </mmd:use_constraint>
+  <mmd:personnel>
+    <mmd:role>Investigator</mmd:role>
+    <mmd:name>Norwegian Meteorological Institute</mmd:name>
+    <mmd:email>[email protected]</mmd:email>
+    <mmd:organisation>Norwegian Meteorological Institute</mmd:organisation>
+  </mmd:personnel>
+  <mmd:personnel>
+    <mmd:role>Metadata author</mmd:role>
+    <mmd:name>DIVISION FOR OBSERVATION QUALITY AND DATA PROCESSING</mmd:name>
+    <mmd:email>[email protected]</mmd:email>
+    <mmd:organisation>Norwegian Meteorological Institute</mmd:organisation>
+  </mmd:personnel>
+  <mmd:personnel>
+    <mmd:role>Technical contact</mmd:role>
+    <mmd:name>Norwegian Meteorological Institute</mmd:name>
+    <mmd:email>[email protected]</mmd:email>
+    <mmd:organisation>Norwegian Meteorological Institute</mmd:organisation>
+  </mmd:personnel>
+  <mmd:data_center>
+    <mmd:data_center_name>
+      <mmd:short_name>MET NORWAY</mmd:short_name>
+      <mmd:long_name>MET NORWAY</mmd:long_name>
+    </mmd:data_center_name>
+    <mmd:data_center_url></mmd:data_center_url>
+  </mmd:data_center>
+  <mmd:platform>
+    <mmd:short_name>Aqua</mmd:short_name>
+    <mmd:long_name>Aqua</mmd:long_name>
+    <mmd:resource>https://www.wmo-sat.info/oscar/satellites/view/aqua</mmd:resource>
+    <mmd:instrument>
+      <mmd:short_name>MODIS</mmd:short_name>
+      <mmd:long_name>MODIS</mmd:long_name>
+      <mmd:resource>https://www.wmo-sat.info/oscar/instruments/view/modis</mmd:resource>
+    </mmd:instrument>
+  </mmd:platform>
+  <mmd:spatial_representation>grid</mmd:spatial_representation>
+  <mmd:dataset_citation>
+    <mmd:author>Norwegian Meteorological Institute</mmd:author>
+    <mmd:publication_date>2020-05-07</mmd:publication_date>
+    <mmd:title>Direct Broadcast data processed in satellite swath to L1C.</mmd:title>
+  </mmd:dataset_citation>
+</mmd:mmd>