Merge pull request #527 from OP-TED/feature/TED4-139

ted_sws/core/model/transform.py

@@ -9,7 +9,7 @@
 import abc
 from datetime import datetime
 from enum import Enum
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from ted_sws.core.model import PropertyBaseModel
 
@@ -36,11 +36,33 @@ class NoticeFileResource(FileResource):
     notice_id: str
 
 
-class MetadataConstraints(MappingSuiteComponent):
+
+class MetadataConstraintsStandardForm(MappingSuiteComponent):
+    """
+    Metadata constraints structure for Standard forms
+    """
+    eforms_subtype: List[str]
+    start_date: Optional[List[str]]
+    end_date: Optional[List[str]]
+    min_xsd_version: List[str]
+    max_xsd_version: Optional[List[str]]
+
+
+class MetadataConstraintsEform(MappingSuiteComponent):
+    """
+    Metadata constraints structure for eForms
     """
+    eforms_subtype: List[str]
+    start_date: Optional[List[str]]
+    end_date: Optional[List[str]]
+    eforms_sdk_versions: List[str]
+
 
+class MetadataConstraints(MappingSuiteComponent):
+    """
+         Metadata constraints
     """
-    constraints: dict
+    constraints: Union[MetadataConstraintsStandardForm, MetadataConstraintsEform]
 
 
 class TransformationRuleSet(MappingSuiteComponent):

ted_sws/data_manager/adapters/mapping_suite_repository.py

@@ -9,7 +9,8 @@
 
 from ted_sws import config
 from ted_sws.core.model.transform import MappingSuite, FileResource, TransformationRuleSet, SHACLTestSuite, \
-    SPARQLTestSuite, MetadataConstraints, TransformationTestData, ConceptualMapping
+    SPARQLTestSuite, MetadataConstraints, TransformationTestData, ConceptualMapping, MappingSuiteType, \
+    MetadataConstraintsStandardForm, MetadataConstraintsEform
 from ted_sws.data_manager.adapters import inject_date_string_fields, remove_date_string_fields
 from ted_sws.data_manager.adapters.repository_abc import MappingSuiteRepositoryABC
 from ted_sws.mapping_suite_processor.adapters.conceptual_mapping_reader import ConceptualMappingReader
@@ -25,8 +26,17 @@
 MS_CONCEPTUAL_MAPPING_FILE_NAME = "conceptual_mappings.xlsx"
 MS_OUTPUT_FOLDER_NAME = "output"
 MS_TEST_SUITE_REPORT = "test_suite_report"
-MS_CREATED_AT = "created_at"
+MS_CREATED_AT_KEY = "created_at"
 MONGODB_COLLECTION_ID = "_id"
+MS_METADATA_IDENTIFIER_KEY = 'identifier'
+MS_STANDARD_METADATA_VERSION_KEY = 'version'
+MS_EFORMS_METADATA_VERSION_KEY = 'mapping_version'
+MS_METADATA_CONSTRAINTS_KEY = 'metadata_constraints'
+MS_CONSTRAINTS_KEY = 'constraints'
+MS_TITLE_KEY = 'title'
+MS_HASH_DIGEST_KEY = 'mapping_suite_hash_digest'
+MS_MAPPING_TYPE_KEY = 'mapping_type'
+MS_ONTOLOGY_VERSION_KEY = 'ontology_version'
 
 
 class MappingSuiteRepositoryMongoDB(MappingSuiteRepositoryABC):
@@ -55,8 +65,8 @@ def _create_dict_from_mapping_suite(self, mapping_suite: MappingSuite) -> dict:
         """
         mapping_suite_dict = mapping_suite.dict()
         mapping_suite_dict[MONGODB_COLLECTION_ID] = mapping_suite.get_mongodb_id()
-        mapping_suite_dict[MS_CREATED_AT] = datetime.fromisoformat(mapping_suite_dict[MS_CREATED_AT])
-        inject_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT)
+        mapping_suite_dict[MS_CREATED_AT_KEY] = datetime.fromisoformat(mapping_suite_dict[MS_CREATED_AT_KEY])
+        inject_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT_KEY)
         return mapping_suite_dict
 
     def _create_mapping_suite_from_dict(self, mapping_suite_dict: dict) -> Optional[MappingSuite]:
@@ -67,8 +77,8 @@ def _create_mapping_suite_from_dict(self, mapping_suite_dict: dict) -> Optional[
         """
         if mapping_suite_dict:
             mapping_suite_dict.pop(MONGODB_COLLECTION_ID, None)
-            mapping_suite_dict[MS_CREATED_AT] = mapping_suite_dict[MS_CREATED_AT].isoformat()
-            remove_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT)
+            mapping_suite_dict[MS_CREATED_AT_KEY] = mapping_suite_dict[MS_CREATED_AT_KEY].isoformat()
+            remove_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT_KEY)
             return MappingSuite(**mapping_suite_dict)
         return None
 
@@ -134,7 +144,6 @@ def _read_package_metadata(self, package_path: pathlib.Path) -> dict:
         package_metadata_path = package_path / MS_METADATA_FILE_NAME
         package_metadata_content = package_metadata_path.read_text(encoding="utf-8")
         package_metadata = json.loads(package_metadata_content)
-        package_metadata['metadata_constraints'] = MetadataConstraints(**package_metadata['metadata_constraints'])
         return package_metadata
 
     def _read_transformation_rule_set(self, package_path: pathlib.Path) -> TransformationRuleSet:
@@ -343,16 +352,30 @@ def _read_mapping_suite_package(self, mapping_suite_identifier: str) -> Optional
         package_path = self.repository_path / mapping_suite_identifier
         if package_path.is_dir():
             package_metadata = self._read_package_metadata(package_path)
-            package_metadata["version"] = package_metadata[
-                "version"] if "version" in package_metadata else package_metadata["mapping_version"]
-            package_metadata["identifier"] = package_metadata[
-                "identifier"] if "identifier" in package_metadata else mapping_suite_identifier
-            package_metadata["transformation_rule_set"] = self._read_transformation_rule_set(package_path)
-            package_metadata["shacl_test_suites"] = self._read_shacl_test_suites(package_path)
-            package_metadata["sparql_test_suites"] = self._read_sparql_test_suites(package_path)
-            package_metadata["transformation_test_data"] = self._read_test_data_package(package_path)
-            package_metadata["conceptual_mapping"] = self._read_conceptual_mapping(package_path)
-            return MappingSuite(**package_metadata)
+            if MS_MAPPING_TYPE_KEY in package_metadata and package_metadata[
+                MS_MAPPING_TYPE_KEY] == MappingSuiteType.ELECTRONIC_FORMS:
+                package_metadata[MS_METADATA_CONSTRAINTS_KEY] = MetadataConstraints(constraints=MetadataConstraintsEform(
+                    **package_metadata[MS_METADATA_CONSTRAINTS_KEY][MS_CONSTRAINTS_KEY]))
+            else:
+                package_metadata[MS_METADATA_CONSTRAINTS_KEY] = MetadataConstraints(constraints=MetadataConstraintsStandardForm(
+                    **package_metadata[MS_METADATA_CONSTRAINTS_KEY][MS_CONSTRAINTS_KEY]))
+            mapping_suite = MappingSuite(metadata_constraints=package_metadata[MS_METADATA_CONSTRAINTS_KEY],
+                                         created_at=package_metadata[MS_CREATED_AT_KEY],
+                                         title=package_metadata[MS_TITLE_KEY],
+                                         ontology_version=package_metadata[MS_ONTOLOGY_VERSION_KEY],
+                                         mapping_suite_hash_digest=package_metadata[MS_HASH_DIGEST_KEY],
+                                         mapping_type=package_metadata[MS_MAPPING_TYPE_KEY] if MS_MAPPING_TYPE_KEY in package_metadata else MappingSuiteType.STANDARD_FORMS,
+                                         version=package_metadata[
+                                             MS_STANDARD_METADATA_VERSION_KEY] if MS_STANDARD_METADATA_VERSION_KEY in package_metadata else \
+                                             package_metadata[MS_EFORMS_METADATA_VERSION_KEY],
+                                         identifier=package_metadata[
+                                             MS_METADATA_IDENTIFIER_KEY] if MS_METADATA_IDENTIFIER_KEY in package_metadata else mapping_suite_identifier,
+                                         transformation_rule_set=self._read_transformation_rule_set(package_path),
+                                         shacl_test_suites=self._read_shacl_test_suites(package_path),
+                                         sparql_test_suites=self._read_sparql_test_suites(package_path),
+                                         transformation_test_data=self._read_test_data_package(package_path),
+                                         conceptual_mapping=self._read_conceptual_mapping(package_path) if not MS_MAPPING_TYPE_KEY in package_metadata else []) #TODO remove conceptual_mapping value assignment when conceptual mapping reader is removed
+            return mapping_suite
         return None
 
     @classmethod

ted_sws/mapping_suite_processor/services/mapping_suite_validation_service.py

@@ -2,13 +2,11 @@
 import pathlib
 from typing import Optional
 
-from ted_sws.data_manager.adapters.mapping_suite_repository import MS_METADATA_FILE_NAME
+from ted_sws.data_manager.adapters.mapping_suite_repository import MS_METADATA_FILE_NAME, \
+    MS_STANDARD_METADATA_VERSION_KEY, MS_METADATA_IDENTIFIER_KEY, \
+    MS_EFORMS_METADATA_VERSION_KEY
 from ted_sws.mapping_suite_processor.adapters.mapping_suite_structure_checker import MappingSuiteStructureValidator
 
-MAPPING_SUITE_METADATA_IDENTIFIER_KEY = 'identifier'
-STANDARD_MAPPING_SUITE_METADATA_VERSION_KEY = 'version'
-EFORMS_MAPPING_SUITE_METADATA_VERSION_KEY = 'mapping_version'
-
 
 def get_mapping_suite_id_from_file_system(mapping_suite_path: pathlib.Path) -> Optional[str]:
     """
@@ -20,10 +18,10 @@ def get_mapping_suite_id_from_file_system(mapping_suite_path: pathlib.Path) -> O
 
     if mapping_suite_metadata_path.exists() and mapping_suite_metadata_path.is_file():
         mapping_suite_metadata = json.loads(mapping_suite_metadata_path.read_text(encoding="utf-8"))
-        identifier_value = mapping_suite_metadata[MAPPING_SUITE_METADATA_IDENTIFIER_KEY]
+        identifier_value = mapping_suite_metadata[MS_METADATA_IDENTIFIER_KEY]
         version_value = mapping_suite_metadata[
-            STANDARD_MAPPING_SUITE_METADATA_VERSION_KEY] if STANDARD_MAPPING_SUITE_METADATA_VERSION_KEY in mapping_suite_metadata else \
-        mapping_suite_metadata[EFORMS_MAPPING_SUITE_METADATA_VERSION_KEY]
+            MS_STANDARD_METADATA_VERSION_KEY] if MS_STANDARD_METADATA_VERSION_KEY in mapping_suite_metadata else \
+        mapping_suite_metadata[MS_EFORMS_METADATA_VERSION_KEY]
         return f"{identifier_value}_v{version_value}"
     return None
 

ted_sws/notice_metadata_processor/services/notice_eligibility.py

@@ -43,12 +43,12 @@ def is_version_in_range(notice_metadata: NormalisedMetadata, mapping_suite: Mapp
         # we need to extract only the version i.e 1.7 or 1.7.9
         eforms_sdk_version = notice_xsd_version.rsplit('-', 1)[1]
         constraint_version_range = [format_version_with_zero_patch(version) for version in
-                                    constraints[EFORMS_SDK_VERSIONS_KEY]]
+                                    constraints.eforms_sdk_versions]
         return format_version_with_zero_patch(eforms_sdk_version) in constraint_version_range
     elif mapping_suite.mapping_type == MappingSuiteType.STANDARD_FORMS and notice_metadata.notice_source == NoticeSource.STANDARD_FORM:
         notice_xsd_version = notice_metadata.xsd_version
-        constraint_min_xsd_version = constraints[MIN_XSD_VERSION_KEY][0]
-        constraint_max_xsd_version = constraints[MAX_XSD_VERSION_KEY][0]
+        constraint_min_xsd_version = constraints.min_xsd_version[0]
+        constraint_max_xsd_version = constraints.max_xsd_version[0]
         return constraint_min_xsd_version <= notice_xsd_version <= constraint_max_xsd_version
 
     return False
@@ -63,17 +63,15 @@ def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetada
     """
 
     constraints = mapping_suite.metadata_constraints.constraints
-
     eform_subtype = notice_metadata.eforms_subtype
     notice_publication_date = datetime.datetime.fromisoformat(notice_metadata.publication_date)
     in_version_range = is_version_in_range(notice_metadata=notice_metadata, mapping_suite=mapping_suite)
-
     in_date_range = is_date_in_range(publication_date=notice_publication_date,
-                                     constraint_start_date_value=constraints[START_DATE_KEY],
-                                     constraint_end_date_value=constraints[END_DATE_KEY])
+                                     constraint_start_date_value=constraints.start_date,
+                                     constraint_end_date_value=constraints.end_date)
     eform_subtype_constraint_values = [str(eforms_subtype_value) for eforms_subtype_value in
-                                       constraints[E_FORMS_SUBTYPE_KEY]]
-    covered_eform_type = eform_subtype in eform_subtype_constraint_values
+                                       constraints.eforms_subtype]
+    covered_eform_type = str(eform_subtype) in eform_subtype_constraint_values
 
     return in_date_range and in_version_range and covered_eform_type
 

tests/e2e/data_manager/conftest.py

@@ -3,7 +3,7 @@
 from ted_sws.core.model.manifestation import RDFManifestation
 from ted_sws.core.model.notice import NoticeStatus
 from ted_sws.core.model.transform import FileResource, SPARQLTestSuite, MetadataConstraints, TransformationRuleSet, \
-    SHACLTestSuite, TransformationTestData, MappingSuite
+    SHACLTestSuite, TransformationTestData, MappingSuite, MetadataConstraintsStandardForm
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepositoryInFileSystem
 from tests import TEST_DATA_PATH
 
@@ -190,7 +190,7 @@ def sparql_test_suite_with_invalid_query(invalid_sparql_file):
 
 @pytest.fixture
 def dummy_mapping_suite(sparql_test_suite, shacl_test_suite):
-    metadata_constrains = MetadataConstraints(constraints=dict())
+    metadata_constrains = MetadataConstraints(constraints=MetadataConstraintsStandardForm(eforms_subtype=[29],min_xsd_version=["R2.0.9.S01.E01"]))
     file_name = "fake_title.txt"
     empty_file_resource = FileResource(file_name=file_name, file_content="no content here", original_name=file_name)
     transformation_rule_set = TransformationRuleSet(resources=[empty_file_resource],

tests/test_data/notice_transformer/mapping_suite_processor_repository/test_package/metadata.json

@@ -3,6 +3,7 @@
   "created_at": "2022-03-20T19:46:14.069518",
   "identifier": "test_package",
   "version": "0.1",
+    "mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
   "ontology_version": "3.0.0.alpha",
   "metadata_constraints": {
     "constraints": {

tests/test_data/notice_transformer/test_repository/test_package/metadata.json

@@ -2,6 +2,7 @@
   "title": "no_title",
   "created_at": "2022-03-20T19:46:14.069518",
   "identifier": "test_package",
+  "mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
   "version": "1.0.1",
   "ontology_version": "3.0.0.alpha",
   "metadata_constraints": {

tests/test_data/notice_transformer/test_repository/test_package2/metadata.json

@@ -3,6 +3,7 @@
   "created_at": "2022-03-20T19:46:14.069518",
   "identifier": "test_package2",
   "version": "2.1.6",
+    "mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
   "ontology_version": "3.0.0.alpha",
   "metadata_constraints": {
     "constraints": {

tests/test_data/notice_transformer/test_repository/test_package3/metadata.json

@@ -4,6 +4,7 @@
   "identifier": "test_package3",
   "version": "3.1.6",
   "ontology_version": "3.0.0.alpha",
+    "mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
   "metadata_constraints": {
     "constraints": {
       "eforms_subtype": [

tests/test_data/notice_transformer/test_repository/test_package4/metadata.json

@@ -1,6 +1,7 @@
 {
   "identifier": "package_EF16",
   "title": "Package EF16 v1.2",
+  "created_at": "2022-03-20T19:46:14.069518",
   "description": "This is the conceptual mapping for bla bla bla",
   "mapping_version": "3.0.0-alpha.1",
   "ontology_version": "4.0.0",

tests/test_data/notice_validator/test_repository/test_package_F03/metadata.json

@@ -1 +1,20 @@
-{"title": "F03", "identifier": "test_package_F03", "created_at": "2022-04-29T16:41:52.018830", "version": "0.0.1", "ontology_version": "3.0.0.alpha", "xsd_version": "R2.0.9.S05.E01", "metadata_constraints": {"constraints": {"form_number": ["F03"], "legal_basis": ["*"], "year": ["*"], "notice_type": [], "form_type": []}}}
+{
+  "title": "F03",
+  "identifier": "test_package_F03",
+  "created_at": "2022-04-29T16:41:52.018830",
+  "version": "0.0.1",
+  "ontology_version": "3.0.0.alpha",
+  "mapping_suite_hash_digest": "1740b31999a492bbe80122f766f8d4c1dd7a3f720b95337a88a10cf673ceb223",
+  "metadata_constraints": {
+    "constraints": {
+      "eforms_subtype": [
+        4,12,29,15.1
+      ],
+      "start_date": ["2020-03-03"]
+      ,
+      "end_date": ["2020-09-10"],
+      "min_xsd_version": ["R2.0.9.S01.E01"],
+      "max_xsd_version": ["R2.0.9.S05.E01"]
+    }
+  }
+}

tests/unit/data_manager/conftest.py

@@ -2,7 +2,7 @@
 import pytest
 from ted_sws.core.model.supra_notice import DailySupraNotice
 from ted_sws.core.model.transform import MetadataConstraints, FileResource, TransformationRuleSet, SHACLTestSuite, \
-    SPARQLTestSuite, MappingSuite, TransformationTestData
+    SPARQLTestSuite, MappingSuite, TransformationTestData, MetadataConstraintsStandardForm
 from tests import TEST_DATA_PATH
 
 
@@ -27,7 +27,7 @@ def epo_mapping_suite_package_name():
 
 @pytest.fixture
 def fake_mapping_suite():
-    metadata_constrains = MetadataConstraints(constraints=dict())
+    metadata_constrains = MetadataConstraints(constraints=MetadataConstraintsStandardForm(eforms_subtype=[29],min_xsd_version=["R2.0.9.S01.E01"]))
     file_name = "fake_file.txt"
     empty_file_resource = FileResource(file_name=file_name, file_content="fake content", original_name=file_name)
     transformation_rule_set = TransformationRuleSet(resources=[empty_file_resource],

tests/unit/data_manager/test_mapping_suite_repository.py

@@ -48,10 +48,10 @@ def test_epo_mapping_suite_repository_in_file_system(file_system_repository_with
     assert result_mapping_suite.identifier == "package_EF16"
     assert result_mapping_suite.title == "Package EF16 v1.2"
     assert result_mapping_suite.mapping_type == "eforms"
-    assert result_mapping_suite.metadata_constraints.constraints
+    assert result_mapping_suite.metadata_constraints
     constraints = result_mapping_suite.metadata_constraints.constraints
-    assert constraints["eforms_subtype"]
-    assert constraints["eforms_sdk_versions"]
+    assert constraints.eforms_subtype
+    assert constraints.eforms_sdk_versions
 
 
 def test_mapping_suite_repository_in_file_system(file_system_repository_path, fake_mapping_suite):

tests/unit/mapping_suite_processor/conftest.py

@@ -3,7 +3,7 @@
 import pytest
 
 from ted_sws.core.model.transform import MetadataConstraints, FileResource, TransformationRuleSet, SHACLTestSuite, \
-    SPARQLTestSuite, MappingSuite, TransformationTestData
+    SPARQLTestSuite, MappingSuite, TransformationTestData, MetadataConstraintsStandardForm
 from tests import TEST_DATA_PATH
 
 
@@ -57,7 +57,7 @@ def fake_mapping_suite():
             FileResource(file_name=rml_mapping_rule_file.name, file_content=rml_mapping_rule_file.open().read(),
                          original_name=rml_mapping_rule_file.name))
 
-    metadata_constrains = MetadataConstraints(constraints=dict())
+    metadata_constrains = MetadataConstraints(constraints=MetadataConstraintsStandardForm(eforms_subtype=[29],min_xsd_version=["R2.0.9.S01.E01"]))
     file_name = "fake_file.txt"
     empty_file_resource = FileResource(file_name=file_name, file_content="fake content", original_name=file_name)
 

tests/unit/notice_validator/conftest.py

@@ -6,7 +6,7 @@
     XPATHCoverageValidationResult, SHACLTestSuiteValidationReport, SPARQLTestSuiteValidationReport
 from ted_sws.core.model.notice import NoticeStatus, Notice
 from ted_sws.core.model.transform import FileResource, SPARQLTestSuite, MetadataConstraints, TransformationRuleSet, \
-    SHACLTestSuite, TransformationTestData, MappingSuite
+    SHACLTestSuite, TransformationTestData, MappingSuite, MetadataConstraintsStandardForm
 from tests import TEST_DATA_PATH
 
 
@@ -257,7 +257,7 @@ def sparql_test_suite_with_select_query(sparql_file_select):
 
 @pytest.fixture
 def dummy_mapping_suite(sparql_test_suite, shacl_test_suite):
-    metadata_constrains = MetadataConstraints(constraints=dict())
+    metadata_constrains = MetadataConstraints(constraints=MetadataConstraintsStandardForm(eforms_subtype=[29],min_xsd_version=["R2.0.9.S01.E01"]))
     file_name = "fake_title.txt"
     empty_file_resource = FileResource(file_name=file_name, file_content="no content here", original_name=file_name)
     transformation_rule_set = TransformationRuleSet(resources=[empty_file_resource],