Merge pull request #310 from meaningfy-ws/develop

mapping_workbench/backend/fields_registry/services/data.py

@@ -7,13 +7,13 @@
 
 
 async def get_structural_element_by_unique_fields(
-        sdk_element_id, name, bt_id, absolute_xpath, project_id: PydanticObjectId
+        sdk_element_id, bt_id, absolute_xpath, project_id: PydanticObjectId, name: str = None
 ) -> StructuralElement:
     project_link = Project.link_from_id(project_id)
     return await StructuralElement.find_one(
         StructuralElement.project == project_link,
         StructuralElement.sdk_element_id == sdk_element_id,
-        StructuralElement.name == name,
+        #StructuralElement.name == name,
         StructuralElement.bt_id == bt_id,
         StructuralElement.absolute_xpath == absolute_xpath
     )

mapping_workbench/backend/logger/adapters/console_logger.py

@@ -8,5 +8,7 @@ class LoggerConsole(LoggerABC):
     def log(self, log_record: LogRecord):
         if log_record.log_severity == LogSeverity.ERROR:
             print(str(log_record), file=sys.stderr)
+        elif log_record.log_severity == LogSeverity.WARNING:
+            print(str(log_record))
         else:
             print(str(log_record))

mapping_workbench/backend/logger/adapters/sys_logger.py

@@ -17,5 +17,7 @@ class LoggerSys(LoggerABC):
     def log(self, log_record: LogRecord):
         if log_record.log_severity == LogSeverity.ERROR:
             logger.error(str(log_record))
+        elif log_record.log_severity == LogSeverity.WARNING:
+            logger.warning(str(log_record))
         else:
             logger.info(str(log_record))

mapping_workbench/backend/logger/models/logger_record.py

@@ -11,6 +11,7 @@
 
 class LogSeverity(Enum):
     INFO = "INFO"
+    WARNING = "WARNING"
     ERROR = "ERROR"
 
 

mapping_workbench/backend/logger/services/logger_factory.py

@@ -45,3 +45,11 @@ def log_all_info(self, message: str):
         )
         for logger in self.loggers.values():
             logger.log(log_record)
+
+    def log_all_warning(self, message: str):
+        log_record = LogRecord(
+            log_severity=LogSeverity.WARNING,
+            message=message
+        )
+        for logger in self.loggers.values():
+            logger.log(log_record)

mapping_workbench/backend/package_importer/adapters/eforms/importer.py

@@ -58,15 +58,20 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
         for mono_rule in mono_package.conceptual_rules:
             source_structural_element: StructuralElement = await get_structural_element_by_unique_fields(
                 sdk_element_id=mono_rule.eforms_sdk_id,
-                name=mono_rule.field_name,
                 bt_id=mono_rule.bt_id,
                 absolute_xpath=mono_rule.absolute_xpath,
-                project_id=self.project.id
+                project_id=self.project.id,
+                #name=mono_rule.field_name
             )
 
             if not source_structural_element:
                 continue
 
+            if source_structural_element.name != mono_rule.field_name:
+                m = f"Field[{source_structural_element.sdk_element_id}] has Imported Name ({mono_rule.field_name}) <> Current Name ({source_structural_element.name})"
+                mwb_logger.log_all_warning(m)
+                self.warnings.append(m)
+
             # A conceptual mapping rule may have same structural element but different Ontology Fragment
             rule: ConceptualMappingRule = await ConceptualMappingRule.find_one(
                 ConceptualMappingRule.source_structural_element == StructuralElement.link_from_id(source_structural_element.id),

mapping_workbench/backend/package_importer/adapters/importer_abc.py

@@ -1,12 +1,11 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Dict, Tuple
+from typing import Dict, Tuple, List
 
 from mapping_workbench.backend.conceptual_mapping_rule.models.entity import ConceptualMappingRule
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage
 from mapping_workbench.backend.mapping_rule_registry.models.entity import MappingGroup
 from mapping_workbench.backend.package_importer.models.imported_mapping_suite import ImportedMappingSuite
-from mapping_workbench.backend.package_validator.models.test_data_validation import CMRuleSDKElement
 from mapping_workbench.backend.project.models.entity import Project
 from mapping_workbench.backend.resource_collection.models.entity import ResourceFile, ResourceCollection, \
     ResourceFileFormat
@@ -15,7 +14,7 @@
 from mapping_workbench.backend.shacl_test_suite.models.entity import SHACLTestFileResourceFormat, SHACLTestSuite, \
     SHACLTestFileResource
 from mapping_workbench.backend.sparql_test_suite.models.entity import SPARQLTestFileResourceFormat, SPARQLTestSuite, \
-    SPARQLTestFileResource, SPARQLQueryValidationType
+    SPARQLTestFileResource, SPARQLQueryValidationType, SPARQLCMRule
 from mapping_workbench.backend.sparql_test_suite.services.data import SPARQL_CM_ASSERTIONS_SUITE_TITLE, \
     SPARQL_INTEGRATION_TESTS_SUITE_TITLE
 from mapping_workbench.backend.test_data_suite.models.entity import TestDataSuite, TestDataFileResource, \
@@ -27,13 +26,15 @@
 
 class PackageImporterABC(ABC):
     package: MappingPackage
+    warnings: List[str] = []
 
     def __init__(self, project: Project, user: User):
         self.project = project
         self.project_link = Project.link_from_id(self.project.id)
         self.user = user
         self.package = None
 
+
     @abstractmethod
     async def import_from_mono_mapping_suite(self, mono_package: ImportedMappingSuite):
         """
@@ -186,7 +187,7 @@ async def add_sparql_test_suites_from_mono(self, mono_package: ImportedMappingSu
                 )
 
                 metadata = self.extract_metadata_from_sparql_query(resource_content)
-                cm_rule_sdk_element = CMRuleSDKElement(
+                cm_rule_sdk_element = SPARQLCMRule(
                     sdk_element_id=None,
                     sdk_element_title=metadata['title'] if 'title' in metadata else None,
                     sdk_element_xpath=metadata['xpath'] if 'xpath' in metadata else None

mapping_workbench/backend/package_importer/adapters/standard/importer.py

@@ -5,6 +5,7 @@
 from mapping_workbench.backend.conceptual_mapping_rule.services.data import get_conceptual_mapping_rule_by_key
 from mapping_workbench.backend.fields_registry.models.field_registry import StructuralElement
 from mapping_workbench.backend.fields_registry.services.data import get_structural_element_by_unique_fields
+from mapping_workbench.backend.logger.services import mwb_logger
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage
 from mapping_workbench.backend.package_importer.adapters.importer_abc import PackageImporterABC
 from mapping_workbench.backend.package_importer.models.imported_mapping_suite import ImportedMappingSuite
@@ -41,10 +42,10 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
         for mono_rule in mono_package.conceptual_rules:
             source_structural_element: StructuralElement = await get_structural_element_by_unique_fields(
                 sdk_element_id=mono_rule.field_name,
-                name=mono_rule.field_name,
                 bt_id=mono_rule.bt_id,
                 absolute_xpath=mono_rule.absolute_xpath,
-                project_id=self.project.id
+                project_id=self.project.id,
+                #name=mono_rule.field_name
             )
 
             if not source_structural_element:
@@ -79,6 +80,7 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
                 if self.package.id not in rule.refers_to_mapping_package_ids:
                     rule.refers_to_mapping_package_ids.append(self.package.id)
 
+            rule.xpath_condition = mono_rule.xpath_condition
             rule.target_class_path = mono_rule.class_path
             rule.target_property_path = mono_rule.property_path
             rule.sort_order = sort_order

mapping_workbench/backend/package_importer/entrypoints/api/routes.py

@@ -2,7 +2,7 @@
 from fastapi import APIRouter, status, Form, UploadFile, Depends
 
 from mapping_workbench.backend.mapping_package import PackageType
-from mapping_workbench.backend.mapping_package.models.entity import MappingPackage
+from mapping_workbench.backend.package_importer.models.imported_mapping_suite import ImportedMappingSuiteResponse
 from mapping_workbench.backend.package_importer.services import tasks
 from mapping_workbench.backend.package_importer.services.import_mapping_suite import \
     import_mapping_package_from_archive, clear_project_data
@@ -36,7 +36,8 @@ async def route_clear_project_data(
     "/import/archive",
     description=f"Import {NAME_FOR_ONE} Archive",
     name=f"{NAME_FOR_ONE}:import_archive",
-    status_code=status.HTTP_201_CREATED
+    status_code=status.HTTP_201_CREATED,
+    response_model=ImportedMappingSuiteResponse
 )
 async def route_import_package_archive(
         project: PydanticObjectId = Form(...),
@@ -46,11 +47,11 @@ async def route_import_package_archive(
 ):
     if not package_type:
         package_type = PackageType.EFORMS
-    mapping_package: MappingPackage = await import_mapping_package_from_archive(
+    imported_mapping_package: ImportedMappingSuiteResponse = await import_mapping_package_from_archive(
         file.file.read(), await get_project(project), package_type, user
     )
 
-    return mapping_package.model_dump()
+    return imported_mapping_package
 
 
 @router.post(

mapping_workbench/backend/package_importer/models/imported_mapping_suite.py

@@ -1,6 +1,8 @@
 from typing import List, Optional
 from pydantic import BaseModel, Field, ConfigDict
 
+from mapping_workbench.backend.mapping_package.models.entity import MappingPackage
+
 
 class MappingMetadata(BaseModel):
     identifier: str = Field(None, alias="Identifier")
@@ -35,7 +37,7 @@ class EFormsMappingConceptualRule(BaseModel):
     bt_id: Optional[str] = Field(None, alias="BT ID")
     mapping_group_id: Optional[str] = Field(None, alias="Mapping Group ID")
     absolute_xpath: str = Field(..., alias="Absolute XPath")
-    xpath_condition: Optional[str] = Field(..., alias="XPath Condition")
+    xpath_condition: Optional[str] = Field(None, alias="XPath Condition")
     class_path: Optional[str] = Field(None, alias="Class Path")
     property_path: Optional[str] = Field(None, alias="Property Path")
     status: Optional[str] = Field(None, alias="Status")
@@ -55,7 +57,7 @@ class StandardMappingConceptualRule(BaseModel):
     bt_name: Optional[str] = Field(None, alias="eForm BT Name (Provisional/Indicative) (O)")
     absolute_xpath: Optional[str] = Field(..., alias="Field XPath (M)")
     relative_xpath: Optional[str] = Field(..., alias="Field XPath (M)")
-    xpath_condition: Optional[str] = Field(..., alias="Field XPath condition (M)")
+    xpath_condition: Optional[str] = Field(None, alias="Field XPath condition (M)")
     class_path: Optional[str] = Field(None, alias="Class path (M)")
     property_path: Optional[str] = Field(None, alias="Property path (M)")
     reference_to_integration_tests: Optional[str] = Field(None, alias="Reference to Integration Tests (O)")
@@ -108,3 +110,7 @@ class ImportedStandardMappingSuite(ImportedMappingSuite):
     metadata: StandardMappingMetadata
     conceptual_rules: List[StandardMappingConceptualRule] = []
 
+
+class ImportedMappingSuiteResponse(BaseModel):
+    mapping_package: MappingPackage
+    warnings: List[str] = []

mapping_workbench/backend/package_importer/services/import_mapping_suite.py

@@ -3,25 +3,27 @@
 import tempfile
 import zipfile
 
-from mapping_workbench.backend.core.services.exceptions import InvalidResourceException, InvalidPackageTypeException
+from mapping_workbench.backend.core.services.exceptions import InvalidResourceException
 from mapping_workbench.backend.mapping_package import PackageType
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage
 from mapping_workbench.backend.package_importer.adapters.eforms.importer import EFormsPackageImporter
 from mapping_workbench.backend.package_importer.adapters.importer_abc import PackageImporterABC
 from mapping_workbench.backend.package_importer.adapters.importer_factory import PackageImporterFactory
 from mapping_workbench.backend.package_importer.adapters.standard.importer import StandardPackageImporter
+from mapping_workbench.backend.package_importer.models.imported_mapping_suite import ImportedMappingSuiteResponse
 from mapping_workbench.backend.package_importer.services.import_mono_eforms_mapping_suite import \
     import_eforms_mapping_suite_from_file_system
 from mapping_workbench.backend.package_importer.services.import_mono_standard_mapping_suite import \
     import_standard_mapping_suite_from_file_system
 from mapping_workbench.backend.project.models.entity import Project
+from mapping_workbench.backend.tasks.models.task_response import TaskResponse
 from mapping_workbench.backend.user.models.user import User
 
 
 async def import_mapping_package(
         mapping_package_dir_path: pathlib.Path, project: Project,
         package_type: PackageType, user: User = None
-) -> MappingPackage:
+) -> ImportedMappingSuiteResponse:
     if package_type == PackageType.STANDARD:
         monolith_mapping_suite = import_standard_mapping_suite_from_file_system(mapping_package_dir_path)
     else:  # package_type == PackageType.EFORMS:
@@ -31,12 +33,17 @@ async def import_mapping_package(
         package_type=package_type, project=project, user=user
     )
     package: MappingPackage = await importer.import_from_mono_mapping_suite(monolith_mapping_suite)
-    return package
+
+    return ImportedMappingSuiteResponse(
+        mapping_package=package,
+        warnings=importer.warnings
+    )
 
 
 async def import_mapping_package_from_archive(
-        file_content: bytes, project: Project, package_type: PackageType, user: User = None
-) -> MappingPackage:
+        file_content: bytes, project: Project, package_type: PackageType, user: User = None,
+        task_response: TaskResponse = None
+) -> ImportedMappingSuiteResponse:
     zf = zipfile.ZipFile(io.BytesIO(file_content))
     tempdir = tempfile.TemporaryDirectory()
     tempdir_name = tempdir.name
@@ -48,7 +55,10 @@ async def import_mapping_package_from_archive(
     except AssertionError as error:
         raise InvalidResourceException(str(error))
 
-    return await import_mapping_package(dir_contents[0], project, package_type, user)
+    result = await import_mapping_package(dir_contents[0], project, package_type, user)
+    if task_response:
+        task_response.data = result
+    return result
 
 
 async def clear_project_data(project: Project, package_type: PackageType = None):

mapping_workbench/backend/package_importer/services/tasks.py

@@ -1,8 +1,9 @@
+from mapping_workbench.backend.mapping_package import PackageType
 from mapping_workbench.backend.package_importer.services.import_mapping_suite import \
     import_mapping_package_from_archive
-from mapping_workbench.backend.mapping_package import PackageType
 from mapping_workbench.backend.project.models.entity import Project
 from mapping_workbench.backend.task_manager.services.task_wrapper import run_task
+from mapping_workbench.backend.tasks.models.task_response import TaskResponse
 from mapping_workbench.backend.user.models.user import User
 
 
@@ -11,8 +12,11 @@ def task_import_mapping_package(
         project: Project,
         package_type: PackageType,
         user: User = None
-):
+) -> TaskResponse:
+    task_response: TaskResponse = TaskResponse()
+
     run_task(
         import_mapping_package_from_archive,
-        file_content, project, package_type, user
+        file_content, project, package_type, user, task_response
     )
+    return task_response

mapping_workbench/backend/package_validator/adapters/sparql_validator.py

@@ -75,28 +75,48 @@ def process_sparql_result(self, sparql_query_result: SPARQLQueryResult):
             xpath_validation_results = xpath_validation.results
             sparql_query_xpath = sparql_query_result.query.cm_rule.sdk_element_xpath.strip() \
                 if sparql_query_result.query.cm_rule else None
+            sparql_xpath_condition = sparql_query_result.query.cm_rule.xpath_condition.xpath_condition.strip() \
+                if sparql_query_result.query.cm_rule else None
             validation_xpaths = set()
+            validation_xpath_conditions = set()
             for xpath_assertion in xpath_validation_results:
                 if xpath_assertion.is_covered:
                     validation_xpaths.add(xpath_assertion.sdk_element_xpath.strip())
+                if xpath_assertion.xpath_conditions:
+                    validation_xpath_conditions |= set([
+                        (xpath_condition.xpath_condition or '').strip()
+                        for xpath_condition in xpath_assertion.xpath_conditions
+                        if xpath_condition.meets_xpath_condition
+                    ])
             sparql_query_result.fields_covered = (not sparql_query_xpath or (
                     sparql_query_xpath in validation_xpaths
             ))
+            sparql_query_result.meets_xpath_condition = (not sparql_xpath_condition or (
+                    sparql_xpath_condition in validation_xpath_conditions
+            ))
 
             # Refined result
-            result = self.refined_result(ask_answer, sparql_query_result, result)
+            result = self.refined_result(ask_answer, sparql_query_result)
 
         sparql_query_result.result = result
 
     @classmethod
-    def refined_result(cls, ask_answer, sparql_query_result, result: SPARQLQueryRefinedResultType) \
+    def refined_result(cls, ask_answer, sparql_query_result: SPARQLQueryResult) \
             -> SPARQLQueryRefinedResultType:
-        if ask_answer and sparql_query_result.fields_covered:
-            result = SPARQLQueryRefinedResultType.VALID.value
-        elif not ask_answer and not sparql_query_result.fields_covered:
-            result = SPARQLQueryRefinedResultType.UNVERIFIABLE.value
-        elif ask_answer and not sparql_query_result.fields_covered:
-            result = SPARQLQueryRefinedResultType.WARNING.value
-        elif not ask_answer and sparql_query_result.fields_covered:
-            result = SPARQLQueryRefinedResultType.INVALID.value
-        return result
+
+        if sparql_query_result.fields_covered:
+            if ask_answer:
+                if sparql_query_result.meets_xpath_condition:
+                    return SPARQLQueryRefinedResultType.VALID.value
+                else:
+                    return SPARQLQueryRefinedResultType.WARNING.value
+            else:
+                if sparql_query_result.meets_xpath_condition:
+                    return SPARQLQueryRefinedResultType.INVALID.value
+                else:
+                    return SPARQLQueryRefinedResultType.VALID.value
+        else:
+            if ask_answer:
+                return SPARQLQueryRefinedResultType.WARNING.value
+            else:
+                return SPARQLQueryRefinedResultType.UNVERIFIABLE.value