Merge pull request #356 from meaningfy-ws/feature/MWB-858

source files exporter + updates/fixes

mapping_workbench/backend/conceptual_mapping_rule/models/entity.py

@@ -251,3 +251,4 @@ class Settings(BaseProjectResourceEntity.Settings):
 
 class ConceptualMappingRuleData(ConceptualMappingRule):
     source_structural_element: Optional[StructuralElement] = None
+    mapping_groups: Optional[List[MappingGroup]] = None

mapping_workbench/backend/conceptual_mapping_rule/services/cm2shacl.py

@@ -4,8 +4,8 @@
 
 from mapping_workbench.backend.conceptual_mapping_rule.adapters.cm2shacl import CMtoSHACL
 from mapping_workbench.backend.conceptual_mapping_rule.models.entity import ConceptualMappingRuleData
-from mapping_workbench.backend.conceptual_mapping_rule.services.data import get_conceptual_mapping_rules_for_project, \
-    get_conceptual_mapping_rules_with_elements_for_project_and_package
+from mapping_workbench.backend.conceptual_mapping_rule.services.data import \
+    get_conceptual_mapping_rules_with_data_for_project_and_package
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage
 from mapping_workbench.backend.ontology.services.namespaces import get_prefixes_definitions
 from mapping_workbench.backend.project.models.entity import Project
@@ -43,7 +43,7 @@ async def generate_shacl_shapes_from_cm_rules(
         ).delete()
 
     cm_rules: List[ConceptualMappingRuleData] = \
-        await get_conceptual_mapping_rules_with_elements_for_project_and_package(project_id, mapping_package)
+        await get_conceptual_mapping_rules_with_data_for_project_and_package(project_id, mapping_package)
 
     shacl_shapes: List[SHACLTestFileResource] = CMtoSHACL(
         prefixes=(prefixes or (await get_prefixes_definitions(project_id))),

mapping_workbench/backend/conceptual_mapping_rule/services/data.py

@@ -36,7 +36,7 @@ async def get_conceptual_mapping_rules_for_project_and_package(
     return items
 
 
-async def get_conceptual_mapping_rules_with_elements_for_project_and_package(
+async def get_conceptual_mapping_rules_with_data_for_project_and_package(
         project_id: PydanticObjectId,
         mapping_package: MappingPackage = None
 ) -> \
@@ -56,6 +56,11 @@ async def get_conceptual_mapping_rules_with_elements_for_project_and_package(
         item_data.source_structural_element = source_structural_element \
             if isinstance(source_structural_element, StructuralElement) else None
 
+        mapping_groups_data = []
+        if item.mapping_groups:
+            mapping_groups_data = [await mapping_group.fetch() for mapping_group in item.mapping_groups]
+        item_data.mapping_groups = mapping_groups_data
+
         items_data.append(item_data)
 
     return items_data

mapping_workbench/backend/core/adapters/exporter.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+
+class ArchiveExporter:
+    @classmethod
+    def write_to_file(cls, file_path: Path, file_content: str):
+        file_path.write_text(file_content, encoding="utf-8")
+
+    @classmethod
+    def create_dir(cls, path: Path):
+        path.mkdir(parents=True, exist_ok=True)

mapping_workbench/backend/core/services/request.py

@@ -8,13 +8,17 @@
 from mapping_workbench.backend.user.models.user import User
 
 
-def request_update_data(entity_data: BaseModel, user: User = None) -> dict:
-    data = entity_data.model_dump(exclude_unset=True)
-
+def request_data_update_refs(data: dict, entity_data: BaseModel) -> dict:
     for field in entity_data.model_fields:
         prop = entity_data.__dict__[field]
         if isinstance(prop, (Link, PydanticObjectId)):
             data[field] = prop
+    return data
+
+
+def request_update_data(entity_data: BaseModel, user: User = None) -> dict:
+    data = entity_data.model_dump(exclude_unset=True)
+    data = request_data_update_refs(data, entity_data)
 
     if user:
         data['updated_by'] = User.link_from_id(user.id)
@@ -25,6 +29,7 @@ def request_update_data(entity_data: BaseModel, user: User = None) -> dict:
 
 def request_create_data(entity_data: BaseModel, user: User = None) -> dict:
     data = entity_data.model_dump()
+    data = request_data_update_refs(data, entity_data)
 
     if user:
         data['created_by'] = User.link_from_id(user.id)

mapping_workbench/backend/mapping_package/entrypoints/api/routes.py

@@ -101,7 +101,7 @@ async def route_create_default_mapping_package(
         epo_version=package_epo_version,
         eform_subtypes=[],
         eforms_sdk_versions=package_eforms_sdk_versions,
-        project=project
+        project=Project.link_from_id(project.id)
     )
     return await create_mapping_package(data, user=user)
 

mapping_workbench/backend/mapping_package/models/entity.py

@@ -181,6 +181,7 @@ async def get_conceptual_mapping_rules_states(self) -> List[ConceptualMappingRul
             Eq(ConceptualMappingRule.refers_to_mapping_package_ids, self.id),
             Eq(ConceptualMappingRule.project, self.project)
         ).to_list()
+
         conceptual_mapping_rule_states = [
             await conceptual_mapping_rule.get_state() for conceptual_mapping_rule in conceptual_mapping_rules
         ]
@@ -205,10 +206,12 @@ async def get_test_data_suites_states(self) -> List[TestDataSuiteState]:
             In(TestDataSuite.id, test_data_suites_ids),
             Eq(TestDataSuite.project, self.project)
         ).to_list()
+
         if test_data_suites:
             for test_data_suite in test_data_suites:
                 test_data_suite_state = await test_data_suite.get_state()
                 test_data_suites_states.append(test_data_suite_state)
+
         return test_data_suites_states
 
     async def get_shacl_test_suites_states(self) -> List[SHACLTestSuiteState]:

mapping_workbench/backend/mapping_rule_registry/services/data.py

@@ -0,0 +1,15 @@
+from typing import List
+
+from beanie import PydanticObjectId
+
+from mapping_workbench.backend.mapping_rule_registry.models.entity import MappingGroup
+from mapping_workbench.backend.project.models.entity import Project
+
+
+async def get_mapping_groups_for_project(project_id: PydanticObjectId) -> \
+        List[MappingGroup]:
+    items: List[MappingGroup] = await MappingGroup.find(
+        MappingGroup.project == Project.link_from_id(project_id)
+    ).to_list()
+
+    return items

mapping_workbench/backend/package_exporter/adapters/cm_exporter.py

@@ -6,7 +6,12 @@
 import pandas as pd
 from pandas import DataFrame
 
+from mapping_workbench.backend.conceptual_mapping_rule.services.data import \
+    get_conceptual_mapping_rules_with_data_for_project_and_package
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage, MappingPackageState
+from mapping_workbench.backend.mapping_rule_registry.services.data import get_mapping_groups_for_project
+from mapping_workbench.backend.project.models.entity import Project
+from mapping_workbench.backend.resource_collection.services.data import get_resource_files_for_project
 
 
 class CMExporterException(Exception):
@@ -16,7 +21,7 @@ class CMExporterException(Exception):
 class CMExporter(ABC):
 
     @abstractmethod
-    def export(self, mapping_package: MappingPackage) -> 'CMExporter':
+    def export_for_package_state(self, mapping_package: MappingPackage) -> 'CMExporter':
         pass
 
     @abstractmethod
@@ -78,7 +83,7 @@ def generate_metadata_table(cls, mapping_package_state: MappingPackageState) ->
         return metadata_table
 
     @classmethod
-    def generate_rules_table(cls, mapping_package_state: MappingPackageState) -> DataFrame:
+    def generate_rules_table(cls, conceptual_mapping_rules) -> DataFrame:
         def prepare_notes(notes: List) -> str:
             return '; '.join(note.comment for note in notes) if notes else ""
 
@@ -99,17 +104,17 @@ def prepare_notes(notes: List) -> str:
             "Feedback Notes (private)"
         ])
         cm_rules = sorted(
-            mapping_package_state.conceptual_mapping_rules, key=lambda x: (x.sort_order is not None, x.sort_order)
+            conceptual_mapping_rules, key=lambda x: (x.sort_order is not None, x.sort_order)
         )
         for idx, cm_rule in enumerate(cm_rules):
-            mapping_groups_ids = map(lambda x: x.name, cm_rule.mapping_groups)
+            mapping_groups_ids = [x.name for x in (cm_rule.mapping_groups or []) if x.name is not None]
             rules_table.loc[idx] = [
                 cm_rule.min_sdk_version,
                 cm_rule.max_sdk_version,
                 cm_rule.source_structural_element.sdk_element_id,
                 cm_rule.source_structural_element.name,
                 cm_rule.source_structural_element.bt_id,
-                ', '.join(mapping_groups_ids),
+                ', '.join(mapping_groups_ids or []),
                 cm_rule.source_structural_element.absolute_xpath,
                 cm_rule.xpath_condition,
                 cm_rule.target_class_path,
@@ -122,15 +127,15 @@ def prepare_notes(notes: List) -> str:
         return rules_table
 
     @classmethod
-    def generate_mapping_groups_table(cls, mapping_package_state: MappingPackageState) -> DataFrame:
+    def generate_mapping_groups_table(cls, mapping_groups) -> DataFrame:
         mapping_groups_table: DataFrame = pd.DataFrame(columns=[
             "Mapping Group ID",
             "Instance Type (ontology Class)",
             "Iterator XPath",
             "Instance URI Template",
             "TripleMap"
         ])
-        for idx, mapping_group in enumerate(mapping_package_state.mapping_groups):
+        for idx, mapping_group in enumerate(mapping_groups):
             mapping_groups_table.loc[idx] = [
                 mapping_group.name,
                 mapping_group.class_uri,
@@ -141,22 +146,42 @@ def generate_mapping_groups_table(cls, mapping_package_state: MappingPackageStat
         return mapping_groups_table
 
     @classmethod
-    def generate_resources_table(cls, mapping_package_state: MappingPackageState) -> DataFrame:
+    def generate_resources_table(cls, resources) -> DataFrame:
         filename_col_name = "File name"
         resources_table: DataFrame = pd.DataFrame(columns=[filename_col_name])
 
-        for idx, resource in enumerate(mapping_package_state.resources):
+        for idx, resource in enumerate(resources):
             resources_table.loc[idx] = [resource.filename]
 
         return resources_table
 
-    def export(self, mapping_package_state: MappingPackageState) -> 'CMExporter':
+    def export_for_package_state(self, mapping_package_state: MappingPackageState) -> 'CMExporter':
         if not isinstance(mapping_package_state, MappingPackageState):
             raise CMExporterException()
         self.cm_tables[self.metadata_table_name] = self.generate_metadata_table(mapping_package_state)
-        self.cm_tables[self.rules_table_name] = self.generate_rules_table(mapping_package_state)
-        self.cm_tables[self.mapping_groups_table_name] = self.generate_mapping_groups_table(mapping_package_state)
-        self.cm_tables[self.resources_table_name] = self.generate_resources_table(mapping_package_state)
+        self.cm_tables[self.rules_table_name] = self.generate_rules_table(
+            mapping_package_state.conceptual_mapping_rules)
+        self.cm_tables[self.mapping_groups_table_name] = self.generate_mapping_groups_table(
+            mapping_package_state.mapping_groups
+        )
+        self.cm_tables[self.resources_table_name] = self.generate_resources_table(
+            mapping_package_state.resources
+        )
+
+        return self
+
+    async def export_for_project(self, project: Project) -> 'CMExporter':
+        if not isinstance(project, Project):
+            raise CMExporterException()
+        self.cm_tables[self.rules_table_name] = self.generate_rules_table(
+            await get_conceptual_mapping_rules_with_data_for_project_and_package(project.id)
+        )
+        self.cm_tables[self.mapping_groups_table_name] = self.generate_mapping_groups_table(
+            await get_mapping_groups_for_project(project.id)
+        )
+        self.cm_tables[self.resources_table_name] = self.generate_resources_table(
+            await get_resource_files_for_project(project_id=project.id)
+        )
 
         return self
 
@@ -165,6 +190,8 @@ def fetch_excel(self) -> bytes:
 
         with pd.ExcelWriter(output_bytes, engine='xlsxwriter') as excel_writer:
             for table_name, table in self.cm_tables.items():
+                if table is None:
+                    continue
                 table.to_excel(excel_writer, sheet_name=table_name, index=False)
 
         output_bytes.seek(0)

mapping_workbench/backend/package_exporter/adapters/eforms/package_state_exporter.py

@@ -6,6 +6,7 @@
 import pandas as pd
 
 from mapping_workbench.backend.core.adapters.archiver import ZipArchiver, ARCHIVE_ZIP_FORMAT
+from mapping_workbench.backend.core.adapters.exporter import ArchiveExporter
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackageState
 from mapping_workbench.backend.package_exporter.adapters.mapping_package_hasher import MappingPackageHasher
 from mapping_workbench.backend.package_exporter.adapters.mapping_package_reporter import MappingPackageReporter
@@ -23,7 +24,7 @@
 from mapping_workbench.backend.user.models.user import User
 
 
-class PackageStateExporter:
+class PackageStateExporter(ArchiveExporter):
     package_state: MappingPackageState
 
     def __init__(self, package_state: MappingPackageState, project: Project, user: User = None):
@@ -68,14 +69,6 @@ async def export(self) -> bytes:
         with open(self.archive_file_path, 'rb') as zip_file:
             return zip_file.read()
 
-    @classmethod
-    def write_to_file(cls, file_path: Path, file_content: str):
-        file_path.write_text(file_content, encoding="utf-8")
-
-    @classmethod
-    def create_dir(cls, path: Path):
-        path.mkdir(parents=True, exist_ok=True)
-
     def create_dirs(self):
         self.create_dir(self.package_path)
         self.create_dir(self.archive_path)

mapping_workbench/backend/package_exporter/services/export_conceptual_mapping.py

@@ -4,15 +4,22 @@
     ConceptualMappingRule
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage, MappingPackageState
 from mapping_workbench.backend.package_exporter.adapters.cm_exporter import EFormsCMExporter
+from mapping_workbench.backend.project.models.entity import Project
 
 
 async def generate_eforms_conceptual_mapping_excel_by_mapping_package_state(
         mapping_package_state: MappingPackageState) -> bytes:
     cm_exporter = EFormsCMExporter()
-    result_excel = cm_exporter.export(mapping_package_state).fetch_excel()
+    result_excel = cm_exporter.export_for_package_state(mapping_package_state).fetch_excel()
     return result_excel
 
 
 async def generate_eforms_conceptual_mapping_excel_by_mapping_package(mapping_package: MappingPackage) -> bytes:
     mapping_package_state: MappingPackageState = await mapping_package.get_state()
     return await generate_eforms_conceptual_mapping_excel_by_mapping_package_state(mapping_package_state)
+
+
+async def generate_conceptual_mapping_excel_by_project(project: Project) -> bytes:
+    cm_exporter = EFormsCMExporter()
+    result_excel = (await cm_exporter.export_for_project(project)).fetch_excel()
+    return result_excel

mapping_workbench/backend/package_importer/adapters/eforms/importer.py

@@ -50,7 +50,7 @@ async def add_mapping_groups_from_mono(self, mono_package: ImportedMappingSuite)
             if not group:
                 group = MappingGroup(name=mono_group.mapping_group_id)
 
-            group.project = self.project
+            group.project = self.project_link
 
             group.class_uri = mono_group.ontology_class
             group.iterator_xpath = mono_group.iterator_xpath
@@ -89,7 +89,7 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
             rule: ConceptualMappingRule = await ConceptualMappingRule.find_one(
                 ConceptualMappingRule.source_structural_element == StructuralElement.link_from_id(
                     source_structural_element.id),
-                ConceptualMappingRule.project == Project.link_from_id(self.project.id),
+                ConceptualMappingRule.project == self.project_link,
                 ConceptualMappingRule.target_class_path == mono_rule.class_path,
                 ConceptualMappingRule.target_property_path == mono_rule.property_path
             )
@@ -103,7 +103,7 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
             )
 
             # rule: ConceptualMappingRule = ConceptualMappingRule()
-            rule.project = self.project
+            rule.project = self.project_link
             if source_structural_element:
                 rule.source_structural_element = source_structural_element
 

mapping_workbench/backend/package_importer/adapters/importer_abc.py

@@ -350,7 +350,7 @@ async def add_mapping_package_from_mono(self, mono_package: ImportedMappingSuite
         else:
             package = MappingPackage(**metadata)
 
-        package.project = self.project
+        package.project = self.project_link
         package.test_data_suites = []
         package.shacl_test_suites = []
         package.sparql_test_suites = []

mapping_workbench/backend/package_importer/adapters/standard/importer.py

@@ -77,7 +77,7 @@ async def add_mapping_rules_from_mono(self, mono_package: ImportedMappingSuite):
                     source_structural_element=StructuralElement.link_from_id(source_structural_element.id)
                 )
 
-            rule.project = self.project
+            rule.project = self.project_link
             if source_structural_element:
                 rule.source_structural_element = source_structural_element
 

mapping_workbench/backend/package_importer/services/import_mapping_suite.py

@@ -38,9 +38,10 @@ async def import_mapping_package(
     )
     package: MappingPackage = await importer.import_from_mono_mapping_suite(monolith_mapping_suite)
 
-    task_response.update_result(TaskResultData(
-        warnings=importer.warnings
-    ))
+    if task_response:
+        task_response.update_result(TaskResultData(
+            warnings=importer.warnings
+        ))
 
     return ImportedMappingSuiteResponse(
         mapping_package=package