Feature/mwb 873

.github/workflows/e2e-tests.yml

@@ -54,7 +54,7 @@ jobs:
 
       - name: Run PR e2e Tests
         run: |
-          ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} 'source ~/.bashrc && cd ${{ secrets.STAGING_WORK_DIR }} && python3 -m venv venv && source venv/bin/activate && make dotenv-file-frontend && make install && make test-e2e && exit'
+          ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} 'source ~/.bashrc && cd ${{ secrets.STAGING_WORK_DIR }} && python3 -m venv venv && source venv/bin/activate && make dotenv-file-frontend && make install && make test-e2e && exit'
 
       - uses: actions/checkout@v2
         with:
@@ -75,7 +75,7 @@ jobs:
       - name: Revert to develop branch
         if: always()
         run: |
-          ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "source ~/.bashrc && cd ${{ secrets.STAGING_WORK_DIR }} && git checkout develop && git pull && make staging-dotenv-file && make stop-frontend && make build-frontend && make stop-backend && make build-backend && exit"
+          ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "source ~/.bashrc && cd ${{ secrets.STAGING_WORK_DIR }} && git checkout develop && git pull && make staging-dotenv-file && make stop-frontend && make build-frontend && make stop-backend && make build-backend && exit"
 
       - name: cleanup
         if: always()

mapping_workbench/backend/core/services/project_initilisers.py

@@ -7,6 +7,7 @@
 from mapping_workbench.backend.config import settings
 from mapping_workbench.backend.fields_registry.models.field_registry import StructuralElementsVersionedView, \
     StructuralElement
+from mapping_workbench.backend.fields_registry.models.pool import PoolSDKField, PoolSDKFieldsVersionedView
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage, MappingPackageStateGate
 from mapping_workbench.backend.mapping_rule_registry.models.entity import MappingRuleRegistry, MappingGroup
 from mapping_workbench.backend.ontology.models.namespace import Namespace, NamespaceCustom
@@ -72,5 +73,7 @@ async def init_project_models(mongodb_database: AsyncIOMotorDatabase):
             StructuralElementsVersionedView,
             XSDFileResource,
             ConceptualMappingGroupBeanie,
+            PoolSDKField,
+            PoolSDKFieldsVersionedView
         ],
     )

mapping_workbench/backend/fields_registry/entrypoints/api/routes.py

@@ -31,7 +31,7 @@
 NAME_FOR_MANY = "fields_registries"
 NAME_FOR_ONE = "fields_registry"
 
-TASK_IMPORT_EFORMS_FROM_GITHUB_NAME = f"{NAME_FOR_ONE}:tasks:import_eforms_from_github"
+TASK_IMPORT_EFORMS_XSD_NAME = f"{NAME_FOR_ONE}:tasks:import_eforms_xsd"
 
 router = APIRouter(
     prefix=ROUTE_PREFIX,
@@ -188,20 +188,20 @@ async def route_search_structural_elements_versioned_view_by_eforms_version(
 
 
 @router.post(
-    "/tasks/import_eforms_from_github",
-    description=f"Task Import eforms from github",
-    name=TASK_IMPORT_EFORMS_FROM_GITHUB_NAME,
+    "/tasks/import_eforms_xsd",
+    description=f"Task Import eForms XSD",
+    name=TASK_IMPORT_EFORMS_XSD_NAME,
     status_code=status.HTTP_201_CREATED
 )
-async def route_task_import_eforms_from_github(
+async def route_task_import_eforms_xsd(
         github_repository_url: str = Form(...),
         branch_or_tag_name: str = Form(...),
         project_id: PydanticObjectId = Form(...),
         user: User = Depends(current_active_user)
 ):
     return add_task(
-        tasks.task_import_eforms_from_github,
-        f"Importing eForms from {github_repository_url}, branch: {branch_or_tag_name}",
+        tasks.task_import_eforms_xsd,
+        f"Importing eForms XSD versions: {branch_or_tag_name}",
         None,
         user.email,
         False,

mapping_workbench/backend/fields_registry/models/eforms_fields_structure.py

@@ -20,23 +20,38 @@ class EFormsField(BaseModel):
     legal_type: Optional[str] = Field(default=None, alias='legalType')
     repeatable: EFormsFieldsRepeatableAttribute
 
-    def generate_hash_id(self, project_id: str = None) -> str:
-        fields_to_hash = [project_id, self.id, self.xpath_absolute, self.xpath_relative, self.repeatable.value,
+    def generate_hash_id(self, project_id: str = None, sdk_version: str = None) -> str:
+        fields_to_hash = [project_id, sdk_version, self.id, self.xpath_absolute, self.xpath_relative,
+                          self.repeatable.value,
                           self.parent_node_id, self.name, self.bt_id, self.value_type,
                           self.legal_type]
         str_content = "_".join(map(str, fields_to_hash))
         return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest())
 
 
+def generate_project_eforms_field_hash_id(
+        sdk_element_id: str,
+        repeatable: bool,
+        parent_node_id: Optional[str] = None,
+        absolute_xpath: str = None,
+        relative_xpath: str = None,
+        project_id: str = None
+):
+    fields_to_hash = [project_id, sdk_element_id, absolute_xpath, relative_xpath, repeatable, parent_node_id]
+    str_content = "_".join(map(str, fields_to_hash))
+    return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest())
+
+
 def generate_eforms_node_hash_id(
         id: str,
         repeatable: bool,
         parent_id: Optional[str] = None,
         xpath_absolute: str = None,
         xpath_relative: str = None,
-        project_id: str = None
+        project_id: str = None,
+        sdk_version: str = None
 ):
-    fields_to_hash = [project_id, id, xpath_absolute, xpath_relative, repeatable, parent_id]
+    fields_to_hash = [project_id, sdk_version, id, xpath_absolute, xpath_relative, repeatable, parent_id]
     str_content = "_".join(map(str, fields_to_hash))
     return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest())
 
@@ -48,14 +63,15 @@ class EFormsNode(BaseModel):
     xpath_relative: str = Field(..., alias='xpathRelative')
     repeatable: bool
 
-    def generate_hash_id(self, project_id: str = None):
+    def generate_hash_id(self, project_id: str = None, sdk_version: str = None):
         return generate_eforms_node_hash_id(
             id=self.id,
             repeatable=self.repeatable,
             parent_id=self.parent_id,
             xpath_absolute=self.xpath_absolute,
             xpath_relative=self.xpath_relative,
-            project_id=project_id
+            project_id=project_id,
+            sdk_version=sdk_version
         )
 
 

mapping_workbench/backend/fields_registry/models/field_registry.py

@@ -1,14 +1,16 @@
+import datetime
 from abc import ABC
 from typing import Optional, List, Literal
 
 import pymongo
-from beanie import Link
+from beanie import Link, PydanticObjectId
 from pydantic import Field, BaseModel
 from pymongo import IndexModel
 
 from mapping_workbench.backend.core.models.api_response import APIListPaginatedResponse
 from mapping_workbench.backend.core.models.base_project_resource_entity import BaseProjectResourceEntity, \
     BaseProjectResourceEntityOutSchema, BaseProjectResourceEntityInSchema
+from mapping_workbench.backend.fields_registry.models.pool import PoolSDKField
 from mapping_workbench.backend.state_manager.models.state_object import StatefulObjectABC, ObjectState
 
 
@@ -117,6 +119,27 @@
     def set_state(self, state: StructuralElementState):
         raise Exception("Setting the state of a structural element is not supported.")
 
+    @classmethod
+    def from_pool_sdk_field(cls, sdk_field: PoolSDKField, project_id: PydanticObjectId) -> 'StructuralElement':
+        # Create and return an instance of StructuralElement by mapping fields
+        return cls(
+            id=sdk_field.element_id,
+            sdk_element_id=sdk_field.sdk_element_id,
+            absolute_xpath=sdk_field.absolute_xpath,
+            relative_xpath=sdk_field.relative_xpath,
+            parent_node_id=sdk_field.parent_node_id,
+            name=sdk_field.name,
+            bt_id=sdk_field.bt_id,
+            value_type=sdk_field.value_type,
+            legal_type=sdk_field.legal_type,
+            element_type=sdk_field.element_type,
+            descriptions=sdk_field.descriptions or [],
+            is_used_in_conceptual_mapping_rules=False,
+            versions=[sdk_field.version],
+            created_at=datetime.datetime.now(),
+            updated_at=datetime.datetime.now()
+        )
+
     class Settings(BaseProjectResourceEntity.Settings):
         name = "structural_elements_registry"
 

mapping_workbench/backend/fields_registry/models/pool.py

@@ -0,0 +1,41 @@
+from typing import Optional, List, Literal
+
+from beanie import Link, Document
+
+
+class PoolSDKField(Document):
+    """
+
+    """
+    element_id: str = None
+    sdk_element_id: Optional[str] = None
+    absolute_xpath: Optional[str] = None
+    relative_xpath: Optional[str] = None
+    repeatable: Optional[bool] = None
+    parent_node_id: Optional[str] = None
+    descriptions: Optional[List[str]] = None
+    sdk_version: str = None
+    version: str = None
+    name: Optional[str] = None
+    bt_id: Optional[str] = None
+    value_type: Optional[str] = None
+    legal_type: Optional[str] = None
+    element_type: Literal["node", "field"] = "field"
+    order: Optional[int] = None
+
+    class Settings:
+        name = "pool_sdk_fields"
+
+
+class PoolSDKFieldsVersionedView(Document):
+    """
+
+    """
+    view_id: str
+    eforms_sdk_version: str
+    eforms_subtype: str
+    ordered_elements: List[Link[PoolSDKField]] = None
+
+    class Settings:
+        name = "pool_sdk_fields_versioned_view"
+