Added SDK fields preloading

mapping_workbench/backend/fields_registry/models/eforms_fields_structure.py

@@ -29,8 +29,15 @@ def generate_hash_id(self, project_id: str = None, sdk_version: str = None) -> s
         return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest())
 
 
-def generate_project_eforms_field_hash_id(element_id: str, project_id: str) -> str:
-    fields_to_hash = [element_id, project_id, id]
+def generate_project_eforms_field_hash_id(
+        sdk_element_id: str,
+        repeatable: bool,
+        parent_node_id: Optional[str] = None,
+        absolute_xpath: str = None,
+        relative_xpath: str = None,
+        project_id: str = None
+):
+    fields_to_hash = [project_id, sdk_element_id, absolute_xpath, relative_xpath, repeatable, parent_node_id]
     str_content = "_".join(map(str, fields_to_hash))
     return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest())
 

mapping_workbench/backend/fields_registry/models/pool.py

@@ -14,6 +14,7 @@ class PoolSDKField(Document):
     repeatable: Optional[bool] = None
     parent_node_id: Optional[str] = None
     descriptions: Optional[List[str]] = None
+    sdk_version: str = None
     version: str = None
     name: Optional[str] = None
     bt_id: Optional[str] = None

mapping_workbench/backend/fields_registry/services/import_fields_registry.py

@@ -36,7 +36,14 @@ async def import_eforms_fields_from_pool_to_project(project_link: Link[Project],
     sdk_fields = await PoolSDKField.find(PoolSDKField.version == version).to_list()
 
     for sdk_field in sdk_fields:
-        field_hash_id = generate_project_eforms_field_hash_id(sdk_field.element_id, project_id)
+        field_hash_id = generate_project_eforms_field_hash_id(
+            sdk_element_id=sdk_field.sdk_element_id,
+            repeatable=sdk_field.repeatable,
+            parent_node_id=sdk_field.parent_node_id,
+            absolute_xpath=sdk_field.absolute_xpath,
+            relative_xpath=sdk_field.relative_xpath,
+            project_id=project_id
+        )
         structural_node = await StructuralElement.find_one(
             StructuralElement.id == field_hash_id,
             StructuralElement.project == project_link
@@ -45,14 +52,14 @@ async def import_eforms_fields_from_pool_to_project(project_link: Link[Project],
             structural_node = StructuralElement(
                 id=field_hash_id,
                 sdk_element_id=sdk_field.sdk_element_id,
-                absolute_xpath=sdk_field.xpath_absolute,
-                relative_xpath=sdk_field.xpath_relative,
+                absolute_xpath=sdk_field.absolute_xpath,
+                relative_xpath=sdk_field.relative_xpath,
                 repeatable=sdk_field.repeatable,
                 name=sdk_field.name,
                 bt_id=sdk_field.bt_id,
                 value_type=sdk_field.value_type,
                 legal_type=sdk_field.legal_type,
-                parent_node_id=sdk_field.parent_id,
+                parent_node_id=sdk_field.parent_node_id,
                 project=project_link,
                 element_type=sdk_field.element_type,
                 created_at=datetime.now(tzlocal())
@@ -66,13 +73,18 @@ async def import_eforms_fields_from_pool_to_project(project_link: Link[Project],
     return len(sdk_fields) > 0
 
 
+def field_version_from_sdk_version(sdk_version: str) -> str:
+    return sdk_version.replace("eforms-sdk-", "")
+
+
 async def import_eforms_fields_to_pool(eforms_fields_content: dict) -> dict:
     """
 
     """
     eforms_sdk_fields = EFormsSDKFields(**eforms_fields_content)
 
     eforms_sdk_version = eforms_sdk_fields.sdk_version
+    field_version = field_version_from_sdk_version(eforms_sdk_version)
     result_dict = {VERSION_KEY: eforms_sdk_version}
     id_to_hash_mapping = {}
     for eforms_node in eforms_sdk_fields.xml_structure:
@@ -91,7 +103,8 @@ async def import_eforms_fields_to_pool(eforms_fields_content: dict) -> dict:
                 relative_xpath=eforms_node.xpath_relative,
                 repeatable=eforms_node.repeatable,
                 parent_node_id=eforms_node.parent_id,
-                version=eforms_sdk_version,
+                sdk_version=eforms_sdk_version,
+                version=field_version,
                 element_type="node"
             )
             await sdk_field.save()
@@ -115,7 +128,8 @@ async def import_eforms_fields_to_pool(eforms_fields_content: dict) -> dict:
                 value_type=eforms_field.value_type,
                 legal_type=eforms_field.legal_type,
                 parent_node_id=eforms_field.parent_node_id,
-                version=eforms_sdk_version
+                sdk_version=eforms_sdk_version,
+                version=field_version
             )
             await sdk_field.save()
         id_to_hash_mapping[eforms_field.id] = sdk_field.element_id
@@ -155,7 +169,7 @@ async def import_notice_types_versioned_view_to_pool(
 
     pool_sdk_fields_versioned_view_id = f"{fields_metadata[VERSION_KEY]}_{notice_type_structure.notice_type_id}"
 
-    pool_sdk_fields_versioned_view = PoolSDKFieldsVersionedView.find_one(
+    pool_sdk_fields_versioned_view = await PoolSDKFieldsVersionedView.find_one(
         PoolSDKFieldsVersionedView.view_id == pool_sdk_fields_versioned_view_id
     )
     if not pool_sdk_fields_versioned_view:
@@ -216,19 +230,21 @@ async def import_eforms_fields_from_folder_to_pool(
 
 
 async def import_eforms_xsd(
-        github_repository_url: str,
         branch_or_tag_name: str,
+        github_repository_url: str = None,
         project_link: Link[Project] = None
 ):
     versions = [item.strip() for item in branch_or_tag_name.split(',')]
     for version in versions:
-        if not await import_eforms_fields_from_pool_to_project(project_link=project_link, version=version):
+        if not await import_eforms_fields_from_pool_to_project(project_link=project_link, version=version) \
+                and github_repository_url:
             await import_eforms_fields_from_github_repository(
                 github_repository_url=github_repository_url,
                 branch_or_tag_name=version,
                 project_link=project_link
             )
 
+
 async def import_eforms_fields_from_github_repository(
         github_repository_url: str,
         branch_or_tag_name: str,

mapping_workbench/backend/fields_registry/services/tasks.py

@@ -12,5 +12,5 @@ def task_import_eforms_xsd(
 ):
     run_task(
         import_eforms_xsd,
-        github_repository_url, branch_or_tag_name, project_link
+        branch_or_tag_name, github_repository_url, project_link
     )

mapping_workbench/frontend/src/api/fields-overview/index.js

@@ -19,7 +19,7 @@ class FieldsOverview  extends SectionApi {
         this.isProjectResource = true;
     }
 
-    importEFormsFromGithub(request) {
+    importEFormsXSD(request) {
         try {
             const endpoint = this.paths['import_eforms_xsd'];
             const headers = {"Content-Type": "multipart/form-data"};

mapping_workbench/frontend/src/api/fields-registry/index.js

@@ -24,7 +24,7 @@ class FieldsRegistryApi extends SectionApi {
         this.isProjectResource = true;
     }
 
-    importEFormsFromGithub(request) {
+    importEFormsXSD(request) {
         try {
             let endpoint = this.paths['import_eforms_xsd'];
             const headers = {"Content-Type": "multipart/form-data"};

mapping_workbench/frontend/src/pages/app/fields-and-nodes/overview/import.js

@@ -49,7 +49,7 @@ const Page = () => {
             setIsRunning(true)
             values['project_id'] = sessionApi.getSessionProject();
             const toastId = toastLoad(`Importing eForm Fields ... `)
-            sectionApi.importEFormsFromGithub(values)
+            sectionApi.importEFormsXSD(values)
                 .then((res) => {
                     helpers.setStatus({success: true});
                     toastSuccess(`${res.task_name} successfully started.`, toastId)

mapping_workbench/frontend/src/sections/app/project/edit-form.js

@@ -84,7 +84,7 @@ export const EditForm = (props) => {
 
     const handleImportFieldRegistry = (values, projectId) => {
         const toastId = toastLoad(`Importing eForm Fields ... `)
-        fieldsRegistryApi.importEFormsFromGithub({
+        fieldsRegistryApi.importEFormsXSD({
             ...values,
             project_id: projectId,
             checked: undefined

tests/e2e/backend/fields_registry/test_import_fields_registry.py

@@ -1,17 +1,18 @@
 import pytest
 from beanie import PydanticObjectId
 
-from mapping_workbench.backend.fields_registry.models.field_registry import StructuralElementsVersionedView, \
-    StructuralElement
-from mapping_workbench.backend.fields_registry.services.import_fields_registry import \
-    import_eforms_fields_from_github_repository
+from mapping_workbench.backend.fields_registry.models.field_registry import StructuralElement
+from mapping_workbench.backend.fields_registry.models.pool import PoolSDKFieldsVersionedView
+from mapping_workbench.backend.fields_registry.services.import_fields_registry import import_eforms_xsd
 from mapping_workbench.backend.project.models.entity import Project
 
 
 @pytest.mark.asyncio
-async def _test_import_eforms_fields_from_github_repository(eforms_sdk_github_repository_url,
-                                                           eforms_sdk_github_repository_v1_9_1_tag_name):
-    await import_eforms_fields_from_github_repository(
+async def test_import_eforms_xsd(
+        eforms_sdk_github_repository_url,
+        eforms_sdk_github_repository_v1_9_1_tag_name
+):
+    await import_eforms_xsd(
         github_repository_url=eforms_sdk_github_repository_url,
         branch_or_tag_name=eforms_sdk_github_repository_v1_9_1_tag_name,
         project_link=Project.link_from_id(PydanticObjectId())
@@ -25,13 +26,13 @@ async def _test_import_eforms_fields_from_github_repository(eforms_sdk_github_re
     assert imported_nodes
     assert len(imported_nodes) == 286
 
-    imported_versioned_view = await StructuralElementsVersionedView.find().to_list()
+    imported_versioned_view = await PoolSDKFieldsVersionedView.find().to_list()
     assert imported_versioned_view
     assert len(imported_versioned_view) == 45
     for versioned_view in imported_versioned_view:
         assert versioned_view.ordered_elements
         assert len(versioned_view.ordered_elements) > 0
-    imported_versioned_view = await StructuralElementsVersionedView.find(
-        StructuralElementsVersionedView.eforms_subtype == "1").to_list()
+    imported_versioned_view = await PoolSDKFieldsVersionedView.find(
+        PoolSDKFieldsVersionedView.eforms_subtype == "1").to_list()
     assert imported_versioned_view
     assert len(imported_versioned_view) == 1