diff --git a/mapping_workbench/backend/fields_registry/models/eforms_fields_structure.py b/mapping_workbench/backend/fields_registry/models/eforms_fields_structure.py index e543356d5..b94c80de1 100644 --- a/mapping_workbench/backend/fields_registry/models/eforms_fields_structure.py +++ b/mapping_workbench/backend/fields_registry/models/eforms_fields_structure.py @@ -29,8 +29,15 @@ def generate_hash_id(self, project_id: str = None, sdk_version: str = None) -> s return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest()) -def generate_project_eforms_field_hash_id(element_id: str, project_id: str) -> str: - fields_to_hash = [element_id, project_id, id] +def generate_project_eforms_field_hash_id( + sdk_element_id: str, + repeatable: bool, + parent_node_id: Optional[str] = None, + absolute_xpath: str = None, + relative_xpath: str = None, + project_id: str = None +): + fields_to_hash = [project_id, sdk_element_id, absolute_xpath, relative_xpath, repeatable, parent_node_id] str_content = "_".join(map(str, fields_to_hash)) return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest()) diff --git a/mapping_workbench/backend/fields_registry/models/pool.py b/mapping_workbench/backend/fields_registry/models/pool.py index f15c1d0c5..cbfecb67a 100644 --- a/mapping_workbench/backend/fields_registry/models/pool.py +++ b/mapping_workbench/backend/fields_registry/models/pool.py @@ -14,6 +14,7 @@ class PoolSDKField(Document): repeatable: Optional[bool] = None parent_node_id: Optional[str] = None descriptions: Optional[List[str]] = None + sdk_version: str = None version: str = None name: Optional[str] = None bt_id: Optional[str] = None diff --git a/mapping_workbench/backend/fields_registry/services/import_fields_registry.py b/mapping_workbench/backend/fields_registry/services/import_fields_registry.py index da83fbd88..328f8111d 100644 --- a/mapping_workbench/backend/fields_registry/services/import_fields_registry.py +++ b/mapping_workbench/backend/fields_registry/services/import_fields_registry.py @@ -36,7 +36,14 @@ async def import_eforms_fields_from_pool_to_project(project_link: Link[Project], sdk_fields = await PoolSDKField.find(PoolSDKField.version == version).to_list() for sdk_field in sdk_fields: - field_hash_id = generate_project_eforms_field_hash_id(sdk_field.element_id, project_id) + field_hash_id = generate_project_eforms_field_hash_id( + sdk_element_id=sdk_field.sdk_element_id, + repeatable=sdk_field.repeatable, + parent_node_id=sdk_field.parent_node_id, + absolute_xpath=sdk_field.absolute_xpath, + relative_xpath=sdk_field.relative_xpath, + project_id=project_id + ) structural_node = await StructuralElement.find_one( StructuralElement.id == field_hash_id, StructuralElement.project == project_link @@ -45,14 +52,14 @@ async def import_eforms_fields_from_pool_to_project(project_link: Link[Project], structural_node = StructuralElement( id=field_hash_id, sdk_element_id=sdk_field.sdk_element_id, - absolute_xpath=sdk_field.xpath_absolute, - relative_xpath=sdk_field.xpath_relative, + absolute_xpath=sdk_field.absolute_xpath, + relative_xpath=sdk_field.relative_xpath, repeatable=sdk_field.repeatable, name=sdk_field.name, bt_id=sdk_field.bt_id, value_type=sdk_field.value_type, legal_type=sdk_field.legal_type, - parent_node_id=sdk_field.parent_id, + parent_node_id=sdk_field.parent_node_id, project=project_link, element_type=sdk_field.element_type, created_at=datetime.now(tzlocal()) @@ -66,6 +73,10 @@ async def import_eforms_fields_from_pool_to_project(project_link: Link[Project], return len(sdk_fields) > 0 +def field_version_from_sdk_version(sdk_version: str) -> str: + return sdk_version.replace("eforms-sdk-", "") + + async def import_eforms_fields_to_pool(eforms_fields_content: dict) -> dict: """ @@ -73,6 +84,7 @@ async def import_eforms_fields_to_pool(eforms_fields_content: dict) -> dict: eforms_sdk_fields = EFormsSDKFields(**eforms_fields_content) eforms_sdk_version = eforms_sdk_fields.sdk_version + field_version = field_version_from_sdk_version(eforms_sdk_version) result_dict = {VERSION_KEY: eforms_sdk_version} id_to_hash_mapping = {} for eforms_node in eforms_sdk_fields.xml_structure: @@ -91,7 +103,8 @@ async def import_eforms_fields_to_pool(eforms_fields_content: dict) -> dict: relative_xpath=eforms_node.xpath_relative, repeatable=eforms_node.repeatable, parent_node_id=eforms_node.parent_id, - version=eforms_sdk_version, + sdk_version=eforms_sdk_version, + version=field_version, element_type="node" ) await sdk_field.save() @@ -115,7 +128,8 @@ async def import_eforms_fields_to_pool(eforms_fields_content: dict) -> dict: value_type=eforms_field.value_type, legal_type=eforms_field.legal_type, parent_node_id=eforms_field.parent_node_id, - version=eforms_sdk_version + sdk_version=eforms_sdk_version, + version=field_version ) await sdk_field.save() id_to_hash_mapping[eforms_field.id] = sdk_field.element_id @@ -155,7 +169,7 @@ async def import_notice_types_versioned_view_to_pool( pool_sdk_fields_versioned_view_id = f"{fields_metadata[VERSION_KEY]}_{notice_type_structure.notice_type_id}" - pool_sdk_fields_versioned_view = PoolSDKFieldsVersionedView.find_one( + pool_sdk_fields_versioned_view = await PoolSDKFieldsVersionedView.find_one( PoolSDKFieldsVersionedView.view_id == pool_sdk_fields_versioned_view_id ) if not pool_sdk_fields_versioned_view: @@ -216,19 +230,21 @@ async def import_eforms_fields_from_folder_to_pool( async def import_eforms_xsd( - github_repository_url: str, branch_or_tag_name: str, + github_repository_url: str = None, project_link: Link[Project] = None ): versions = [item.strip() for item in branch_or_tag_name.split(',')] for version in versions: - if not await import_eforms_fields_from_pool_to_project(project_link=project_link, version=version): + if not await import_eforms_fields_from_pool_to_project(project_link=project_link, version=version) \ + and github_repository_url: await import_eforms_fields_from_github_repository( github_repository_url=github_repository_url, branch_or_tag_name=version, project_link=project_link ) + async def import_eforms_fields_from_github_repository( github_repository_url: str, branch_or_tag_name: str, diff --git a/mapping_workbench/backend/fields_registry/services/tasks.py b/mapping_workbench/backend/fields_registry/services/tasks.py index 5ee51a4c5..421e20e39 100644 --- a/mapping_workbench/backend/fields_registry/services/tasks.py +++ b/mapping_workbench/backend/fields_registry/services/tasks.py @@ -12,5 +12,5 @@ def task_import_eforms_xsd( ): run_task( import_eforms_xsd, - github_repository_url, branch_or_tag_name, project_link + branch_or_tag_name, github_repository_url, project_link ) diff --git a/mapping_workbench/frontend/src/api/fields-overview/index.js b/mapping_workbench/frontend/src/api/fields-overview/index.js index 1a76c3642..3452fa5cc 100644 --- a/mapping_workbench/frontend/src/api/fields-overview/index.js +++ b/mapping_workbench/frontend/src/api/fields-overview/index.js @@ -19,7 +19,7 @@ class FieldsOverview extends SectionApi { this.isProjectResource = true; } - importEFormsFromGithub(request) { + importEFormsXSD(request) { try { const endpoint = this.paths['import_eforms_xsd']; const headers = {"Content-Type": "multipart/form-data"}; diff --git a/mapping_workbench/frontend/src/api/fields-registry/index.js b/mapping_workbench/frontend/src/api/fields-registry/index.js index ba1bd2057..52c85a0c0 100644 --- a/mapping_workbench/frontend/src/api/fields-registry/index.js +++ b/mapping_workbench/frontend/src/api/fields-registry/index.js @@ -24,7 +24,7 @@ class FieldsRegistryApi extends SectionApi { this.isProjectResource = true; } - importEFormsFromGithub(request) { + importEFormsXSD(request) { try { let endpoint = this.paths['import_eforms_xsd']; const headers = {"Content-Type": "multipart/form-data"}; diff --git a/mapping_workbench/frontend/src/pages/app/fields-and-nodes/overview/import.js b/mapping_workbench/frontend/src/pages/app/fields-and-nodes/overview/import.js index 767da52c2..cfd9a24c7 100644 --- a/mapping_workbench/frontend/src/pages/app/fields-and-nodes/overview/import.js +++ b/mapping_workbench/frontend/src/pages/app/fields-and-nodes/overview/import.js @@ -49,7 +49,7 @@ const Page = () => { setIsRunning(true) values['project_id'] = sessionApi.getSessionProject(); const toastId = toastLoad(`Importing eForm Fields ... `) - sectionApi.importEFormsFromGithub(values) + sectionApi.importEFormsXSD(values) .then((res) => { helpers.setStatus({success: true}); toastSuccess(`${res.task_name} successfully started.`, toastId) diff --git a/mapping_workbench/frontend/src/sections/app/project/edit-form.js b/mapping_workbench/frontend/src/sections/app/project/edit-form.js index ccddc2255..26252b8c2 100644 --- a/mapping_workbench/frontend/src/sections/app/project/edit-form.js +++ b/mapping_workbench/frontend/src/sections/app/project/edit-form.js @@ -84,7 +84,7 @@ export const EditForm = (props) => { const handleImportFieldRegistry = (values, projectId) => { const toastId = toastLoad(`Importing eForm Fields ... `) - fieldsRegistryApi.importEFormsFromGithub({ + fieldsRegistryApi.importEFormsXSD({ ...values, project_id: projectId, checked: undefined diff --git a/tests/e2e/backend/fields_registry/test_import_fields_registry.py b/tests/e2e/backend/fields_registry/test_import_fields_registry.py index 76929a103..00312b7d2 100644 --- a/tests/e2e/backend/fields_registry/test_import_fields_registry.py +++ b/tests/e2e/backend/fields_registry/test_import_fields_registry.py @@ -1,17 +1,18 @@ import pytest from beanie import PydanticObjectId -from mapping_workbench.backend.fields_registry.models.field_registry import StructuralElementsVersionedView, \ - StructuralElement -from mapping_workbench.backend.fields_registry.services.import_fields_registry import \ - import_eforms_fields_from_github_repository +from mapping_workbench.backend.fields_registry.models.field_registry import StructuralElement +from mapping_workbench.backend.fields_registry.models.pool import PoolSDKFieldsVersionedView +from mapping_workbench.backend.fields_registry.services.import_fields_registry import import_eforms_xsd from mapping_workbench.backend.project.models.entity import Project @pytest.mark.asyncio -async def _test_import_eforms_fields_from_github_repository(eforms_sdk_github_repository_url, - eforms_sdk_github_repository_v1_9_1_tag_name): - await import_eforms_fields_from_github_repository( +async def test_import_eforms_xsd( + eforms_sdk_github_repository_url, + eforms_sdk_github_repository_v1_9_1_tag_name +): + await import_eforms_xsd( github_repository_url=eforms_sdk_github_repository_url, branch_or_tag_name=eforms_sdk_github_repository_v1_9_1_tag_name, project_link=Project.link_from_id(PydanticObjectId()) @@ -25,13 +26,13 @@ async def _test_import_eforms_fields_from_github_repository(eforms_sdk_github_re assert imported_nodes assert len(imported_nodes) == 286 - imported_versioned_view = await StructuralElementsVersionedView.find().to_list() + imported_versioned_view = await PoolSDKFieldsVersionedView.find().to_list() assert imported_versioned_view assert len(imported_versioned_view) == 45 for versioned_view in imported_versioned_view: assert versioned_view.ordered_elements assert len(versioned_view.ordered_elements) > 0 - imported_versioned_view = await StructuralElementsVersionedView.find( - StructuralElementsVersionedView.eforms_subtype == "1").to_list() + imported_versioned_view = await PoolSDKFieldsVersionedView.find( + PoolSDKFieldsVersionedView.eforms_subtype == "1").to_list() assert imported_versioned_view assert len(imported_versioned_view) == 1