From 22f35bf1f4a0d558f0dc1cc7280abb0ac0a232ff Mon Sep 17 00:00:00 2001 From: CaptainOfHacks <39195263+CaptainOfHacks@users.noreply.github.com> Date: Sat, 27 Jan 2024 16:20:49 +0200 Subject: [PATCH] update MappingSuite to support eForms --- ted_sws/core/model/transform.py | 10 +++++ .../services/notice_eligibility.py | 38 +++++++++++-------- tests/conftest.py | 10 ++++- .../test_package4/metadata.json | 3 +- tests/unit/data_manager/conftest.py | 9 +++++ .../test_mapping_suite_repository.py | 16 ++++++++ .../notice_metadata_processor/conftest.py | 1 - .../test_eligibility.py | 12 ++++-- 8 files changed, 76 insertions(+), 23 deletions(-) diff --git a/ted_sws/core/model/transform.py b/ted_sws/core/model/transform.py index 6c1db2520..7ddd0f663 100644 --- a/ted_sws/core/model/transform.py +++ b/ted_sws/core/model/transform.py @@ -8,6 +8,7 @@ """ """ import abc from datetime import datetime +from enum import Enum from typing import List, Optional from ted_sws.core.model import PropertyBaseModel @@ -188,6 +189,14 @@ class ConceptualMapping(MappingSuiteComponent): cl2_organisations: List[ConceptualMappingControlList] = [] +class MappingSuiteType(str, Enum): + STANDARD_FORMS = "standard_forms" + ELECTRONIC_FORMS = "eforms" + + def __str__(self): + return self.value + + class MappingSuite(MappingSuiteComponent): """ @@ -199,6 +208,7 @@ class MappingSuite(MappingSuiteComponent): ontology_version: str = "0.0.1" git_latest_commit_hash: str = "no_hash" mapping_suite_hash_digest: str = "no_hash" + mapping_type: Optional[MappingSuiteType] = MappingSuiteType.STANDARD_FORMS metadata_constraints: MetadataConstraints transformation_rule_set: TransformationRuleSet shacl_test_suites: List[SHACLTestSuite] diff --git a/ted_sws/notice_metadata_processor/services/notice_eligibility.py b/ted_sws/notice_metadata_processor/services/notice_eligibility.py index ec5864bd7..fc65765c3 100644 --- a/ted_sws/notice_metadata_processor/services/notice_eligibility.py +++ b/ted_sws/notice_metadata_processor/services/notice_eligibility.py @@ -5,13 +5,13 @@ from ted_sws.core.model.metadata import NormalisedMetadata, NoticeSource from ted_sws.core.model.notice import Notice -from ted_sws.core.model.transform import MappingSuite +from ted_sws.core.model.transform import MappingSuite, MappingSuiteType from ted_sws.data_manager.adapters.repository_abc import MappingSuiteRepositoryABC, NoticeRepositoryABC from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_metadata import START_DATE_KEY, END_DATE_KEY, \ MIN_XSD_VERSION_KEY, MAX_XSD_VERSION_KEY, E_FORMS_SUBTYPE_KEY, EFORMS_SDK_VERSIONS_KEY -def format_version_with_zero_patch(version_string:str) -> semantic_version.Version: +def format_version_with_zero_patch(version_string: str) -> semantic_version.Version: """ This will take a string version (1.7 or 1.7.6) and will transform it to a semantic version with 0 as patch 1.7 -> 1.7.0 @@ -35,6 +35,25 @@ def is_date_in_range(publication_date, constraint_start_date_value, constraint_e return start_date <= publication_date <= end_date +def is_version_in_range(notice_metadata: NormalisedMetadata, mapping_suite: MappingSuite) -> bool: + constraints = mapping_suite.metadata_constraints.constraints + if mapping_suite.mapping_type == MappingSuiteType.ELECTRONIC_FORMS and notice_metadata.notice_source == NoticeSource.ELECTRONIC_FORM: + notice_xsd_version = notice_metadata.eform_sdk_version + # eform sdk version value in metadata example: eforms-sdk-1.7 or eforms-sdk-1.7.9 + # we need to extract only the version i.e 1.7 or 1.7.9 + eforms_sdk_version = notice_xsd_version.rsplit('-', 1)[1] + constraint_version_range = [format_version_with_zero_patch(version) for version in + constraints[EFORMS_SDK_VERSIONS_KEY]] + return format_version_with_zero_patch(eforms_sdk_version) in constraint_version_range + elif mapping_suite.mapping_type == MappingSuiteType.STANDARD_FORMS and notice_metadata.notice_source == NoticeSource.STANDARD_FORM: + notice_xsd_version = notice_metadata.xsd_version + constraint_min_xsd_version = constraints[MIN_XSD_VERSION_KEY][0] + constraint_max_xsd_version = constraints[MAX_XSD_VERSION_KEY][0] + return constraint_min_xsd_version <= notice_xsd_version <= constraint_max_xsd_version + + return False + + def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetadata): """ Check if mapping suite is valid for notice @@ -47,20 +66,7 @@ def check_package(mapping_suite: MappingSuite, notice_metadata: NormalisedMetada eform_subtype = notice_metadata.eforms_subtype notice_publication_date = datetime.datetime.fromisoformat(notice_metadata.publication_date) - - if notice_metadata.notice_source == NoticeSource.ELECTRONIC_FORM: - notice_xsd_version = notice_metadata.eform_sdk_version - # eform sdk version value in metadata example: eforms-sdk-1.7 or eforms-sdk-1.7.9 - # we need to extract only the version i.e 1.7 or 1.7.9 - eforms_sdk_version = notice_xsd_version.rsplit('-', 1)[1] - constraint_version_range = [format_version_with_zero_patch(version) for version in - constraints[EFORMS_SDK_VERSIONS_KEY]] - in_version_range = format_version_with_zero_patch(eforms_sdk_version) in constraint_version_range - else: - notice_xsd_version = notice_metadata.xsd_version - constraint_min_xsd_version = constraints[MIN_XSD_VERSION_KEY][0] - constraint_max_xsd_version = constraints[MAX_XSD_VERSION_KEY][0] - in_version_range = constraint_min_xsd_version <= notice_xsd_version <= constraint_max_xsd_version + in_version_range = is_version_in_range(notice_metadata=notice_metadata, mapping_suite=mapping_suite) in_date_range = is_date_in_range(publication_date=notice_publication_date, constraint_start_date_value=constraints[START_DATE_KEY], diff --git a/tests/conftest.py b/tests/conftest.py index f39f54344..1eba4e368 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -212,6 +212,7 @@ def normalised_metadata_object(): return NormalisedMetadata(**data) + @pytest.fixture def eform_normalised_metadata_object(): data = { @@ -242,6 +243,7 @@ def eform_normalised_metadata_object(): return NormalisedMetadata(**data) + @pytest.fixture @mongomock.patch(servers=(('server.example.com', 27017),)) def mongodb_client(): @@ -302,4 +304,10 @@ def eform_notice_622690(): notice = Notice(ted_id=ted_id) notice.set_xml_manifestation(xml_manifestation) notice.set_original_metadata(original_metadata) - return notice \ No newline at end of file + return notice + + +@pytest.fixture +def indexed_eform_notice_622690(eform_notice_622690): + eform_notice_622690.set_xml_metadata(XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"])) + return eform_notice_622690 diff --git a/tests/test_data/notice_transformer/test_repository/test_package4/metadata.json b/tests/test_data/notice_transformer/test_repository/test_package4/metadata.json index 9aba313b7..cb7672dd4 100644 --- a/tests/test_data/notice_transformer/test_repository/test_package4/metadata.json +++ b/tests/test_data/notice_transformer/test_repository/test_package4/metadata.json @@ -4,10 +4,9 @@ "description": "This is the conceptual mapping for bla bla bla", "mapping_version": "3.0.0-alpha.1", "ontology_version": "4.0.0", + "mapping_type": "eforms", "metadata_constraints": { "constraints": { - "min_xsd_version": ["R2.0.9.S04.E01"], - "max_xsd_version": ["R2.0.9.S04.E01"], "eforms_subtype": [ "16", "10", diff --git a/tests/unit/data_manager/conftest.py b/tests/unit/data_manager/conftest.py index 2ed7fcf3c..75b9ec2ee 100644 --- a/tests/unit/data_manager/conftest.py +++ b/tests/unit/data_manager/conftest.py @@ -16,6 +16,15 @@ def file_system_repository_path(): return TEST_DATA_PATH / "notice_transformer" / "test_file_system_repository" +@pytest.fixture +def file_system_repository_with_packages_path(): + return TEST_DATA_PATH / "notice_transformer" / "test_repository" + + +@pytest.fixture +def epo_mapping_suite_package_name(): + return "test_package4" + @pytest.fixture def fake_mapping_suite(): metadata_constrains = MetadataConstraints(constraints=dict()) diff --git a/tests/unit/data_manager/test_mapping_suite_repository.py b/tests/unit/data_manager/test_mapping_suite_repository.py index 9010c703a..b20ab5b91 100644 --- a/tests/unit/data_manager/test_mapping_suite_repository.py +++ b/tests/unit/data_manager/test_mapping_suite_repository.py @@ -38,6 +38,22 @@ def test_mapping_suite_repository_mongodb_update_invalid_id(mongodb_client, fake mongodb_client.drop_database(aggregates_database_name) +def test_epo_mapping_suite_repository_in_file_system(file_system_repository_with_packages_path, + epo_mapping_suite_package_name): + assert file_system_repository_with_packages_path.exists() + mapping_suite_repository = MappingSuiteRepositoryInFileSystem( + repository_path=file_system_repository_with_packages_path) + result_mapping_suite = mapping_suite_repository.get(reference=epo_mapping_suite_package_name) + assert result_mapping_suite + assert result_mapping_suite.identifier == "package_EF16" + assert result_mapping_suite.title == "Package EF16 v1.2" + assert result_mapping_suite.mapping_type == "eforms" + assert result_mapping_suite.metadata_constraints.constraints + constraints = result_mapping_suite.metadata_constraints.constraints + assert constraints["eforms_subtype"] + assert constraints["eforms_sdk_versions"] + + def test_mapping_suite_repository_in_file_system(file_system_repository_path, fake_mapping_suite): mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=file_system_repository_path) mapping_suite_repository.clear_repository() diff --git a/tests/unit/notice_metadata_processor/conftest.py b/tests/unit/notice_metadata_processor/conftest.py index 2f3435e8b..b56f6610e 100644 --- a/tests/unit/notice_metadata_processor/conftest.py +++ b/tests/unit/notice_metadata_processor/conftest.py @@ -11,4 +11,3 @@ def notice_eligibility_repository_path(): @pytest.fixture def file_system_repository_path(): return TEST_DATA_PATH / "notice_transformer" / "mapping_suite_processor_repository" - diff --git a/tests/unit/notice_metadata_processor/test_eligibility.py b/tests/unit/notice_metadata_processor/test_eligibility.py index ca90509dc..22ba880f3 100644 --- a/tests/unit/notice_metadata_processor/test_eligibility.py +++ b/tests/unit/notice_metadata_processor/test_eligibility.py @@ -9,8 +9,6 @@ notice_eligibility_checker, notice_eligibility_checker_by_id, format_version_with_zero_patch, is_date_in_range -# TODO remove min and max xsd version when the mapping loader is refactored tests/test_data/notice_transformer/test_repository/test_package4/metadata.json -# TODO Add test for eform using mappinSuiteRepository def test_non_eligibility_by_notice(notice_eligibility_repository_path, indexed_notice): mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=notice_eligibility_repository_path) normalise_notice(notice=indexed_notice) @@ -18,6 +16,13 @@ def test_non_eligibility_by_notice(notice_eligibility_repository_path, indexed_n assert indexed_notice.status == NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION +def test_eforms_eligibility_by_notice(notice_eligibility_repository_path, indexed_eform_notice_622690): + mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=notice_eligibility_repository_path) + normalise_notice(notice=indexed_eform_notice_622690) + notice_eligibility_checker(notice=indexed_eform_notice_622690, mapping_suite_repository=mapping_suite_repository) + assert indexed_eform_notice_622690.status == NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION + + def test_eligibility_by_notice(notice_eligibility_repository_path, notice_2020): mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=notice_eligibility_repository_path) normalise_notice(notice=notice_2020) @@ -42,7 +47,8 @@ def test_eligibility_by_notice_id(notice_eligibility_repository_path, notice_202 assert notice_2020.status == NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION -def test_check_mapping_suite(notice_eligibility_repository_path, normalised_metadata_object, eform_normalised_metadata_object): +def test_check_mapping_suite(notice_eligibility_repository_path, normalised_metadata_object, + eform_normalised_metadata_object): mapping_suite_repository = MappingSuiteRepositoryInFileSystem(repository_path=notice_eligibility_repository_path) is_valid = check_package(mapping_suite=mapping_suite_repository.get("test_package"), notice_metadata=normalised_metadata_object)