From 774c953c99c7ce23a00bac1069352a4dd185b8d1 Mon Sep 17 00:00:00 2001 From: CaptainOfHacks <39195263+CaptainOfHacks@users.noreply.github.com> Date: Tue, 10 Oct 2023 18:00:34 +0300 Subject: [PATCH] code cleaning and update to pydantic v2 --- .../notice_batch_processor_pipelines.py | 2 +- ted_sws/__init__.py | 3 ++- .../services/limes_configurator.py | 2 +- .../daily_notices_metadata_repository.py | 3 +-- .../adapters/manifestation_repository.py | 2 +- .../adapters/mapping_suite_repository.py | 4 +-- .../adapters/metadata_repository.py | 2 +- .../adapters/notice_repository.py | 8 +++--- .../adapters/supra_notice_repository.py | 4 +-- .../services/export_notice_from_mongodb.py | 6 ++--- .../services/notice_xml_indexer.py | 4 +-- .../event_manager/adapters/log/__init__.py | 4 +-- ted_sws/event_manager/services/log.py | 2 +- .../conceptual_mapping_generate_metadata.py | 2 +- .../services/mapping_suite_digest_service.py | 3 ++- .../services/entity_deduplication.py | 3 ++- .../services/metadata_normalizer.py | 4 +-- .../adapters/template_generator.py | 2 +- ted_sws/notice_packager/model/metadata.py | 4 +-- .../services/notice_publisher.py | 1 - .../adapters/notice_batch_transformer.py | 1 + .../entrypoints/api/digest_service/main.py | 3 ++- .../api/digest_service/routes/hashing.py | 1 + .../entrypoints/api/digest_service/server.py | 2 +- .../adapters/xpath_coverage_runner.py | 2 +- .../check_availability_of_notice_in_cellar.py | 1 + .../services/shacl_test_suite_runner.py | 4 +-- .../services/sparql_test_suite_runner.py | 6 ++--- tests/clean_mongo_db.py | 3 ++- tests/conftest.py | 4 +-- tests/e2e/conftest.py | 1 - tests/e2e/dags/__init__.py | 2 -- tests/e2e/dags/_test_notice_transmisions.py | 7 ++--- tests/e2e/dags/conftest.py | 2 -- .../_test_notice_unpacking_in_file_system.py | 13 +++++----- ...t_supra_notice_unpacking_in_file_system.py | 1 + .../data_manager/test_fuseki_triple_store.py | 1 + ...github_mapping_suite_package_downloader.py | 1 + tests/e2e/master_data_registry/conftest.py | 1 + .../test_entity_deduplication.py | 1 - .../test_rdf_fragment_processor.py | 1 + .../e2e/notice_fetcher/test_notice_fetcher.py | 1 + .../adapters/test_notice_publisher.py | 4 +-- .../services/test_notice_publisher.py | 1 - ...ad_transformed_notice_into_triple_store.py | 3 ++- tests/fakes/fake_notice.py | 2 +- tests/fakes/fake_repository.py | 2 +- tests/fakes/fake_rml_mapper.py | 2 +- tests/fakes/fake_triple_store.py | 1 + .../features/master_data_registry/conftest.py | 1 + .../test_notice_extractor.py | 11 ++++---- tests/features/notice_packager/conftest.py | 1 - tests/features/notice_transformer/conftest.py | 9 ++----- tests/unit/conftest.py | 1 + tests/unit/core/model/test_metadata.py | 2 +- tests/unit/core/model/test_notice_creation.py | 4 +-- .../core/model/test_notice_rdf_validation.py | 3 +-- .../model/test_notice_status_transition.py | 2 +- .../unit/core/model/test_notice_validation.py | 2 +- tests/unit/core/model/test_pydantic.py | 4 +-- tests/unit/data_manager/conftest.py | 2 ++ .../test_mapping_suite_repository.py | 1 - .../data_manager/test_notice_repository.py | 5 ++-- tests/unit/digest_service/conftest.py | 3 ++- tests/unit/digest_service/test_api_hashing.py | 3 ++- .../unit/mapping_suite_processor/conftest.py | 2 -- .../notice_fetcher/test_notice_fetcher.py | 1 + .../test_metadata_extractor.py | 26 ++++++++++--------- .../test_metadata_normaliser.py | 2 +- .../notice_metadata_processor/test_model.py | 8 +++--- tests/unit/notice_packager/conftest.py | 3 ++- .../notice_packager/test_notice_packager.py | 2 +- .../test_template_generator.py | 1 - tests/unit/notice_transformer/conftest.py | 6 +++-- .../test_shacl_test_suite_runner.py | 2 +- .../test_validation_summary_runner.py | 2 +- tests/unit/supra_notice_manager/conftest.py | 6 ++--- 77 files changed, 132 insertions(+), 122 deletions(-) diff --git a/dags/pipelines/notice_batch_processor_pipelines.py b/dags/pipelines/notice_batch_processor_pipelines.py index 3febd6f87..76b0474a6 100644 --- a/dags/pipelines/notice_batch_processor_pipelines.py +++ b/dags/pipelines/notice_batch_processor_pipelines.py @@ -22,7 +22,7 @@ def notices_batch_distillation_pipeline(notice_ids: List[str], mongodb_client: M for notice_id in notice_ids: notice = notice_repository.get(reference=notice_id) notice.set_distilled_rdf_manifestation( - distilled_rdf_manifestation=notice.rdf_manifestation.copy()) + distilled_rdf_manifestation=notice.rdf_manifestation.model_copy()) notices.append(notice) for cet_uri in CET_URIS: deduplicate_entities_by_cet_uri(notices=notices, cet_uri=cet_uri) diff --git a/ted_sws/__init__.py b/ted_sws/__init__.py index 976d2a089..898c8237c 100644 --- a/ted_sws/__init__.py +++ b/ted_sws/__init__.py @@ -9,10 +9,11 @@ __version__ = "0.0.1" +import base64 import json import os import pathlib -import base64 + import dotenv from ted_sws.core.adapters.config_resolver import EnvConfigResolver, AirflowAndEnvConfigResolver, env_property diff --git a/ted_sws/alignment_oracle/services/limes_configurator.py b/ted_sws/alignment_oracle/services/limes_configurator.py index 5524991a4..8884077c8 100644 --- a/ted_sws/alignment_oracle/services/limes_configurator.py +++ b/ted_sws/alignment_oracle/services/limes_configurator.py @@ -28,7 +28,7 @@ def generate_xml_config_from_limes_config(limes_config_params: LimesConfigParams :param limes_config_params: :return: """ - return TEMPLATES.get_template(LIMES_CONFIG_TEMPLATE).render(limes_config_params.dict()) + return TEMPLATES.get_template(LIMES_CONFIG_TEMPLATE).render(limes_config_params.model_dump()) def generate_default_limes_config_params(source_sparql_endpoint: str, diff --git a/ted_sws/data_manager/adapters/daily_notices_metadata_repository.py b/ted_sws/data_manager/adapters/daily_notices_metadata_repository.py index 2a81068c7..0061b60a8 100644 --- a/ted_sws/data_manager/adapters/daily_notices_metadata_repository.py +++ b/ted_sws/data_manager/adapters/daily_notices_metadata_repository.py @@ -1,11 +1,10 @@ -from datetime import datetime, time, date +from datetime import datetime, date from typing import Iterator, Optional, List from pymongo import MongoClient, ASCENDING from ted_sws import config from ted_sws.core.model.supra_notice import DailyNoticesMetadata -from ted_sws.data_manager.adapters import inject_date_string_fields from ted_sws.data_manager.adapters.repository_abc import DailyNoticesMetadataRepositoryABC DAILY_NOTICES_METADATA_AGGREGATION_DATE = "aggregation_date" diff --git a/ted_sws/data_manager/adapters/manifestation_repository.py b/ted_sws/data_manager/adapters/manifestation_repository.py index e437d095d..76a106539 100644 --- a/ted_sws/data_manager/adapters/manifestation_repository.py +++ b/ted_sws/data_manager/adapters/manifestation_repository.py @@ -60,7 +60,7 @@ def _update_manifestation(self, reference: str, manifestation: Manifestation, up :return: """ if manifestation is not None: - manifestation_dict = manifestation.dict() + manifestation_dict = manifestation.model_dump() manifestation_dict[AGGREGATE_REFERENCE_ID] = reference manifestation_dict[MANIFESTATION_TYPE_ID] = self._manifestation_type reference = self._build_reference(base_reference=reference) diff --git a/ted_sws/data_manager/adapters/mapping_suite_repository.py b/ted_sws/data_manager/adapters/mapping_suite_repository.py index f290bb44d..355f3723e 100644 --- a/ted_sws/data_manager/adapters/mapping_suite_repository.py +++ b/ted_sws/data_manager/adapters/mapping_suite_repository.py @@ -53,7 +53,7 @@ def _create_dict_from_mapping_suite(self, mapping_suite: MappingSuite) -> dict: :param mapping_suite: :return: """ - mapping_suite_dict = mapping_suite.dict() + mapping_suite_dict = mapping_suite.model_dump() mapping_suite_dict[MONGODB_COLLECTION_ID] = mapping_suite.get_mongodb_id() mapping_suite_dict[MS_CREATED_AT] = datetime.fromisoformat(mapping_suite_dict[MS_CREATED_AT]) inject_date_string_fields(data=mapping_suite_dict, date_field_name=MS_CREATED_AT) @@ -189,7 +189,7 @@ def _write_package_metadata(self, mapping_suite: MappingSuite): package_path = self.repository_path / mapping_suite.identifier package_path.mkdir(parents=True, exist_ok=True) metadata_path = package_path / MS_METADATA_FILE_NAME - package_metadata = mapping_suite.dict() + package_metadata = mapping_suite.model_dump() [package_metadata.pop(key, None) for key in ["transformation_rule_set", "shacl_test_suites", "sparql_test_suites"]] with metadata_path.open("w", encoding="utf-8") as f: diff --git a/ted_sws/data_manager/adapters/metadata_repository.py b/ted_sws/data_manager/adapters/metadata_repository.py index 37c0faba0..c95806d10 100644 --- a/ted_sws/data_manager/adapters/metadata_repository.py +++ b/ted_sws/data_manager/adapters/metadata_repository.py @@ -37,7 +37,7 @@ def _update_metadata(self, reference: str, metadata: Metadata, upsert: bool = Fa :return: """ if metadata is not None: - metadata_dict = metadata.dict() + metadata_dict = metadata.model_dump() metadata_dict[AGGREGATE_REFERENCE_ID] = reference metadata_dict[METADATA_TYPE_ID] = self._metadata_type reference = self._build_reference(base_reference=reference) diff --git a/ted_sws/data_manager/adapters/notice_repository.py b/ted_sws/data_manager/adapters/notice_repository.py index f7232c7a6..de9ba4706 100644 --- a/ted_sws/data_manager/adapters/notice_repository.py +++ b/ted_sws/data_manager/adapters/notice_repository.py @@ -95,7 +95,7 @@ def add(self, notice: Notice): :return: """ notice_file_path = self.repository_path / f"{notice.ted_id}.json" - notice_dict = notice.dict() + notice_dict = notice.model_dump() notice_dict[NOTICE_STATUS] = str(notice_dict[NOTICE_STATUS]) notice_file_path.write_text(data=json.dumps(notice_dict), encoding="utf-8") @@ -243,14 +243,14 @@ def _create_dict_from_notice(notice: Notice) -> dict: :return: """ - notice_dict = notice.dict(include={NOTICE_TED_ID: True, NOTICE_STATUS: True, - NOTICE_CREATED_AT: True, VALIDATION_SUMMARY: True}) + notice_dict = notice.model_dump(include={NOTICE_TED_ID: True, NOTICE_STATUS: True, + NOTICE_CREATED_AT: True, VALIDATION_SUMMARY: True}) notice_dict[MONGODB_COLLECTION_ID] = notice_dict[NOTICE_TED_ID] notice_dict[NOTICE_STATUS] = str(notice_dict[NOTICE_STATUS]) notice_dict[NOTICE_CREATED_AT] = datetime.fromisoformat(notice_dict[NOTICE_CREATED_AT]) if notice._normalised_metadata: - normalised_metadata_dict = create_normalised_metadata_view(notice._normalised_metadata).dict() + normalised_metadata_dict = create_normalised_metadata_view(notice._normalised_metadata).model_dump() if normalised_metadata_dict[METADATA_PUBLICATION_DATE]: normalised_metadata_dict[METADATA_PUBLICATION_DATE] = datetime.fromisoformat( normalised_metadata_dict[METADATA_PUBLICATION_DATE]) diff --git a/ted_sws/data_manager/adapters/supra_notice_repository.py b/ted_sws/data_manager/adapters/supra_notice_repository.py index 6614eb9ea..ee28f0ad2 100644 --- a/ted_sws/data_manager/adapters/supra_notice_repository.py +++ b/ted_sws/data_manager/adapters/supra_notice_repository.py @@ -35,7 +35,7 @@ def _create_dict_from_daily_supra_notice(self, daily_supra_notice: DailySupraNot :param daily_supra_notice: :return: """ - daily_supra_notice_dict = daily_supra_notice.dict() + daily_supra_notice_dict = daily_supra_notice.model_dump() daily_supra_notice_dict[DAILY_SUPRA_NOTICE_TED_PUBLICATION_DATE] = datetime.combine( daily_supra_notice_dict[DAILY_SUPRA_NOTICE_TED_PUBLICATION_DATE], time()) daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID] = daily_supra_notice_dict[ @@ -57,7 +57,7 @@ def _create_daily_supra_notice_from_dict(self, daily_supra_notice_dict: dict) -> remove_date_string_fields(data=daily_supra_notice_dict, date_field_name=DAILY_SUPRA_NOTICE_TED_PUBLICATION_DATE) remove_date_string_fields(data=daily_supra_notice_dict, date_field_name=DAILY_SUPRA_NOTICE_CREATED_AT) - daily_supra_notice = DailySupraNotice.parse_obj(daily_supra_notice_dict) + daily_supra_notice = DailySupraNotice.model_validate(daily_supra_notice_dict) return daily_supra_notice return None diff --git a/ted_sws/data_manager/services/export_notice_from_mongodb.py b/ted_sws/data_manager/services/export_notice_from_mongodb.py index 31fa0639e..45d458929 100644 --- a/ted_sws/data_manager/services/export_notice_from_mongodb.py +++ b/ted_sws/data_manager/services/export_notice_from_mongodb.py @@ -40,7 +40,7 @@ def write_in_file(data: Union[str, bytes], terminal_path: str): if notice.validation_summary: write_in_file(notice.validation_summary.object_data, "validation_summary.html") if notice.xml_metadata: - write_in_file(json.dumps(notice.xml_metadata.dict()), "xml_metadata.json") + write_in_file(json.dumps(notice.xml_metadata.model_dump()), "xml_metadata.json") if notice.mets_manifestation and notice.mets_manifestation.object_data: mets_package_file_name = "mets_manifestation.zip" @@ -54,14 +54,14 @@ def write_in_file(data: Union[str, bytes], terminal_path: str): if notice.rdf_manifestation and notice.rdf_manifestation.shacl_validations: for shacl_validation in notice.rdf_manifestation.shacl_validations: write_in_file(shacl_validation.object_data, "shacl_validation.html") - shacl_validation_json = json.dumps(shacl_validation.validation_results.dict()) + shacl_validation_json = json.dumps(shacl_validation.validation_results.model_dump()) write_in_file(shacl_validation_json, "shacl_validation.json") if notice.rdf_manifestation and notice.rdf_manifestation.sparql_validations: for sparql_validation in notice.rdf_manifestation.sparql_validations: write_in_file(sparql_validation.object_data, "sparql_validation.html") sparql_validation_json = json.dumps( - [validation_result.dict() for validation_result in sparql_validation.validation_results]) + [validation_result.model_dump() for validation_result in sparql_validation.validation_results]) write_in_file(sparql_validation_json, "sparql_validation.json") diff --git a/ted_sws/data_sampler/services/notice_xml_indexer.py b/ted_sws/data_sampler/services/notice_xml_indexer.py index 23ac3dac4..322905c8d 100644 --- a/ted_sws/data_sampler/services/notice_xml_indexer.py +++ b/ted_sws/data_sampler/services/notice_xml_indexer.py @@ -1,5 +1,7 @@ import pathlib +import re import tempfile +import xml.etree.ElementTree as XMLElementTree from typing import List from pymongo import MongoClient @@ -10,8 +12,6 @@ from ted_sws.data_manager.adapters.notice_repository import NoticeRepository from ted_sws.mapping_suite_processor.adapters.conceptual_mapping_reader import ConceptualMappingReader from ted_sws.resources import XSLT_FILES_PATH -import xml.etree.ElementTree as XMLElementTree -import re UNIQUE_XPATHS_XSLT_FILE_PATH = "get_unique_xpaths.xsl" XSLT_PREFIX_RESULT = "" diff --git a/ted_sws/event_manager/adapters/log/__init__.py b/ted_sws/event_manager/adapters/log/__init__.py index f8d19d36d..db8bfed79 100644 --- a/ted_sws/event_manager/adapters/log/__init__.py +++ b/ted_sws/event_manager/adapters/log/__init__.py @@ -2,10 +2,10 @@ import os from enum import Enum -from ted_sws import RUN_ENV_NAME, RUN_TEST_ENV_VAL - from colorama import Fore +from ted_sws import RUN_ENV_NAME, RUN_TEST_ENV_VAL + LOG_ERROR_TEXT = Fore.RED + "{}" + Fore.RESET LOG_SUCCESS_TEXT = Fore.GREEN + "{}" + Fore.RESET LOG_INFO_TEXT = Fore.CYAN + "{}" + Fore.RESET diff --git a/ted_sws/event_manager/services/log.py b/ted_sws/event_manager/services/log.py index 45b31709c..06592f4b6 100644 --- a/ted_sws/event_manager/services/log.py +++ b/ted_sws/event_manager/services/log.py @@ -1,6 +1,6 @@ +from ted_sws.event_manager.adapters.event_logger import EventMessageLogSettings from ted_sws.event_manager.model.event_message import EventMessage, TechnicalEventMessage, NoticeEventMessage, \ MappingSuiteEventMessage -from ted_sws.event_manager.adapters.event_logger import EventMessageLogSettings from ted_sws.event_manager.services.logger_from_context import get_logger, get_cli_logger diff --git a/ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_metadata.py b/ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_metadata.py index 56432cd06..bfbd2d85d 100644 --- a/ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_metadata.py +++ b/ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_metadata.py @@ -6,10 +6,10 @@ from ted_sws.data_manager.adapters.mapping_suite_repository import MS_TRANSFORM_FOLDER_NAME, MS_METADATA_FILE_NAME, \ MS_CONCEPTUAL_MAPPING_FILE_NAME -from ted_sws.mapping_suite_processor.adapters.mapping_suite_hasher import MappingSuiteHasher from ted_sws.mapping_suite_processor.adapters.conceptual_mapping_reader import IDENTIFIER_FIELD, TITLE_FIELD, \ DESCRIPTION_FIELD, VERSION_FIELD, EPO_VERSION_FIELD, E_FORMS_SUBTYPE_FIELD, START_DATE_FIELD, END_DATE_FIELD, \ MIN_XSD_VERSION_FIELD, MAX_XSD_VERSION_FIELD +from ted_sws.mapping_suite_processor.adapters.mapping_suite_hasher import MappingSuiteHasher from ted_sws.mapping_suite_processor.services.conceptual_mapping_reader import mapping_suite_read_metadata # This set of constants refers to keys in metadata.json corresponding to the fields Conceptual Mapping file diff --git a/ted_sws/mapping_suite_processor/services/mapping_suite_digest_service.py b/ted_sws/mapping_suite_processor/services/mapping_suite_digest_service.py index eb2f30387..78d07635d 100644 --- a/ted_sws/mapping_suite_processor/services/mapping_suite_digest_service.py +++ b/ted_sws/mapping_suite_processor/services/mapping_suite_digest_service.py @@ -1,6 +1,7 @@ -from ted_sws.core.model.transform import MappingSuite from urllib.parse import urlparse + from ted_sws import config +from ted_sws.core.model.transform import MappingSuite def update_digest_api_address_for_mapping_suite(mapping_suite: MappingSuite, diff --git a/ted_sws/master_data_registry/services/entity_deduplication.py b/ted_sws/master_data_registry/services/entity_deduplication.py index 3c13b1fbe..925673991 100644 --- a/ted_sws/master_data_registry/services/entity_deduplication.py +++ b/ted_sws/master_data_registry/services/entity_deduplication.py @@ -1,11 +1,12 @@ import pathlib import tempfile +from collections import defaultdict from io import StringIO from typing import List, Tuple, Dict + import rdflib from pymongo import MongoClient from rdflib import RDF, URIRef, OWL -from collections import defaultdict from ted_sws.alignment_oracle.services.generate_alignment_links import generate_alignment_links, TURTLE_SOURCE_DATA_TYPE from ted_sws.alignment_oracle.services.limes_config_resolver import get_limes_config_generator_by_cet_uri diff --git a/ted_sws/notice_metadata_processor/services/metadata_normalizer.py b/ted_sws/notice_metadata_processor/services/metadata_normalizer.py index 47bb08c72..6e39ee2ab 100644 --- a/ted_sws/notice_metadata_processor/services/metadata_normalizer.py +++ b/ted_sws/notice_metadata_processor/services/metadata_normalizer.py @@ -86,8 +86,8 @@ def create_normalised_metadata_view(normalised_metadata: NormalisedMetadata) -> english_name_of_buyer = [name.text for name in normalised_metadata.name_of_buyer if name.language == ENGLISH_LANGUAGE_TAG] name_of_buyer = "no_english_city_of_buyer" if len(english_name_of_buyer) == 0 else english_name_of_buyer[0] - normalised_metadata_dict = normalised_metadata.dict(exclude={TITLE_KEY:True, LONG_TITLE_KEY:True, - BUYER_NAME_KEY:True, BUYER_CITY_KEY:True}) + normalised_metadata_dict = normalised_metadata.model_dump(exclude={TITLE_KEY: True, LONG_TITLE_KEY: True, + BUYER_NAME_KEY: True, BUYER_CITY_KEY: True}) return NormalisedMetadataView(title=title, long_title=long_title, name_of_buyer=name_of_buyer, city_of_buyer=city_of_buyer, **normalised_metadata_dict) diff --git a/ted_sws/notice_packager/adapters/template_generator.py b/ted_sws/notice_packager/adapters/template_generator.py index 9053f11f0..fda549795 100644 --- a/ted_sws/notice_packager/adapters/template_generator.py +++ b/ted_sws/notice_packager/adapters/template_generator.py @@ -19,7 +19,7 @@ class TemplateGenerator: @classmethod def __generate_template(cls, template, data: PackagerMetadata = None) -> str: - template_render = TEMPLATES.get_template(template).render(data.dict()) + template_render = TEMPLATES.get_template(template).render(data.model_dump()) return template_render @classmethod diff --git a/ted_sws/notice_packager/model/metadata.py b/ted_sws/notice_packager/model/metadata.py index a6ccd0603..51958ffa7 100644 --- a/ted_sws/notice_packager/model/metadata.py +++ b/ted_sws/notice_packager/model/metadata.py @@ -12,7 +12,7 @@ import datetime from typing import List, Dict, Optional -from pydantic import validator +from pydantic import field_validator from ted_sws.core.model.metadata import Metadata @@ -95,7 +95,7 @@ class MetsMetadata(Metadata): notice_file_checksum: Optional[str] = None notice_file_checksum_type: Optional[str] = METS_NOTICE_FILE_CHECKSUM_TYPE - @validator('type') + @field_validator('type') def validate_notice_action_type(cls, action_type): validate_mets_type(action_type) return action_type diff --git a/ted_sws/notice_publisher/services/notice_publisher.py b/ted_sws/notice_publisher/services/notice_publisher.py index d89a99b8f..28ef40695 100644 --- a/ted_sws/notice_publisher/services/notice_publisher.py +++ b/ted_sws/notice_publisher/services/notice_publisher.py @@ -6,7 +6,6 @@ from ted_sws.core.model.manifestation import RDFManifestation from ted_sws.core.model.notice import Notice, NoticeStatus from ted_sws.data_manager.adapters.notice_repository import NoticeRepositoryABC -from ted_sws.notice_packager import DEFAULT_NOTICE_PACKAGE_EXTENSION from ted_sws.notice_publisher.adapters.s3_notice_publisher import S3Publisher, DEFAULT_S3_RDF_CONTENT_TYPE from ted_sws.notice_publisher.adapters.sftp_notice_publisher import SFTPPublisher from ted_sws.notice_publisher.adapters.sftp_publisher_abc import SFTPPublisherABC diff --git a/ted_sws/notice_transformer/adapters/notice_batch_transformer.py b/ted_sws/notice_transformer/adapters/notice_batch_transformer.py index 3b565c85e..cb4b1df81 100644 --- a/ted_sws/notice_transformer/adapters/notice_batch_transformer.py +++ b/ted_sws/notice_transformer/adapters/notice_batch_transformer.py @@ -6,6 +6,7 @@ from pathlib import Path from threading import Lock from typing import Optional + from pymongo import MongoClient from ted_sws import config diff --git a/ted_sws/notice_transformer/entrypoints/api/digest_service/main.py b/ted_sws/notice_transformer/entrypoints/api/digest_service/main.py index 064cc9957..51b97d24b 100644 --- a/ted_sws/notice_transformer/entrypoints/api/digest_service/main.py +++ b/ted_sws/notice_transformer/entrypoints/api/digest_service/main.py @@ -2,7 +2,8 @@ from ted_sws.notice_transformer.entrypoints.api.digest_service.routes.hashing import route as hashing_route, \ ROUTE_PREFIX as HASHING_ROUTE_PREFIX -from ted_sws.notice_transformer.entrypoints.api.digest_service.routes.xpath import route as xpath_route, ROUTE_PREFIX as XPATH_ROUTE_PREFIX +from ted_sws.notice_transformer.entrypoints.api.digest_service.routes.xpath import route as xpath_route, \ + ROUTE_PREFIX as XPATH_ROUTE_PREFIX API_VERSION = "1" diff --git a/ted_sws/notice_transformer/entrypoints/api/digest_service/routes/hashing.py b/ted_sws/notice_transformer/entrypoints/api/digest_service/routes/hashing.py index 7ec932118..2237a4f0f 100644 --- a/ted_sws/notice_transformer/entrypoints/api/digest_service/routes/hashing.py +++ b/ted_sws/notice_transformer/entrypoints/api/digest_service/routes/hashing.py @@ -2,6 +2,7 @@ import uuid from enum import Enum from typing import Optional + import shortuuid from fastapi import APIRouter diff --git a/ted_sws/notice_transformer/entrypoints/api/digest_service/server.py b/ted_sws/notice_transformer/entrypoints/api/digest_service/server.py index b8d057b90..e3e84c36c 100644 --- a/ted_sws/notice_transformer/entrypoints/api/digest_service/server.py +++ b/ted_sws/notice_transformer/entrypoints/api/digest_service/server.py @@ -3,11 +3,11 @@ import click from ted_sws import config -from ted_sws.notice_transformer.entrypoints.api.digest_service.main import API_PREFIX from ted_sws.event_manager.adapters.event_handler_config import ConsoleLoggerConfig from ted_sws.event_manager.adapters.event_logger import EventLogger from ted_sws.event_manager.model.event_message import EventMessage from ted_sws.event_manager.services.logger_from_context import get_env_logger +from ted_sws.notice_transformer.entrypoints.api.digest_service.main import API_PREFIX API_HOST: str = config.ID_MANAGER_PROD_API_HOST API_PORT: int = config.ID_MANAGER_API_PORT diff --git a/ted_sws/notice_validator/adapters/xpath_coverage_runner.py b/ted_sws/notice_validator/adapters/xpath_coverage_runner.py index 6ce93e551..01d8ad379 100644 --- a/ted_sws/notice_validator/adapters/xpath_coverage_runner.py +++ b/ted_sws/notice_validator/adapters/xpath_coverage_runner.py @@ -144,7 +144,7 @@ def xpath_coverage_validation_summary_report(self, @classmethod def json_report(cls, report: XPATHCoverageValidationReport) -> dict: - return report.dict() + return report.model_dump() @classmethod def html_report(cls, report: XPATHCoverageValidationReport, metadata: dict = None) -> str: diff --git a/ted_sws/notice_validator/services/check_availability_of_notice_in_cellar.py b/ted_sws/notice_validator/services/check_availability_of_notice_in_cellar.py index c76e775a7..d92295243 100644 --- a/ted_sws/notice_validator/services/check_availability_of_notice_in_cellar.py +++ b/ted_sws/notice_validator/services/check_availability_of_notice_in_cellar.py @@ -1,5 +1,6 @@ import time from typing import List, Set + from pymongo import MongoClient from ted_sws import config diff --git a/ted_sws/notice_validator/services/shacl_test_suite_runner.py b/ted_sws/notice_validator/services/shacl_test_suite_runner.py index 7108047c5..e639f01b3 100644 --- a/ted_sws/notice_validator/services/shacl_test_suite_runner.py +++ b/ted_sws/notice_validator/services/shacl_test_suite_runner.py @@ -75,7 +75,7 @@ def generate_shacl_report(shacl_test_suite_execution: SHACLTestSuiteValidationRe :return: """ if with_html: - template_data: dict = shacl_test_suite_execution.dict() + template_data: dict = shacl_test_suite_execution.model_dump() template_data[NOTICE_IDS_FIELD] = notice_ids html_report = TEMPLATES.get_template(SHACL_TEST_SUITE_EXECUTION_HTML_REPORT_TEMPLATE).render(template_data) shacl_test_suite_execution.object_data = html_report @@ -165,7 +165,7 @@ def generate_shacl_validation_summary_report(report_notices: List[ReportNotice], report.mapping_suite_ids = list(set(report.mapping_suite_ids)) if with_html: - template_data: dict = report.dict() + template_data: dict = report.model_dump() template_data[TEMPLATE_METADATA_KEY] = metadata html_report = TEMPLATES.get_template(SHACL_SUMMARY_HTML_REPORT_TEMPLATE).render(template_data) report.object_data = html_report diff --git a/ted_sws/notice_validator/services/sparql_test_suite_runner.py b/ted_sws/notice_validator/services/sparql_test_suite_runner.py index d3124746a..8e48ca675 100644 --- a/ted_sws/notice_validator/services/sparql_test_suite_runner.py +++ b/ted_sws/notice_validator/services/sparql_test_suite_runner.py @@ -154,7 +154,7 @@ def __init__(self, sparql_test_suite_execution: SPARQLTestSuiteValidationReport, def generate_report(self) -> SPARQLTestSuiteValidationReport: if self.with_html: - template_data: dict = self.sparql_test_suite_execution.dict() + template_data: dict = self.sparql_test_suite_execution.model_dump() template_data[NOTICE_IDS_FIELD] = self.notice_ids html_report = TEMPLATES.get_template(SPARQL_TEST_SUITE_EXECUTION_HTML_REPORT_TEMPLATE).render(template_data) self.sparql_test_suite_execution.object_data = html_report @@ -187,7 +187,7 @@ def process_sparql_validation_summary_report_data_with_notice( else: validation_query_result = SPARQLValidationSummaryQueryResult( test_suite_identifier=test_suite_id, - **validation.dict() + **validation.model_dump() ) notice_data: ReportPackageNoticeData = ReportPackageNoticeData( notice_id=notice.ted_id, @@ -221,7 +221,7 @@ def add_sparql_validation_summary_html_report( report: SPARQLValidationSummaryReport, metadata: dict = None ): - template_data: dict = report.dict() + template_data: dict = report.model_dump() template_data[TEMPLATE_METADATA_KEY] = metadata html_report = TEMPLATES.get_template(SPARQL_SUMMARY_HTML_REPORT_TEMPLATE).render(template_data) report.object_data = html_report diff --git a/tests/clean_mongo_db.py b/tests/clean_mongo_db.py index 2b05e8043..d315a2cf2 100644 --- a/tests/clean_mongo_db.py +++ b/tests/clean_mongo_db.py @@ -1,7 +1,8 @@ +from logging import getLogger + from pymongo import MongoClient from ted_sws import config -from logging import getLogger logger = getLogger(__name__) diff --git a/tests/conftest.py b/tests/conftest.py index eff7320dd..511aef47a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,13 +12,13 @@ from ted_sws.core.model.metadata import TEDMetadata, LanguageTaggedString, NormalisedMetadata, XMLMetadata from ted_sws.core.model.notice import Notice, NoticeStatus from ted_sws.data_manager.adapters.notice_repository import NoticeRepositoryInFileSystem +from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter +from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher from ted_sws.notice_metadata_processor.services.metadata_normalizer import TITLE_KEY, LONG_TITLE_KEY, NOTICE_TYPE_KEY, \ NOTICE_NUMBER_KEY, OJS_TYPE_KEY, OJS_NUMBER_KEY, LANGUAGE_KEY, EU_INSTITUTION_KEY, SENT_DATE_KEY, DEADLINE_DATE_KEY, \ BUYER_COUNTRY_KEY, BUYER_NAME_KEY, BUYER_CITY_KEY, PUBLICATION_DATE_KEY, FORM_NUMBER_KEY, \ FORM_TYPE_KEY, LEGAL_BASIS_DIRECTIVE_KEY, EXTRACTED_LEGAL_BASIS_KEY, \ PLACE_OF_PERFORMANCE_KEY, E_FORMS_SUBTYPE_KEY, XSD_VERSION_KEY -from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter -from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher from tests import TEST_DATA_PATH from tests.fakes.fake_repository import FakeNoticeRepository from tests.fakes.fake_ted_api import FakeRequestAPI diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 4499a05bd..9198b1c81 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -8,7 +8,6 @@ from ted_sws import config from ted_sws.data_manager.adapters.notice_repository import NoticeRepository from ted_sws.data_manager.adapters.triple_store import AllegroGraphTripleStore, FusekiAdapter - from tests import TEST_DATA_PATH diff --git a/tests/e2e/dags/__init__.py b/tests/e2e/dags/__init__.py index e18be2d5c..77bb21b6c 100644 --- a/tests/e2e/dags/__init__.py +++ b/tests/e2e/dags/__init__.py @@ -1,5 +1,3 @@ -from datetime import timezone - # from airflow.models import DagRun, TaskInstance # from airflow.utils.types import DagRunType diff --git a/tests/e2e/dags/_test_notice_transmisions.py b/tests/e2e/dags/_test_notice_transmisions.py index 6bd1f8975..2136efdb9 100644 --- a/tests/e2e/dags/_test_notice_transmisions.py +++ b/tests/e2e/dags/_test_notice_transmisions.py @@ -6,19 +6,20 @@ # Email: stefan.stratulat1997@gmail.com import pickle -from ted_sws.data_manager.adapters.notice_repository import NoticeRepository from deepdiff import DeepDiff +from ted_sws.data_manager.adapters.notice_repository import NoticeRepository + def test_notice_is_convertible_to_dict(notice_2021): notice = notice_2021 notice_dict = NoticeRepository._create_dict_from_notice(notice) result_notice = NoticeRepository._create_notice_from_repository_result(notice_dict) - assert DeepDiff(notice.dict(), result_notice.dict(), ignore_order=True) == {} + assert DeepDiff(notice.model_dump(), result_notice.model_dump(), ignore_order=True) == {} def test_notice_is_convertible_to_pickle(notice_2021): notice = notice_2021 notice_dump = pickle.dumps(notice) result_notice = pickle.loads(notice_dump) - assert DeepDiff(notice.dict(), result_notice.dict(), ignore_order=True) == {} + assert DeepDiff(notice.model_dump(), result_notice.model_dump(), ignore_order=True) == {} diff --git a/tests/e2e/dags/conftest.py b/tests/e2e/dags/conftest.py index 410b3eef4..c85fc8a86 100644 --- a/tests/e2e/dags/conftest.py +++ b/tests/e2e/dags/conftest.py @@ -1,5 +1,3 @@ -import os - import pytest # from airflow.models import DagBag diff --git a/tests/e2e/data_manager/_test_notice_unpacking_in_file_system.py b/tests/e2e/data_manager/_test_notice_unpacking_in_file_system.py index acaca012a..24d936a5d 100644 --- a/tests/e2e/data_manager/_test_notice_unpacking_in_file_system.py +++ b/tests/e2e/data_manager/_test_notice_unpacking_in_file_system.py @@ -1,11 +1,10 @@ import base64 -import itertools import json import pathlib +import zipfile from typing import Union from pymongo import MongoClient -import zipfile from ted_sws import config from ted_sws.core.model.notice import Notice, NoticeStatus @@ -15,16 +14,16 @@ def unpack_notice(notice: Notice, unpack_path: pathlib.Path): def write_in_file(data: Union[str, bytes], terminal_path: str): write_path = unpack_path / terminal_path - if type(data) == str: + if isinstance(data, str): write_path.write_text(data=data, encoding="utf-8") - elif type(data) == bytes: + elif isinstance(data, bytes): write_path.write_bytes(data) write_in_file(notice.rdf_manifestation.object_data, "rdf_manifestation.ttl") write_in_file(notice.distilled_rdf_manifestation.object_data, "distilled_rdf_manifestation.ttl") write_in_file(notice.xml_manifestation.object_data, "xml_manifestation.xml") write_in_file(notice.validation_summary.object_data, "validation_summary.html") - write_in_file(json.dumps(notice.xml_metadata.dict()), "xml_metadata.json") + write_in_file(json.dumps(notice.xml_metadata.model_dump()), "xml_metadata.json") mets_package_file_name = "mets_manifestation.zip" unpack_mets_package_dir_name = "mets_manifestation" write_in_file(base64.b64decode(notice.mets_manifestation.object_data.encode(encoding="utf-8")), @@ -34,13 +33,13 @@ def write_in_file(data: Union[str, bytes], terminal_path: str): zip_ref.extractall(unpack_path / unpack_mets_package_dir_name) for shacl_validation in notice.rdf_manifestation.shacl_validations: write_in_file(shacl_validation.object_data, "shacl_validation.html") - shacl_validation_json = json.dumps(shacl_validation.validation_results.dict()) + shacl_validation_json = json.dumps(shacl_validation.validation_results.model_dump()) write_in_file(shacl_validation_json, "shacl_validation.json") for sparql_validation in notice.rdf_manifestation.sparql_validations: write_in_file(sparql_validation.object_data, "sparql_validation.html") sparql_validation_json = json.dumps( - [validation_result.dict() for validation_result in sparql_validation.validation_results]) + [validation_result.model_dump() for validation_result in sparql_validation.validation_results]) write_in_file(sparql_validation_json, "sparql_validation.json") diff --git a/tests/e2e/data_manager/_test_supra_notice_unpacking_in_file_system.py b/tests/e2e/data_manager/_test_supra_notice_unpacking_in_file_system.py index 587869438..75dd14a66 100644 --- a/tests/e2e/data_manager/_test_supra_notice_unpacking_in_file_system.py +++ b/tests/e2e/data_manager/_test_supra_notice_unpacking_in_file_system.py @@ -3,6 +3,7 @@ from typing import Union from pymongo import MongoClient + from ted_sws import config from ted_sws.core.model.supra_notice import DailySupraNotice from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository diff --git a/tests/e2e/data_manager/test_fuseki_triple_store.py b/tests/e2e/data_manager/test_fuseki_triple_store.py index 66fb68457..9e0d0c4bc 100644 --- a/tests/e2e/data_manager/test_fuseki_triple_store.py +++ b/tests/e2e/data_manager/test_fuseki_triple_store.py @@ -1,4 +1,5 @@ import pytest + from ted_sws.data_manager.adapters.triple_store import FusekiAdapter, FusekiException from tests import TEST_DATA_PATH diff --git a/tests/e2e/mapping_suite_processor/test_github_mapping_suite_package_downloader.py b/tests/e2e/mapping_suite_processor/test_github_mapping_suite_package_downloader.py index ca7f37aba..f1d1fb7a2 100644 --- a/tests/e2e/mapping_suite_processor/test_github_mapping_suite_package_downloader.py +++ b/tests/e2e/mapping_suite_processor/test_github_mapping_suite_package_downloader.py @@ -1,4 +1,5 @@ import pathlib + from ted_sws import config from ted_sws.mapping_suite_processor.adapters.github_package_downloader import GitHubMappingSuitePackageDownloader from tests.e2e.mapping_suite_processor import MAPPING_SUITE_PACKAGE_NAME diff --git a/tests/e2e/master_data_registry/conftest.py b/tests/e2e/master_data_registry/conftest.py index b29c5870b..076f89182 100644 --- a/tests/e2e/master_data_registry/conftest.py +++ b/tests/e2e/master_data_registry/conftest.py @@ -1,4 +1,5 @@ import pathlib + import pytest import rdflib diff --git a/tests/e2e/master_data_registry/test_entity_deduplication.py b/tests/e2e/master_data_registry/test_entity_deduplication.py index 8005dc0ca..0a2fa830e 100644 --- a/tests/e2e/master_data_registry/test_entity_deduplication.py +++ b/tests/e2e/master_data_registry/test_entity_deduplication.py @@ -2,7 +2,6 @@ from rdflib import OWL from ted_sws.data_manager.adapters.triple_store import FusekiAdapter - from ted_sws.master_data_registry.services.entity_deduplication import deduplicate_entities_by_cet_uri, \ deduplicate_procedure_entities from ted_sws.notice_validator.services.entity_deduplication_validation import \ diff --git a/tests/e2e/master_data_registry/test_rdf_fragment_processor.py b/tests/e2e/master_data_registry/test_rdf_fragment_processor.py index 3c50aee14..f85fc896c 100644 --- a/tests/e2e/master_data_registry/test_rdf_fragment_processor.py +++ b/tests/e2e/master_data_registry/test_rdf_fragment_processor.py @@ -1,5 +1,6 @@ import rdflib from rdflib.compare import similar + from ted_sws.data_manager.adapters.triple_store import FusekiAdapter from ted_sws.master_data_registry.services.rdf_fragment_processor import get_rdf_fragment_by_cet_uri_from_notice, \ write_rdf_fragments_in_triple_store diff --git a/tests/e2e/notice_fetcher/test_notice_fetcher.py b/tests/e2e/notice_fetcher/test_notice_fetcher.py index 084dff044..51602ec8a 100644 --- a/tests/e2e/notice_fetcher/test_notice_fetcher.py +++ b/tests/e2e/notice_fetcher/test_notice_fetcher.py @@ -1,4 +1,5 @@ import datetime + from ted_sws.core.model.notice import Notice, NoticeStatus from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher diff --git a/tests/e2e/notice_publisher/adapters/test_notice_publisher.py b/tests/e2e/notice_publisher/adapters/test_notice_publisher.py index 74a98d2e9..4f35ba3b7 100644 --- a/tests/e2e/notice_publisher/adapters/test_notice_publisher.py +++ b/tests/e2e/notice_publisher/adapters/test_notice_publisher.py @@ -1,11 +1,11 @@ +import ssl import tempfile import pytest from ted_sws import config -from ted_sws.notice_publisher.adapters.sftp_notice_publisher import SFTPPublisher from ted_sws.notice_publisher.adapters.s3_notice_publisher import S3Publisher -import ssl +from ted_sws.notice_publisher.adapters.sftp_notice_publisher import SFTPPublisher def test_sftp_notice_publisher(): diff --git a/tests/e2e/notice_publisher/services/test_notice_publisher.py b/tests/e2e/notice_publisher/services/test_notice_publisher.py index 3a193af06..140c8174d 100644 --- a/tests/e2e/notice_publisher/services/test_notice_publisher.py +++ b/tests/e2e/notice_publisher/services/test_notice_publisher.py @@ -9,7 +9,6 @@ from ted_sws.notice_publisher.services.notice_publisher import publish_notice_by_id, publish_notice, \ publish_notice_into_s3, publish_notice_into_s3_by_id, publish_notice_rdf_into_s3_by_id, publish_notice_rdf_into_s3, \ publish_notice_rdf_content_into_s3 -from ted_sws.notice_packager import DEFAULT_NOTICE_PACKAGE_EXTENSION from ted_sws.notice_transformer.services.notice_transformer import DEFAULT_TRANSFORMATION_FILE_EXTENSION diff --git a/tests/e2e/notice_publisher_triple_store/test_load_transformed_notice_into_triple_store.py b/tests/e2e/notice_publisher_triple_store/test_load_transformed_notice_into_triple_store.py index 1c1b9ac3f..eb8858ebd 100644 --- a/tests/e2e/notice_publisher_triple_store/test_load_transformed_notice_into_triple_store.py +++ b/tests/e2e/notice_publisher_triple_store/test_load_transformed_notice_into_triple_store.py @@ -1,7 +1,8 @@ +import pytest + from ted_sws.core.model.manifestation import RDFManifestation from ted_sws.notice_publisher_triple_store.services.load_transformed_notice_into_triple_store import \ load_rdf_manifestation_into_triple_store -import pytest SPARQL_QUERY_TRIPLES = "select * {?s ?p ?o}" SPARQL_QUERY_GRAPH = "SELECT ?g { GRAPH ?g { ?s ?p ?o } }" diff --git a/tests/fakes/fake_notice.py b/tests/fakes/fake_notice.py index be71f2e98..ad0b1386e 100644 --- a/tests/fakes/fake_notice.py +++ b/tests/fakes/fake_notice.py @@ -1,5 +1,5 @@ -from ted_sws.core.model.notice import Notice from ted_sws.core.model.manifestation import XMLManifestation +from ted_sws.core.model.notice import Notice class FakeNotice(Notice): diff --git a/tests/fakes/fake_repository.py b/tests/fakes/fake_repository.py index b39d0cbc9..feefe7029 100644 --- a/tests/fakes/fake_repository.py +++ b/tests/fakes/fake_repository.py @@ -1,7 +1,7 @@ from typing import Iterator -from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC from ted_sws.core.model.notice import Notice, NoticeStatus +from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC class FakeNoticeRepository(NoticeRepositoryABC): diff --git a/tests/fakes/fake_rml_mapper.py b/tests/fakes/fake_rml_mapper.py index 0153c219e..fc06c7bef 100644 --- a/tests/fakes/fake_rml_mapper.py +++ b/tests/fakes/fake_rml_mapper.py @@ -1,6 +1,6 @@ import pathlib -from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapperABC, SerializationFormat +from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapperABC class FakeRMLMapper(RMLMapperABC): diff --git a/tests/fakes/fake_triple_store.py b/tests/fakes/fake_triple_store.py index 0a9f21989..f4d05f2ae 100644 --- a/tests/fakes/fake_triple_store.py +++ b/tests/fakes/fake_triple_store.py @@ -1,5 +1,6 @@ import pandas as pd import rdflib + from ted_sws.data_manager.adapters.sparql_endpoint import TripleStoreEndpointABC diff --git a/tests/features/master_data_registry/conftest.py b/tests/features/master_data_registry/conftest.py index 397d093bb..0b3737793 100644 --- a/tests/features/master_data_registry/conftest.py +++ b/tests/features/master_data_registry/conftest.py @@ -1,4 +1,5 @@ import pathlib + import pytest import rdflib diff --git a/tests/features/notice_metadata_processor/test_notice_extractor.py b/tests/features/notice_metadata_processor/test_notice_extractor.py index 8d8d34156..3393cff02 100644 --- a/tests/features/notice_metadata_processor/test_notice_extractor.py +++ b/tests/features/notice_metadata_processor/test_notice_extractor.py @@ -1,9 +1,10 @@ from pytest_bdd import scenario, given, when, then, parsers -from ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata -from ted_sws.notice_metadata_processor.services.xml_manifestation_metadata_extractor import XMLManifestationMetadataExtractor from ted_sws.notice_fetcher.adapters.ted_api import TedRequestAPI, TedAPIAdapter from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher +from ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata +from ted_sws.notice_metadata_processor.services.xml_manifestation_metadata_extractor import \ + XMLManifestationMetadataExtractor @scenario('notice_extractor.feature', 'Extracting metadata') @@ -28,6 +29,6 @@ def step_impl(xml_manifestation): @then(parsers.parse("extracted {metadata} is possibly available")) def step_impl(extracted_metadata, notice_identifier, metadata): assert isinstance(extracted_metadata, ExtractedMetadata) - assert extracted_metadata.dict().keys() == ExtractedMetadata.__fields__.keys() - assert notice_identifier == extracted_metadata.dict()["notice_publication_number"] - assert metadata in extracted_metadata.dict() + assert extracted_metadata.model_dump().keys() == ExtractedMetadata.model_fields.keys() + assert notice_identifier == extracted_metadata.model_dump()["notice_publication_number"] + assert metadata in extracted_metadata.model_dump() diff --git a/tests/features/notice_packager/conftest.py b/tests/features/notice_packager/conftest.py index 309d705fe..034714015 100644 --- a/tests/features/notice_packager/conftest.py +++ b/tests/features/notice_packager/conftest.py @@ -1,6 +1,5 @@ import pytest -from ted_sws.core.model.manifestation import RDFManifestation from ted_sws.core.model.notice import NoticeStatus, Notice from tests import TEST_DATA_PATH diff --git a/tests/features/notice_transformer/conftest.py b/tests/features/notice_transformer/conftest.py index db02e3c78..7591c1ad4 100644 --- a/tests/features/notice_transformer/conftest.py +++ b/tests/features/notice_transformer/conftest.py @@ -4,16 +4,11 @@ import pymongo import pytest -from ted_sws import config -from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation, METSManifestation, \ - SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport -from ted_sws.core.model.metadata import TEDMetadata, NormalisedMetadata from ted_sws.core.model.notice import Notice, NoticeStatus from ted_sws.core.model.transform import MappingSuite -from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB, \ - MappingSuiteRepositoryInFileSystem +from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem from ted_sws.data_manager.adapters.notice_repository import NoticeRepository -from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapper, SerializationFormat +from ted_sws.notice_transformer.adapters.rml_mapper import SerializationFormat from tests import TEST_DATA_PATH from tests.fakes.fake_rml_mapper import FakeRMLMapper diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index be26fc2a0..a6e8c0ec6 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,4 +1,5 @@ import pathlib + import pytest from ted_sws import config diff --git a/tests/unit/core/model/test_metadata.py b/tests/unit/core/model/test_metadata.py index a22a2c734..ec16bea68 100644 --- a/tests/unit/core/model/test_metadata.py +++ b/tests/unit/core/model/test_metadata.py @@ -14,7 +14,7 @@ def test_metadata(): metadata = TEDMetadata(**{"AA": ["Value here"], "No_key": "Value"}) assert metadata.AA == ["Value here"] - assert "No_key" not in metadata.dict().keys() + assert "No_key" not in metadata.model_dump().keys() def test_dict_comparison(): diff --git a/tests/unit/core/model/test_notice_creation.py b/tests/unit/core/model/test_notice_creation.py index 20ae67c55..3aedda57e 100644 --- a/tests/unit/core/model/test_notice_creation.py +++ b/tests/unit/core/model/test_notice_creation.py @@ -49,8 +49,8 @@ def test_notice_invalid_creation(): def test_notice_status_validation(indexed_notice): indexed_notice.update_status_to(NoticeStatus.NORMALISED_METADATA) - assert "status" in indexed_notice.dict().keys() - assert "_status" not in indexed_notice.dict().keys() + assert "status" in indexed_notice.model_dump().keys() + assert "_status" not in indexed_notice.model_dump().keys() def test_notice_status_comparison(): diff --git a/tests/unit/core/model/test_notice_rdf_validation.py b/tests/unit/core/model/test_notice_rdf_validation.py index 9c5d57ae4..3c7221c5a 100644 --- a/tests/unit/core/model/test_notice_rdf_validation.py +++ b/tests/unit/core/model/test_notice_rdf_validation.py @@ -8,9 +8,8 @@ """ """ import pytest -from ted_sws.core.model.manifestation import RDFValidationManifestation, SHACLTestSuiteValidationReport, \ +from ted_sws.core.model.manifestation import SHACLTestSuiteValidationReport, \ QueriedSHACLShapeValidationResult -from ted_sws.core.model.notice import NoticeStatus def test_set_notice_rdf_validation(publicly_available_notice, indexed_notice): diff --git a/tests/unit/core/model/test_notice_status_transition.py b/tests/unit/core/model/test_notice_status_transition.py index 968ee128b..d05210cd3 100644 --- a/tests/unit/core/model/test_notice_status_transition.py +++ b/tests/unit/core/model/test_notice_status_transition.py @@ -9,7 +9,7 @@ import pytest -from ted_sws.core.model.notice import Notice, NoticeStatus, UnsupportedStatusTransition +from ted_sws.core.model.notice import NoticeStatus, UnsupportedStatusTransition def test_illegal_status_transitions(indexed_notice): diff --git a/tests/unit/core/model/test_notice_validation.py b/tests/unit/core/model/test_notice_validation.py index d5550bf21..4b8b134a5 100644 --- a/tests/unit/core/model/test_notice_validation.py +++ b/tests/unit/core/model/test_notice_validation.py @@ -8,7 +8,7 @@ """ """ from ted_sws.core.model.manifestation import XPATHCoverageValidationReport, SHACLTestSuiteValidationReport, \ - QueriedSHACLShapeValidationResult, RDFManifestation + QueriedSHACLShapeValidationResult from ted_sws.core.model.notice import NoticeStatus diff --git a/tests/unit/core/model/test_pydantic.py b/tests/unit/core/model/test_pydantic.py index aac9ed659..2d7a2baf7 100644 --- a/tests/unit/core/model/test_pydantic.py +++ b/tests/unit/core/model/test_pydantic.py @@ -50,9 +50,9 @@ def test_pydantic_1(): 'friends': [1, 2, '3'], } - user = User(**external_data) + User(**external_data) - uu = User.construct(external_data_1) + User.model_construct(external_data_1) def test_pydantic_2(): diff --git a/tests/unit/data_manager/conftest.py b/tests/unit/data_manager/conftest.py index 0d14c6310..abb3dd39d 100644 --- a/tests/unit/data_manager/conftest.py +++ b/tests/unit/data_manager/conftest.py @@ -1,5 +1,7 @@ from datetime import date + import pytest + from ted_sws.core.model.supra_notice import DailySupraNotice, DailyNoticesMetadata from ted_sws.core.model.transform import MetadataConstraints, FileResource, TransformationRuleSet, SHACLTestSuite, \ SPARQLTestSuite, MappingSuite, TransformationTestData diff --git a/tests/unit/data_manager/test_mapping_suite_repository.py b/tests/unit/data_manager/test_mapping_suite_repository.py index 14a126b39..e42e85025 100644 --- a/tests/unit/data_manager/test_mapping_suite_repository.py +++ b/tests/unit/data_manager/test_mapping_suite_repository.py @@ -1,4 +1,3 @@ -import pytest from deepdiff import DeepDiff from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryMongoDB, \ diff --git a/tests/unit/data_manager/test_notice_repository.py b/tests/unit/data_manager/test_notice_repository.py index b87b02a9e..e59fb4aca 100644 --- a/tests/unit/data_manager/test_notice_repository.py +++ b/tests/unit/data_manager/test_notice_repository.py @@ -1,8 +1,9 @@ -from ted_sws.data_manager.adapters.notice_repository import NoticeRepository +from mongomock.gridfs import enable_gridfs_integration + from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation from ted_sws.core.model.metadata import TEDMetadata from ted_sws.core.model.notice import Notice, NoticeStatus -from mongomock.gridfs import enable_gridfs_integration +from ted_sws.data_manager.adapters.notice_repository import NoticeRepository enable_gridfs_integration() diff --git a/tests/unit/digest_service/conftest.py b/tests/unit/digest_service/conftest.py index 88e19f8c7..7c823dd28 100644 --- a/tests/unit/digest_service/conftest.py +++ b/tests/unit/digest_service/conftest.py @@ -10,7 +10,8 @@ from ted_sws.notice_transformer.entrypoints.api.digest_service.common import ResponseType from ted_sws.notice_transformer.entrypoints.api.digest_service.main import app -from ted_sws.notice_transformer.entrypoints.api.digest_service.routes.hashing import UUIDInputProcessType, UUIDNamespaceType, UUIDVersion +from ted_sws.notice_transformer.entrypoints.api.digest_service.routes.hashing import UUIDInputProcessType, \ + UUIDNamespaceType, UUIDVersion @pytest.fixture diff --git a/tests/unit/digest_service/test_api_hashing.py b/tests/unit/digest_service/test_api_hashing.py index 25f57ec60..80845a96e 100644 --- a/tests/unit/digest_service/test_api_hashing.py +++ b/tests/unit/digest_service/test_api_hashing.py @@ -1,7 +1,8 @@ import uuid from ted_sws.notice_transformer.entrypoints.api.digest_service.main import API_PREFIX -from ted_sws.notice_transformer.entrypoints.api.digest_service.routes.hashing import string_md5, uuid_ns_by_type, ROUTE_PREFIX +from ted_sws.notice_transformer.entrypoints.api.digest_service.routes.hashing import string_md5, uuid_ns_by_type, \ + ROUTE_PREFIX URL_PREFIX = API_PREFIX + ROUTE_PREFIX diff --git a/tests/unit/mapping_suite_processor/conftest.py b/tests/unit/mapping_suite_processor/conftest.py index 9c447dc65..20aa5b688 100644 --- a/tests/unit/mapping_suite_processor/conftest.py +++ b/tests/unit/mapping_suite_processor/conftest.py @@ -1,5 +1,3 @@ -import mongomock -import pymongo import pytest from ted_sws.core.model.transform import MetadataConstraints, FileResource, TransformationRuleSet, SHACLTestSuite, \ diff --git a/tests/unit/notice_fetcher/test_notice_fetcher.py b/tests/unit/notice_fetcher/test_notice_fetcher.py index 6afa989db..4c9f4de3c 100644 --- a/tests/unit/notice_fetcher/test_notice_fetcher.py +++ b/tests/unit/notice_fetcher/test_notice_fetcher.py @@ -1,4 +1,5 @@ import datetime + from ted_sws.core.model.notice import Notice, NoticeStatus from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher diff --git a/tests/unit/notice_metadata_processor/test_metadata_extractor.py b/tests/unit/notice_metadata_processor/test_metadata_extractor.py index 8c6e69e1e..71a7e931a 100644 --- a/tests/unit/notice_metadata_processor/test_metadata_extractor.py +++ b/tests/unit/notice_metadata_processor/test_metadata_extractor.py @@ -1,16 +1,17 @@ +import xml.etree.ElementTree as ET + from ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata, EncodedValue from ted_sws.notice_metadata_processor.services.xml_manifestation_metadata_extractor import extract_text_from_element, \ extract_attribute_from_element, XMLManifestationMetadataExtractor, extract_code_and_value_from_element -import xml.etree.ElementTree as ET - def test_metadata_extractor(indexed_notice): - metadata_extractor = XMLManifestationMetadataExtractor(xml_manifestation=indexed_notice.xml_manifestation).to_metadata() - extracted_metadata_dict = metadata_extractor.dict() + metadata_extractor = XMLManifestationMetadataExtractor( + xml_manifestation=indexed_notice.xml_manifestation).to_metadata() + extracted_metadata_dict = metadata_extractor.model_dump() assert isinstance(metadata_extractor, ExtractedMetadata) - assert extracted_metadata_dict.keys() == ExtractedMetadata.__fields__.keys() + assert extracted_metadata_dict.keys() == ExtractedMetadata.model_fields.keys() assert "extracted_form_number", "xml_schema" in extracted_metadata_dict.keys() assert "067623-2022" in extracted_metadata_dict["notice_publication_number"] assert "http://publications.europa.eu/resource/schema/ted/R2.0.8/publication TED_EXPORT.xsd" in \ @@ -22,9 +23,9 @@ def test_metadata_extractor_2016(notice_2016): metadata_extractor = XMLManifestationMetadataExtractor( xml_manifestation=notice_2016.xml_manifestation).to_metadata() - extracted_metadata_dict = metadata_extractor.dict() + extracted_metadata_dict = metadata_extractor.model_dump() assert isinstance(metadata_extractor, ExtractedMetadata) - assert extracted_metadata_dict.keys() == ExtractedMetadata.__fields__.keys() + assert extracted_metadata_dict.keys() == ExtractedMetadata.model_fields.keys() assert notice_2016.ted_id in extracted_metadata_dict["notice_publication_number"] @@ -32,9 +33,9 @@ def test_metadata_extractor_2015(notice_2015): metadata_extractor = XMLManifestationMetadataExtractor( xml_manifestation=notice_2015.xml_manifestation).to_metadata() - extracted_metadata_dict = metadata_extractor.dict() + extracted_metadata_dict = metadata_extractor.model_dump() assert isinstance(metadata_extractor, ExtractedMetadata) - assert extracted_metadata_dict.keys() == ExtractedMetadata.__fields__.keys() + assert extracted_metadata_dict.keys() == ExtractedMetadata.model_fields.keys() assert notice_2015.ted_id in extracted_metadata_dict["notice_publication_number"] @@ -42,9 +43,9 @@ def test_metadata_extractor_2018(notice_2018): metadata_extractor = XMLManifestationMetadataExtractor( xml_manifestation=notice_2018.xml_manifestation).to_metadata() - extracted_metadata_dict = metadata_extractor.dict() + extracted_metadata_dict = metadata_extractor.model_dump() assert isinstance(metadata_extractor, ExtractedMetadata) - assert extracted_metadata_dict.keys() == ExtractedMetadata.__fields__.keys() + assert extracted_metadata_dict.keys() == ExtractedMetadata.model_fields.keys() assert notice_2018.ted_id in extracted_metadata_dict["notice_publication_number"] @@ -82,7 +83,8 @@ def test_extract_code_and_value(indexed_notice): assert extracted_data.value == "Services" assert extracted_data.code == "4" - nonexisting_element = doc_root.find("epo:CODED_DATA_SECTION/epo:CODIF_DATA/NC_CONTRACT_NATURE", namespaces=namespace) + nonexisting_element = doc_root.find("epo:CODED_DATA_SECTION/epo:CODIF_DATA/NC_CONTRACT_NATURE", + namespaces=namespace) extracted_data = extract_code_and_value_from_element(element=nonexisting_element) assert extracted_data is None diff --git a/tests/unit/notice_metadata_processor/test_metadata_normaliser.py b/tests/unit/notice_metadata_processor/test_metadata_normaliser.py index 301d63735..08ef0a437 100644 --- a/tests/unit/notice_metadata_processor/test_metadata_normaliser.py +++ b/tests/unit/notice_metadata_processor/test_metadata_normaliser.py @@ -2,12 +2,12 @@ from ted_sws.core.model.notice import NoticeStatus from ted_sws.notice_metadata_processor.services.metadata_constraints import filter_df_by_variables -from ted_sws.resources.mapping_files_registry import MappingFilesRegistry from ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice, normalise_notice_by_id, \ MetadataNormaliser, ExtractedMetadataNormaliser, FORM_NUMBER_KEY, SF_NOTICE_TYPE_KEY, LEGAL_BASIS_KEY, \ DOCUMENT_CODE_KEY from ted_sws.notice_metadata_processor.services.xml_manifestation_metadata_extractor import \ XMLManifestationMetadataExtractor +from ted_sws.resources.mapping_files_registry import MappingFilesRegistry def test_metadata_normaliser_by_notice(indexed_notice): diff --git a/tests/unit/notice_metadata_processor/test_model.py b/tests/unit/notice_metadata_processor/test_model.py index 2f8db491e..f38c25968 100644 --- a/tests/unit/notice_metadata_processor/test_model.py +++ b/tests/unit/notice_metadata_processor/test_model.py @@ -5,8 +5,8 @@ def test_extracted_metadata(notice_id): metadata = ExtractedMetadata(**{"notice_publication_number": notice_id, "No_key": ["Value"]}) assert metadata.notice_publication_number == notice_id - assert "No_key" not in metadata.dict().keys() - assert "country_of_buyer" in metadata.dict().keys() + assert "No_key" not in metadata.model_dump().keys() + assert "country_of_buyer" in metadata.model_dump().keys() def test_dict_extracted_metadata(notice_id): @@ -15,8 +15,8 @@ def test_dict_extracted_metadata(notice_id): "city_of_buyer": [LanguageTaggedString("french", "fr"), LanguageTaggedString("english", "en")] } metadata = ExtractedMetadata(**data_dict) - filtered_metadata = {k: v for k, v in metadata.dict().items() if v is not None} - assert data_dict["city_of_buyer"] == metadata.dict()["city_of_buyer"] + filtered_metadata = {k: v for k, v in metadata.model_dump().items() if v is not None} + assert data_dict["city_of_buyer"] == metadata.model_dump()["city_of_buyer"] assert data_dict["city_of_buyer"][0].text == metadata.city_of_buyer[0].text assert data_dict == filtered_metadata diff --git a/tests/unit/notice_packager/conftest.py b/tests/unit/notice_packager/conftest.py index 124e6efa4..fe32a5d5f 100644 --- a/tests/unit/notice_packager/conftest.py +++ b/tests/unit/notice_packager/conftest.py @@ -13,7 +13,8 @@ import pytest from ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata -from ted_sws.notice_metadata_processor.services.xml_manifestation_metadata_extractor import XMLManifestationMetadataExtractor +from ted_sws.notice_metadata_processor.services.xml_manifestation_metadata_extractor import \ + XMLManifestationMetadataExtractor from ted_sws.notice_packager.model.metadata import PackagerMetadata, NoticeMetadata, WorkMetadata, ExpressionMetadata, \ ManifestationMetadata from tests import TEST_DATA_PATH diff --git a/tests/unit/notice_packager/test_notice_packager.py b/tests/unit/notice_packager/test_notice_packager.py index a3725b859..ce45204fd 100644 --- a/tests/unit/notice_packager/test_notice_packager.py +++ b/tests/unit/notice_packager/test_notice_packager.py @@ -10,7 +10,7 @@ from ted_sws.core.model.manifestation import RDFManifestation from ted_sws.core.model.notice import NoticeStatus from ted_sws.notice_packager.model.metadata import METS_TYPE_CREATE -from ted_sws.notice_packager.services.notice_packager import package_notice, NoticePackager +from ted_sws.notice_packager.services.notice_packager import package_notice def test_notice_packager_with_notice(notice_2018, rdf_content): diff --git a/tests/unit/notice_packager/test_template_generator.py b/tests/unit/notice_packager/test_template_generator.py index f4a3c2b84..330b63fd4 100644 --- a/tests/unit/notice_packager/test_template_generator.py +++ b/tests/unit/notice_packager/test_template_generator.py @@ -6,7 +6,6 @@ # Email: kalean.bl@gmail.com """ """ -import difflib import re import pytest diff --git a/tests/unit/notice_transformer/conftest.py b/tests/unit/notice_transformer/conftest.py index dbac5dc14..de7ad5030 100644 --- a/tests/unit/notice_transformer/conftest.py +++ b/tests/unit/notice_transformer/conftest.py @@ -1,11 +1,13 @@ +from pathlib import Path + import mongomock import pymongo import pytest -from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem from ted_sws.core.model.transform import MappingSuite +from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem from tests import TEST_DATA_PATH -from pathlib import Path + @pytest.fixture def fake_not_mapping_suite_id() -> str: diff --git a/tests/unit/notice_validator/test_shacl_test_suite_runner.py b/tests/unit/notice_validator/test_shacl_test_suite_runner.py index 47a7dffb7..ac5852e9a 100644 --- a/tests/unit/notice_validator/test_shacl_test_suite_runner.py +++ b/tests/unit/notice_validator/test_shacl_test_suite_runner.py @@ -1,7 +1,7 @@ import pytest from ted_sws.core.model.manifestation import RDFManifestation, RDFValidationManifestation, \ - SHACLTestSuiteValidationReport, QueriedSHACLShapeValidationResult + SHACLTestSuiteValidationReport from ted_sws.core.model.notice import NoticeStatus from ted_sws.core.model.validation_report import ReportNotice, SHACLValidationSummaryReport from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem diff --git a/tests/unit/notice_validator/test_validation_summary_runner.py b/tests/unit/notice_validator/test_validation_summary_runner.py index 2004a64e6..e596c935f 100644 --- a/tests/unit/notice_validator/test_validation_summary_runner.py +++ b/tests/unit/notice_validator/test_validation_summary_runner.py @@ -1,9 +1,9 @@ import pytest from ted_sws.core.model.validation_report import ReportNotice +from ted_sws.data_manager.adapters.notice_repository import NoticeRepository from ted_sws.notice_validator.services.validation_summary_runner import validation_summary_report_notice_by_id, \ generate_validation_summary_report_notices -from ted_sws.data_manager.adapters.notice_repository import NoticeRepository def test_validation_summary_runner(fake_validation_notice, mongodb_client): diff --git a/tests/unit/supra_notice_manager/conftest.py b/tests/unit/supra_notice_manager/conftest.py index 014e1d1a8..9c49d1224 100644 --- a/tests/unit/supra_notice_manager/conftest.py +++ b/tests/unit/supra_notice_manager/conftest.py @@ -1,10 +1,10 @@ import pytest -from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository -from tests.fakes.fake_ted_api import FakeRequestAPI -from ted_sws.core.model.notice import Notice from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation +from ted_sws.core.model.notice import Notice +from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository from tests import TEST_DATA_PATH +from tests.fakes.fake_ted_api import FakeRequestAPI @pytest.fixture