diff --git a/ted_sws/core/model/supra_notice.py b/ted_sws/core/model/supra_notice.py index fa6be44e..6277b2e2 100644 --- a/ted_sws/core/model/supra_notice.py +++ b/ted_sws/core/model/supra_notice.py @@ -61,7 +61,7 @@ class Config: orm_mode = True -NOTICE_STATUSES_DEFAULT_STATS = {notice_status: 0 for notice_status in NoticeStatus} +NOTICE_STATUSES_DEFAULT_STATS = {str(notice_status): 0 for notice_status in NoticeStatus} class DailyNoticesMetadata(DailyNoticesMetadataABC): diff --git a/ted_sws/data_manager/adapters/daily_notices_metadata_repository.py b/ted_sws/data_manager/adapters/daily_notices_metadata_repository.py index ff6d8562..2a81068c 100644 --- a/ted_sws/data_manager/adapters/daily_notices_metadata_repository.py +++ b/ted_sws/data_manager/adapters/daily_notices_metadata_repository.py @@ -101,5 +101,5 @@ def list_daily_notices_metadata_aggregation_date(self) -> List[date]: DAILY_NOTICES_METADATA_ID: 0})) if not daily_notices_metadata_list: return [] - return [datetime.fromisoformat(aggregation_date[DAILY_NOTICES_METADATA_AGGREGATION_DATE]) for aggregation_date + return [datetime.fromisoformat(aggregation_date[DAILY_NOTICES_METADATA_AGGREGATION_DATE]).date() for aggregation_date in daily_notices_metadata_list] diff --git a/ted_sws/data_manager/services/daily_notices_metadata_services.py b/ted_sws/data_manager/services/daily_notices_metadata_services.py index 3c43fd3e..ab66163b 100644 --- a/ted_sws/data_manager/services/daily_notices_metadata_services.py +++ b/ted_sws/data_manager/services/daily_notices_metadata_services.py @@ -1,5 +1,5 @@ from datetime import date, datetime, timedelta -from typing import Optional +from typing import Optional, List from dateutil import rrule from pymongo import MongoClient @@ -48,7 +48,7 @@ } -def generate_list_of_dates_from_date_range(start_date: date, end_date: date) -> Optional[list]: +def generate_list_of_dates_from_date_range(start_date: date, end_date: date) -> Optional[List[date]]: """ Given a date range returns all daily dates in that range :param start_date: @@ -57,7 +57,7 @@ def generate_list_of_dates_from_date_range(start_date: date, end_date: date) -> """ if start_date > end_date: return None - return [dt for dt in rrule.rrule(rrule.DAILY, + return [ dt.date() for dt in rrule.rrule(rrule.DAILY, dtstart=start_date, until=end_date)] @@ -102,7 +102,6 @@ def update_daily_notices_metadata_from_ted(start_date: date = None, def update_daily_notices_metadata_with_fetched_data(start_date: date = None, end_date: date = None, - ted_api: TedAPIAdapter = None, mongo_client: MongoClient = None, daily_notices_metadata_repo: DailyNoticesMetadataRepository = None): """ @@ -115,7 +114,6 @@ def update_daily_notices_metadata_with_fetched_data(start_date: date = None, if start_date > end_date: raise Exception("Start date cannot be greater than end date") - ted_api = ted_api or TedAPIAdapter(TedRequestAPI(), config.TED_API_URL) mongo_client = mongo_client or MongoClient(config.MONGO_DB_AUTH_URL) daily_notices_metadata_repo = daily_notices_metadata_repo or DailyNoticesMetadataRepository(mongo_client) notice_repo = NoticeRepository(mongo_client) @@ -125,27 +123,32 @@ def update_daily_notices_metadata_with_fetched_data(start_date: date = None, for day in date_range: daily_notices_metadata = daily_notices_metadata_repo.get(day) + + mapping_suite_packages = [] + fetched_notice_ids = [] + notice_statuses = {notice_status: 0 for notice_status in daily_notices_metadata.notice_statuses.keys()} + for notice_id in daily_notices_metadata.ted_api_notice_ids: notice: Notice = notice_repo.get(notice_id) + if notice: - notice_statuses = {notice_status: 0 for notice_status in daily_notices_metadata.notice_statuses} - mapping_suite_packages = [] - fetched_notice_ids = [] fetched_notice_ids.append(notice_id) notice_status = notice.status - notice_statuses[notice_status] += 1 + notice_statuses[str(notice_status)] += 1 if notice_status >= NoticeStatus.TRANSFORMED: # Having distilled_rdf_manifestation mapping_suite_id = notice.rdf_manifestation.mapping_suite_id - mapping_suite_packages.append(mapping_suite_id) - for current_notice_status, linked_notice_statuses in NOTICE_STATUS_COVERAGE_DOWNSTREAM_TRANSITION.items(): - current_notice_status = current_notice_status - if notice_statuses[current_notice_status] > 0: - for linked_notice_status in linked_notice_statuses: - linked_notice_status = linked_notice_status - notice_statuses[linked_notice_status] += notice_statuses[current_notice_status] - - daily_notices_metadata.notice_statuses = notice_statuses - daily_notices_metadata.mapping_suite_packages = mapping_suite_packages - daily_notices_metadata.fetched_notice_ids = fetched_notice_ids + if mapping_suite_id not in mapping_suite_packages: + mapping_suite_packages.append(mapping_suite_id) + + for current_notice_status, linked_notice_statuses in NOTICE_STATUS_COVERAGE_DOWNSTREAM_TRANSITION.items(): + current_notice_status = str(current_notice_status) + if notice_statuses[current_notice_status] > 0: + for linked_notice_status in linked_notice_statuses: + linked_notice_status = linked_notice_status + notice_statuses[linked_notice_status] += notice_statuses[current_notice_status] + + daily_notices_metadata.notice_statuses = notice_statuses + daily_notices_metadata.mapping_suite_packages = mapping_suite_packages + daily_notices_metadata.fetched_notice_ids = fetched_notice_ids daily_notices_metadata_repo.update(daily_notices_metadata) diff --git a/tests/unit/data_manager/services/test_notices_metadata_services.py b/tests/unit/data_manager/services/test_notices_metadata_services.py index 1ece797f..17a45b2c 100644 --- a/tests/unit/data_manager/services/test_notices_metadata_services.py +++ b/tests/unit/data_manager/services/test_notices_metadata_services.py @@ -1,8 +1,11 @@ from datetime import date +from pymongo import MongoClient + from ted_sws import config from ted_sws.data_manager.adapters.daily_notices_metadata_repository import DailyNoticesMetadataRepository -from ted_sws.data_manager.services.daily_notices_metadata_services import update_daily_notices_metadata_from_ted +from ted_sws.data_manager.services.daily_notices_metadata_services import update_daily_notices_metadata_from_ted, \ + update_daily_notices_metadata_with_fetched_data from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter, TedRequestAPI @@ -24,4 +27,18 @@ def test_update_daily_notices_metadata_from_ted(mongodb_client): # end_date=date(2021, 1, 7), # ted_api=ted_api, # mongo_client=mongodb_client, - # daily_notices_metadata_repo=daily_notices_metadata_repo) \ No newline at end of file + # daily_notices_metadata_repo=daily_notices_metadata_repo) + + +def test_update_daily_notices_metadata_with_fetched_data(): #mongodb_client + mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL) + + ted_api = TedAPIAdapter(TedRequestAPI(), config.TED_API_URL) + + + daily_notices_metadata_repo = DailyNoticesMetadataRepository(mongodb_client) + + update_daily_notices_metadata_with_fetched_data(start_date=date(2021, 1, 7), + end_date=date(2021, 1, 7), + mongo_client=mongodb_client, + daily_notices_metadata_repo=daily_notices_metadata_repo) \ No newline at end of file