diff --git a/ted_data_eu/services/etl_pipelines/ted_data_etl_pipeline.py b/ted_data_eu/services/etl_pipelines/ted_data_etl_pipeline.py index 4a697a7..9c054ab 100644 --- a/ted_data_eu/services/etl_pipelines/ted_data_etl_pipeline.py +++ b/ted_data_eu/services/etl_pipelines/ted_data_etl_pipeline.py @@ -34,6 +34,7 @@ END_DATE_METADATA_FIELD = "end_date" TRIPLE_STORE_ENDPOINT = "notices" TED_NOTICES_LINK = 'https://ted.europa.eu/udl?uri=TED:NOTICE:{notice_id}:TEXT:EN:HTML' +TRIPLE_STORE_ENDPOINT_FIELD = "triple_store_endpoint" PROCEDURE_TYPE_COLUMN_NAME = "procedure_type" WINNER_NUTS_COLUMN_NAME = "winner_nuts" @@ -204,6 +205,7 @@ def extract(self) -> Dict: """ etl_metadata = self.get_metadata() etl_metadata_fields = etl_metadata.keys() + triple_store_endpoint = etl_metadata[TRIPLE_STORE_ENDPOINT_FIELD] if TRIPLE_STORE_ENDPOINT_FIELD in etl_metadata_fields else TRIPLE_STORE_ENDPOINT if START_DATE_METADATA_FIELD in etl_metadata_fields and END_DATE_METADATA_FIELD in etl_metadata_fields: if START_DATE_METADATA_FIELD == END_DATE_METADATA_FIELD: date_range = datetime.strptime(START_DATE_METADATA_FIELD, "\"%Y%m%d\"") @@ -218,7 +220,7 @@ def extract(self) -> Dict: sparql_query_template = Template(config.BQ_PATHS[SPARQL_QUERY_NAME].read_text(encoding='utf-8')) sparql_query_str = sparql_query_template.substitute(date_range=date_range) - triple_store_endpoint = GraphDBAdapter().get_sparql_triple_store_endpoint(repository_name=TRIPLE_STORE_ENDPOINT) + triple_store_endpoint = GraphDBAdapter().get_sparql_triple_store_endpoint(repository_name=triple_store_endpoint) result_table = triple_store_endpoint.with_query(sparql_query_str).fetch_tabular() return {"data": result_table} diff --git a/tests/e2e/test_ted_etl.py b/tests/e2e/test_ted_etl.py index 77504ea..c9d8fab 100644 --- a/tests/e2e/test_ted_etl.py +++ b/tests/e2e/test_ted_etl.py @@ -7,7 +7,7 @@ CONTRACT_VALUE_AVAILABLE_INDICATOR, PROCEDURE_TYPE_INDICATOR, PRODUCT_CODES_AVAILABLE_INDICATOR, LOT_NUTS_0, \ LOT_NUTS_1, LOT_NUTS_2, LOT_NUTS_3, get_country_name_by_code, BUYER_NUTS_COLUMN_NAME, PROCEDURE_ID_COLUMN_NAME, \ PROCEDURE_DESCRIPTION_COLUMN_NAME, PROCEDURE_COLUMN_NAME, TDA_FREE_INDEX_NAME, TDA_STARTER_INDEX_NAME, CPV_RANK_4, \ - CPV_RANK_2, CPV_RANK_1, CPV_RANK_3, LOT_COUNTRY + CPV_RANK_2, CPV_RANK_1, CPV_RANK_3, LOT_COUNTRY, TRIPLE_STORE_ENDPOINT_FIELD def test_get_country_name_by_code(real_country_code_alpha_2, fake_country_code_alpha_2, real_country_code_alpha_3, @@ -20,6 +20,7 @@ def test_get_country_name_by_code(real_country_code_alpha_2, fake_country_code_a def test_etl_pipeline(ted_data_etl_pipelines, etl_pipeline_config, graphdb_triple_store, example_notices, tmp_repository_name): + etl_pipeline_config[TRIPLE_STORE_ENDPOINT_FIELD] = tmp_repository_name for ted_data_etl_pipeline in ted_data_etl_pipelines: graphdb_repositories = graphdb_triple_store.list_repositories() if tmp_repository_name in graphdb_repositories: