fix tests

ted_sws/notice_fetcher/adapters/ted_api.py

@@ -132,17 +132,18 @@ def get_generator_by_query(self, query: dict, result_fields: dict = None) -> Gen
                 documents_content += response_body[RESPONSE_RESULTS]
                 for document_content in documents_content:
                     document_content[DOCUMENT_CONTENT] = self._retrieve_document_content(document_content)
-                    document_id = document_content[DOCUMENT_NOTICE_ID_KEY]
-                    document_id = "0" * (11 - len(document_id)) + document_id
-                    document_content[DOCUMENT_NOTICE_ID_KEY] = document_id
+                    # document_id = document_content[DOCUMENT_NOTICE_ID_KEY]
+                    # document_id = "0" * (11 - len(document_id)) + document_id
+                    # document_content[DOCUMENT_NOTICE_ID_KEY] = document_id
                     del document_content[LINKS_TO_CONTENT_KEY]
+                    print("content exist")
                     yield document_content
         else:
             for document_content in documents_content:
                 document_content[DOCUMENT_CONTENT] = self._retrieve_document_content(document_content)
-                document_id = document_content[DOCUMENT_NOTICE_ID_KEY]
-                document_id = "0" * (11 - len(document_id)) + document_id
-                document_content[DOCUMENT_NOTICE_ID_KEY] = document_id
+                # document_id = document_content[DOCUMENT_NOTICE_ID_KEY]
+                # document_id = "0" * (11 - len(document_id)) + document_id
+                # document_content[DOCUMENT_NOTICE_ID_KEY] = document_id
                 del document_content[LINKS_TO_CONTENT_KEY]
                 yield document_content
 

ted_sws/notice_fetcher/services/notice_fetcher.py

@@ -68,7 +68,10 @@ def _create_notice(self, notice_data: dict) -> Notice:
         :param notice_data:
         :return:
         """
-        xml_manifestation = XMLManifestation(object_data=notice_data["content"])
+        try:
+            xml_manifestation = XMLManifestation(object_data=notice_data["content"])
+        except Exception as e:
+            raise Exception(str(e), notice_data)
         del notice_data["content"]
         ted_id = notice_data["ND"]
         original_metadata = TEDMetadata(**notice_data)

tests/conftest.py

@@ -29,7 +29,7 @@
 
 @pytest.fixture
 def notice_id():
-    return "067623-2022"
+    return "67623-2022"
 
 
 @pytest.fixture

tests/features/conftest.py

@@ -40,7 +40,7 @@ def notice_repository(mongodb_client):
 
 @pytest.fixture
 def f03_notice_2020(notice_repository, ted_api_end_point):
-    notice_search_query = {"q": "ND=[408313-2020]"}
+    notice_search_query = {"query": "ND=408313-2020"}
     NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
                                                 ted_api_url=ted_api_end_point)).fetch_notices_by_query(
@@ -52,12 +52,12 @@ def f03_notice_2020(notice_repository, ted_api_end_point):
 
 @pytest.fixture
 def f18_notice_2022(notice_repository, ted_api_end_point):
-    notice_search_query = {"q": "ND=[067623-2022]"}
+    notice_search_query = {"query": "ND=67623-2022"}
     NoticeFetcher(notice_repository=notice_repository,
                   ted_api_adapter=TedAPIAdapter(request_api=TedRequestAPI(),
                                                 ted_api_url=ted_api_end_point)).fetch_notices_by_query(
         query=notice_search_query)
-    notice = notice_repository.get(reference="067623-2022")
+    notice = notice_repository.get(reference="67623-2022")
     notice.set_xml_metadata(xml_metadata=XMLMetadata(unique_xpaths=["FAKE_INDEX_XPATHS"]))
     return notice
 

tests/features/notice_fetcher/conftest.py

@@ -2,13 +2,14 @@
 
 import pytest
 
+from ted_sws.notice_fetcher.adapters.ted_api import TedAPIAdapter
 from ted_sws.notice_fetcher.services.notice_fetcher import NoticeFetcher
-from tests.fakes.fake_ted_api import FakeTedApiAdapter
+from tests.fakes.fake_ted_api import FakeTedApiAdapter, FakeRequestAPI
 
 
 @pytest.fixture
 def fetch_notice_id():
-    return "067623-2022"
+    return "67623-2022"
 
 
 @pytest.fixture
@@ -33,10 +34,10 @@ def fetch_wildcard_date():
 
 @pytest.fixture
 def fetch_query(fetch_wildcard_date):
-    return {"q": f"PD=[{fetch_wildcard_date}]"}
+    return {"query": f"PD={fetch_wildcard_date}"}
 
 
 @pytest.fixture
 def notice_fetcher(notice_repository, ted_api_end_point):
     return NoticeFetcher(notice_repository=notice_repository,
-                         ted_api_adapter=FakeTedApiAdapter())
+                         ted_api_adapter=TedAPIAdapter(FakeRequestAPI()))

tests/features/notice_metadata_processor/conftest.py

@@ -10,7 +10,7 @@
 
 @pytest.fixture
 def notice_identifier():
-    return "067623-2022"
+    return "67623-2022"
 
 
 @pytest.fixture

tests/features/notice_metadata_processor/test_notice_extractor.py

@@ -31,5 +31,5 @@ def step_impl(xml_manifestation):
 def step_impl(extracted_metadata, notice_identifier, metadata):
     assert isinstance(extracted_metadata, ExtractedMetadata)
     assert extracted_metadata.dict().keys() == ExtractedMetadata.__fields__.keys()
-    assert notice_identifier == extracted_metadata.dict()["notice_publication_number"]
+    assert "0"+notice_identifier == extracted_metadata.dict()["notice_publication_number"]
     assert metadata in extracted_metadata.dict()

tests/unit/notice_fetcher/test_notice_fetcher.py

@@ -4,7 +4,7 @@
 
 
 def test_notice_fetcher_by_identifier(notice_repository, ted_document_search):
-    document_id = "67623-2023"
+    document_id = "067623-2022"
     NoticeFetcher(notice_repository=notice_repository, ted_api_adapter=ted_document_search).fetch_notice_by_id(
         document_id=document_id)
     notice = notice_repository.get(reference=document_id)

tests/unit/notice_metadata_processor/conftest.py

@@ -1,3 +1,6 @@
+import pathlib
+from typing import List
+
 import pytest
 
 from tests import TEST_DATA_PATH
@@ -11,3 +14,9 @@ def notice_eligibility_repository_path():
 @pytest.fixture
 def file_system_repository_path():
     return TEST_DATA_PATH / "notice_transformer" / "mapping_suite_processor_repository"
+
+
+@pytest.fixture
+def eforms_xml_notice_paths() -> List[pathlib.Path]:
+    eforms_xml_notices_path = TEST_DATA_PATH / "eforms_samples"
+    return list(eforms_xml_notices_path.glob("**/*.xml"))

tests/unit/notice_metadata_processor/test_metadata_extractor.py

@@ -1,12 +1,12 @@
 import xml.etree.ElementTree as ET
 
+from ted_sws.core.model.manifestation import XMLManifestation
 from ted_sws.notice_metadata_processor.adapters.notice_metadata_extractor import EformsNoticeMetadataExtractor, \
     DefaultNoticeMetadataExtractor, extract_text_from_element, extract_attribute_from_element, \
     extract_code_and_value_from_element, parse_xml_manifestation, normalised_namespaces_from_xml_manifestation
 from ted_sws.notice_metadata_processor.model.metadata import ExtractedMetadata, EncodedValue
 
 
-
 def test_metadata_extractor(indexed_notice):
     metadata_extractor = DefaultNoticeMetadataExtractor(
         xml_manifestation=indexed_notice.xml_manifestation).extract_metadata()
@@ -109,10 +109,28 @@ def test_metadata_eform_extractor(eform_notice_622690):
     metadata_extractor = EformsNoticeMetadataExtractor(
         xml_manifestation=eform_notice_622690.xml_manifestation).extract_metadata()
     extracted_metadata_dict = metadata_extractor.dict()
-    print(extracted_metadata_dict)
     assert isinstance(metadata_extractor, ExtractedMetadata)
     assert extracted_metadata_dict.keys() == ExtractedMetadata.__fields__.keys()
     assert "extracted_form_number", "xml_schema" in extracted_metadata_dict.keys()
     assert "00622690-2023" in extracted_metadata_dict["notice_publication_number"]
     assert "competition" in extracted_metadata_dict["extracted_eform_type"]
     assert extracted_metadata_dict["extracted_form_number"] == None
+
+
+def _test_metadata_extractor_for_all_eforms_variations(eforms_xml_notice_paths):
+    for xml_notice_path in eforms_xml_notice_paths:
+        notice_id = xml_notice_path.name
+        eforms_subtype = xml_notice_path.parent.name
+        eforms_sdk_version = xml_notice_path.parent.parent.name
+        try:
+            notice_content = xml_notice_path.read_text(encoding="utf-8")
+            extracted_metadata = EformsNoticeMetadataExtractor(
+                xml_manifestation=XMLManifestation(object_data=notice_content)
+            ).extract_metadata()
+            extracted_metadata_dict = extracted_metadata.dict()
+            assert isinstance(extracted_metadata, ExtractedMetadata)
+            assert extracted_metadata_dict.keys() == ExtractedMetadata.__fields__.keys()
+            assert "extracted_form_number", "xml_schema" in extracted_metadata_dict.keys()
+        except Exception as e:
+            raise Exception(f"Exception for {eforms_sdk_version}, {eforms_subtype}, notice_id:{notice_id}, exception:", str(e))
+