Merge branch 'feature/TED6-37' into feature/TED6-23

Makefile

@@ -24,12 +24,12 @@ CAROOT = $(shell pwd)/infra/traefik/certs
 install:
 	@ echo -e "$(BUILD_PRINT)Installing the requirements$(END_BUILD_PRINT)"
 	@ pip install --upgrade pip
-	@ pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt"
+	@ pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-no-providers-3.8.txt"
 
 install-dev:
 	@ echo -e "$(BUILD_PRINT)Installing the dev requirements$(END_BUILD_PRINT)"
 	@ pip install --upgrade pip
-	@ pip install --no-cache-dir -r requirements.dev.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt"
+	@ pip install --no-cache-dir -r requirements.dev.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-no-providers-3.8.txt"
 
 test: test-unit
 
@@ -213,7 +213,7 @@ stop-metabase:
 init-rml-mapper:
 	@ echo -e "RMLMapper folder initialisation!"
 	@ mkdir -p ./.rmlmapper
-	@ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar
+	@ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.2.1/rmlmapper-6.2.1-r368-all.jar -O ./.rmlmapper/rmlmapper.jar
 
 init-limes:
 	@ echo -e "Limes folder initialisation!"

infra/airflow-cluster/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/apache/airflow:2.5.1-python3.8
+FROM docker.io/apache/airflow:2.7.1-python3.8
 
 # quick sudo
 USER root
@@ -14,7 +14,7 @@ COPY requirements.txt /opt/airflow
 # working in the /opt/airflow
 WORKDIR /opt/airflow
 RUN mkdir -p ./.rmlmapper
-RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar
+RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.2.1/rmlmapper-6.2.1-r368-all.jar -O ./.rmlmapper/rmlmapper.jar
 
 
 RUN wget -c https://kumisystems.dl.sourceforge.net/project/saxon/Saxon-HE/10/Java/SaxonHE10-6J.zip -P .saxon/
@@ -26,4 +26,4 @@ RUN wget -c https://github.com/dice-group/LIMES/releases/download/1.7.9/limes.ja
 
 
 RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt"
+RUN pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-no-providers-3.8.txt"

infra/airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/apache/airflow:2.5.1-python3.8
+FROM docker.io/apache/airflow:2.7.1-python3.8
 
 # quick sudo
 USER root
@@ -14,7 +14,7 @@ COPY requirements.txt /opt/airflow
 # working in the /opt/airflow
 WORKDIR /opt/airflow
 RUN mkdir -p ./.rmlmapper ./dags ./ted_sws ./temp
-RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar
+RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.2.1/rmlmapper-6.2.1-r368-all.jar -O ./.rmlmapper/rmlmapper.jar
 
 
 RUN wget -c https://kumisystems.dl.sourceforge.net/project/saxon/Saxon-HE/10/Java/SaxonHE10-6J.zip -P .saxon/
@@ -25,4 +25,4 @@ RUN wget -c https://github.com/dice-group/LIMES/releases/download/1.7.9/limes.ja
 
 
 RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt"
+RUN pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-no-providers-3.8.txt"

requirements.dev.txt

@@ -1,12 +1,12 @@
 # Development libraries
-coverage~=6.3.1
-pytest~=7.0.0
-pytest-bdd~=5.0.0
-pytest-cov~=3.0.0
-pytest-subtests~=0.6.0
-tox~=3.24.5
+allure-pytest-bdd~=2.13.2
+coverage~=7.3.1
+mongomock~=4.1.2
+pycurl~=7.45.2
+pytest~=7.4.2
+pytest-bdd~=6.1.1
+pytest-cov~=4.1.0
+pytest-subtests~=0.11.0
+tox~=4.11.3
 tox-pytest-summary~=0.1.2
-mongomock==4.1.2
-uvicorn[standard]
-allure-pytest-bdd==2.10.0
-pycurl~=7.45.2
+uvicorn[standard]~=0.23.2

requirements.txt

@@ -1,28 +1,28 @@
 # Project dependecies
-pydantic~=1.9.0
-requests~=2.28.2
-deepdiff~=5.7.0
-jinja2~=3.1.2
-python-dotenv~=0.19.2
-pymongo~=4.0.1
-apache-airflow~=2.5.1
-hvac==0.11.2
-SPARQLWrapper==1.8.5
-pandas==1.5.2
-click~=8.1.0
-openpyxl==3.0.9
-colorama~=0.4
-fastapi~=0.77
-python-dateutil~=2.8.2
-rdflib~=6.1.1
-pyshacl~=0.19.0
-agraph-python==101.0.10
+agraph-python~=102.0.0
+apache-airflow~=2.7.1
+certifi~=2023.7.22
+click~=8.1.7
+colorama~=0.4.6
 decorator~=5.1.1
-urllib3[secure]
-semantic-version==2.10.0
-paramiko~=3.0.0
-ordered-set~=4.0.2
+deepdiff~=6.5.0
+fastapi~=0.103.0
+hvac~=1.2.1
+jinja2~=3.1.2
 json2html~=1.3.0
-minio~=7.1.1
-certifi~=2022.12.7
-shortuuid~=1.0.11
+minio~=7.1.16
+openpyxl~=3.1.2
+ordered-set~=4.1.0
+paramiko~=3.3.1
+pandas~=2.0.3
+pydantic~=2.3.0
+pymongo~=4.5.0
+pyshacl~=0.23.0
+python-dateutil~=2.8.2
+python-dotenv~=1.0.0
+rdflib~=6.3.2
+requests~=2.31.0
+semantic-version~=2.10.0
+shortuuid~=1.0.11
+SPARQLWrapper~=2.0.0
+urllib3[secure]~=1.26.16

ted_sws/core/model/__init__.py

@@ -22,24 +22,24 @@ class PropertyBaseModel(BaseModel):
     https://github.com/samuelcolvin/pydantic/issues/935
     is solved
     """
-
-    @classmethod
-    def get_properties(cls):
-        return [
-            prop for prop in dir(cls)
-            if isinstance(getattr(cls, prop), property) and prop not in ("__values__", "fields")
-        ]
-
-    def dict(self, *args, **kwargs) -> 'DictStrAny':
-        self.__dict__.update({prop: getattr(self, prop) for prop in self.get_properties()})
-
-        return super().dict(*args, **kwargs)
+    #TODO : remove this code, because this implementation is not necessary anymore in pydantic 2
+    # @classmethod
+    # def get_properties(cls):
+    #     return [
+    #         prop for prop in dir(cls)
+    #         if isinstance(getattr(cls, prop), property) and prop not in ("__values__", "fields")
+    #     ]
+    #
+    #TODO : remove this code, because this implementation is not necessary anymore in pydantic 2
+    # def dict(self, *args, **kwargs) -> 'DictStrAny':
+    #     self.__dict__.update({prop: getattr(self, prop) for prop in self.get_properties()})
+    #     return super().dict(*args, **kwargs)
 
     def __eq__(self, other) -> bool:
         if not isinstance(other, self.__class__) or not other:
             return False
             # raise ValueError(f"Must compare objects of the same class {self.__class__}")
-        difference = DeepDiff(self.dict(), other.dict())
+        difference = DeepDiff(self.model_dump(), other.model_dump())
         return not difference
 
     def __ne__(self, other):

ted_sws/core/model/manifestation.py

@@ -69,7 +69,7 @@ class XPATHCoverageSummaryResult(PropertyBaseModel):
 
 
 class XPATHCoverageSummaryReport(PropertyBaseModel):
-    mapping_suite_identifier: Optional[str]
+    mapping_suite_identifier: Optional[str] = None
     validation_result: Optional[XPATHCoverageSummaryResult] = XPATHCoverageSummaryResult()
 
 
@@ -86,8 +86,8 @@ class SPARQLSummaryCountReport(PropertyBaseModel):
 
 
 class SPARQLSummaryResult(PropertyBaseModel):
-    test_suite_identifier: Optional[str]
-    mapping_suite_identifier: Optional[str]
+    test_suite_identifier: Optional[str] = None
+    mapping_suite_identifier: Optional[str] = None
     aggregate: Optional[SPARQLSummaryCountReport] = SPARQLSummaryCountReport()
 
 
@@ -107,8 +107,8 @@ class SHACLSummaryResultSeverityReport(PropertyBaseModel):
 
 
 class SHACLSummaryResult(PropertyBaseModel):
-    test_suite_identifier: Optional[str]
-    mapping_suite_identifier: Optional[str]
+    test_suite_identifier: Optional[str] = None
+    mapping_suite_identifier: Optional[str] = None
     result_severity: Optional[SHACLSummaryResultSeverityReport] = SHACLSummaryResultSeverityReport()
 
 
@@ -141,11 +141,11 @@ class XPATHCoverageValidationAssertion(PropertyBaseModel):
     """
 
     """
-    form_field: Optional[str]
-    xpath: Optional[str]
-    count: Optional[int]
-    notice_hit: Optional[Dict[str, int]]
-    query_result: Optional[bool]
+    form_field: Optional[str] = None
+    xpath: Optional[str] = None
+    count: Optional[int] = None
+    notice_hit: Optional[Dict[str, int]] = None
+    query_result: Optional[bool] = None
 
 
 class XPATHCoverageValidationResultBase(PropertyBaseModel):
@@ -157,30 +157,30 @@ class XPATHCoverageValidationResultBase(PropertyBaseModel):
     xpath_not_covered: Optional[List[str]] = []
     xpath_extra: Optional[List[str]] = []
     remarked_xpaths: Optional[List[str]] = []
-    coverage: Optional[float]
-    conceptual_coverage: Optional[float]
+    coverage: Optional[float] = None
+    conceptual_coverage: Optional[float] = None
 
 
 class XPATHCoverageValidationResult(XPATHCoverageValidationResultBase):
     """
     XPATHCoverageValidationResult for Notice
     """
-    notices: Optional[List[ReportNoticeData]]
+    notices: Optional[List[ReportNoticeData]] = None
 
 
 class XPATHCoverageValidationReport(XMLValidationManifestation):
     """
     This is the model structure for Notice(s) XPATHs Coverage Report
     """
 
-    validation_result: Optional[XPATHCoverageValidationResult]
+    validation_result: Optional[XPATHCoverageValidationResult] = None
 
 
 class XMLManifestation(Manifestation):
     """
         Original XML Notice manifestation as published on the TED website.
     """
-    xpath_coverage_validation: XPATHCoverageValidationReport = None
+    xpath_coverage_validation: Optional[XPATHCoverageValidationReport] = None
 
     def add_validation(self, validation: Union[XPATHCoverageValidationReport]):
         if type(validation) == XPATHCoverageValidationReport:
@@ -197,7 +197,7 @@ class METSManifestation(Manifestation):
 
     """
     type: str = METS_TYPE_CREATE
-    package_name: str = None
+    package_name: Optional[str] = None
 
 
 class RDFValidationManifestation(ValidationManifestation):
@@ -206,15 +206,15 @@ class RDFValidationManifestation(ValidationManifestation):
 
     """
     mapping_suite_identifier: str
-    test_suite_identifier: Optional[str]
+    test_suite_identifier: Optional[str] = None
 
 
 class SPARQLQuery(PropertyBaseModel):
     """
     Stores SPARQL query details
     """
-    title: Optional[str]
-    description: Optional[str]
+    title: Optional[str] = None
+    description: Optional[str] = None
     xpath: Optional[List[str]] = []
     query: str
 
@@ -224,13 +224,13 @@ class SPARQLQueryResult(PropertyBaseModel):
     Stores SPARQL query execution result
     """
     query: SPARQLQuery
-    result: Optional[SPARQLQueryRefinedResultType]
-    query_result: Optional[str]
+    result: Optional[SPARQLQueryRefinedResultType] = None
+    query_result: Optional[str] = None
     fields_covered: Optional[bool] = True
     missing_fields: Optional[List[str]] = []
-    error: Optional[str]
-    message: Optional[str]
-    identifier: Optional[str]
+    error: Optional[str] = None
+    message: Optional[str] = None
+    identifier: Optional[str] = None
 
     class Config:
         use_enum_values = True
@@ -248,10 +248,10 @@ class QueriedSHACLShapeValidationResult(PropertyBaseModel):
     Queried SHACL Validation Report which contains the following variables
     ?focusNode ?message ?resultPath ?resultSeverity ?sourceConstraintComponent ?sourceShape ?value
     """
-    conforms: Optional[str]
-    results_dict: Optional[dict]
-    error: Optional[str]
-    identifier: Optional[str]
+    conforms: Optional[str] = None
+    results_dict: Optional[dict] = None
+    error: Optional[str] = None
+    identifier: Optional[str] = None
 
 
 class SHACLTestSuiteValidationReport(RDFValidationManifestation):
@@ -262,7 +262,7 @@ class SHACLTestSuiteValidationReport(RDFValidationManifestation):
 
 
 class EntityDeduplicationReport(Manifestation):
-    object_data: Optional[str]
+    object_data: Optional[str] = None
     number_of_duplicates: int
     number_of_cets: int
     uries: List[str]
@@ -272,10 +272,10 @@ class RDFManifestation(Manifestation):
     """
         Transformed manifestation in RDF format
     """
-    mapping_suite_id = "unknown_mapping_suite_id"
+    mapping_suite_id: str = "unknown_mapping_suite_id"
     shacl_validations: List[SHACLTestSuiteValidationReport] = []
     sparql_validations: List[SPARQLTestSuiteValidationReport] = []
-    deduplication_report: Optional[EntityDeduplicationReport]
+    deduplication_report: Optional[EntityDeduplicationReport] = None
 
     def validation_exists(self, validation, validations):
         """