Merge pull request #32 from meaningfy-ws/feature/MWB-152

Feature/mwb 152

.gitignore

@@ -139,3 +139,4 @@ junit_report.xml
 node_modules
 package-lock.json
 infra/backend/src
+.rmlmapper

Makefile

@@ -7,6 +7,7 @@ BACKEND_INFRA_FOLDER := ${PROJECT_PATH}/${NAME}/backend
 FRONTEND_HOME := ${NAME}/frontend
 FRONTEND_INFRA_FOLDER := ${PROJECT_PATH}/${FRONTEND_HOME}
 
+RML_MAPPER_PATH = ${PROJECT_PATH}/.rmlmapper/rmlmapper.jar
 
 
 # include .env files if they exist
@@ -21,7 +22,7 @@ install: install-backend install-frontend
 
 install-dev: install-dev-backend install-frontend-dev
 
-install-backend:
+install-backend: init-rml-mapper
 	@ echo "Installing BACKEND requirements :: START"
 	@ pip install --upgrade pip
 	@ pip install --no-cache-dir -r requirements.txt
@@ -76,6 +77,7 @@ dev-dotenv-file:
 	@ echo BACKEND_INFRA_FOLDER=${BACKEND_INFRA_FOLDER} >> ${ENV_FILE}
 	@ echo FRONTEND_INFRA_FOLDER=${FRONTEND_INFRA_FOLDER} >> ${ENV_FILE}
 	@ echo NODE_ENV=development >> ${ENV_FILE}
+	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
 	@ vault kv get -format="json" mapping-workbench-dev/app | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> ${ENV_FILE}
 
 staging-dotenv-file:
@@ -85,6 +87,7 @@ staging-dotenv-file:
 	@ echo BACKEND_INFRA_FOLDER=${BACKEND_INFRA_FOLDER} >> ${ENV_FILE}
 	@ echo FRONTEND_INFRA_FOLDER=${FRONTEND_INFRA_FOLDER} >> ${ENV_FILE}
 	@ echo NODE_ENV=development >> ${ENV_FILE}
+	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
 	@ vault kv get -format="json" mapping-workbench-staging/app | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> ${ENV_FILE}
 
 prod-dotenv-file:
@@ -94,15 +97,14 @@ prod-dotenv-file:
 	@ echo BACKEND_INFRA_FOLDER=${BACKEND_INFRA_FOLDER} >> ${ENV_FILE}
 	@ echo FRONTEND_INFRA_FOLDER=${FRONTEND_INFRA_FOLDER} >> ${ENV_FILE}
 	@ echo NODE_ENV=production >> ${ENV_FILE}
+	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
 	@ vault kv get -format="json" mapping-workbench-prod/app | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> ${ENV_FILE}
 
 
 #-----------------------------------------------------------------------------
 # STAGING & PRODUCTION
 #-----------------------------------------------------------------------------
 
-
-
 build-backend:
 	@ echo "Building the BACKEND"
 	@ docker-compose -p ${NAME} --file ./infra/backend/docker-compose.yml --env-file ${ENV_FILE} build --progress plain --no-cache --force-rm
@@ -178,6 +180,9 @@ start-frontend-console-mode:
 start-backend-console-mode:
 	uvicorn mapping_workbench.backend.core.entrypoints.api.main:app --reload
 
+start-mongo-console-mode:
+	mongod --dbpath=/usr/local/var/mongodb/data/
+
 start-mongo-dev: build-externals
 	@ echo "Starting the Mongo services"
 	@ docker-compose -p ${NAME} --file ./infra/mongodb/docker-compose.dev.yml --env-file ${ENV_FILE} up -d
@@ -202,3 +207,8 @@ start-traefik: build-externals
 stop-traefik:
 	@ echo "Stopping the Traefik services"
 	@ docker-compose -p common --file ./infra/traefik/docker-compose.yml --env-file ${ENV_FILE} down
+
+init-rml-mapper:
+	@ echo -e "RMLMapper folder initialization!"
+	@ mkdir -p ./.rmlmapper
+	@ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.2.1/rmlmapper-6.2.1-r368-all.jar -O ./.rmlmapper/rmlmapper.jar

infra/backend/Dockerfile

@@ -5,7 +5,7 @@ ARG PORT
 
 ENV PORT=$PORT
 
-RUN apt update && apt install -y make
+RUN apt update && apt install -y make wget
 
 # Create an application directory
 RUN mkdir -p /app

mapping_workbench/__init__.py

@@ -1 +1,5 @@
+import pathlib
+
 __version__ = "0.1.0"
+__root_path__ = pathlib.Path(__file__).parent.resolve()
+

mapping_workbench/backend/conceptual_mapping_rule/entrypoints/api/routes.py

@@ -3,22 +3,26 @@
 from beanie import PydanticObjectId
 from fastapi import APIRouter, Depends, status, Query
 
+from mapping_workbench.backend.conceptual_mapping_rule.models.api_request import \
+    APIRequestForGenerateCMAssertionsQueries
 from mapping_workbench.backend.conceptual_mapping_rule.models.entity import ConceptualMappingRuleOut, \
     ConceptualMappingRuleCreateIn, \
-    ConceptualMappingRuleUpdateIn, ConceptualMappingRule
+    ConceptualMappingRuleUpdateIn, ConceptualMappingRule, ConceptualMappingRuleTermsValidity
 from mapping_workbench.backend.conceptual_mapping_rule.models.entity_api_response import \
     APIListConceptualMappingRulesPaginatedResponse
 from mapping_workbench.backend.conceptual_mapping_rule.services.api import (
     list_conceptual_mapping_rules,
     create_conceptual_mapping_rule,
     update_conceptual_mapping_rule,
     get_conceptual_mapping_rule,
-    delete_conceptual_mapping_rule, get_conceptual_mapping_rule_out
+    delete_conceptual_mapping_rule, get_conceptual_mapping_rule_out, clone_conceptual_mapping_rule
 )
 from mapping_workbench.backend.core.models.api_response import APIEmptyContentWithIdResponse
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage
 from mapping_workbench.backend.project.models.entity import Project
 from mapping_workbench.backend.security.services.user_manager import current_active_user
+from mapping_workbench.backend.sparql_test_suite.services.sparql_cm_assertions import \
+    clean_sparql_cm_assertions_queries_for_project, generate_and_save_cm_assertions_queries
 from mapping_workbench.backend.user.models.user import User
 
 ROUTE_PREFIX = "/conceptual_mapping_rules"
@@ -40,18 +44,26 @@
 )
 async def route_list_conceptual_mapping_rules(
         project: PydanticObjectId = None,
-        mapping_packages: Annotated[List[PydanticObjectId | str] | None, Query()] = None
+        mapping_packages: Annotated[List[PydanticObjectId | str] | None, Query()] = None,
+        page: int = None,
+        limit: int = None,
+        q: str = None,
+        terms_validity: ConceptualMappingRuleTermsValidity = None
 ):
     filters: dict = {}
     if project:
         filters['project'] = Project.link_from_id(project)
     if mapping_packages is not None:
         filters['mapping_packages'] = {"$in": list(map(lambda x: MappingPackage.link_from_id(x), mapping_packages))}
+    if q is not None:
+        filters['q'] = q
+    if terms_validity:
+        filters['terms_validity'] = terms_validity
 
-    items: List[ConceptualMappingRuleOut] = await list_conceptual_mapping_rules(filters)
+    items, total_count = await list_conceptual_mapping_rules(filters, page, limit)
     return APIListConceptualMappingRulesPaginatedResponse(
         items=items,
-        count=len(items)
+        count=total_count
     )
 
 
@@ -63,10 +75,10 @@ async def route_list_conceptual_mapping_rules(
     status_code=status.HTTP_201_CREATED
 )
 async def route_create_conceptual_mapping_rule(
-        conceptual_mapping_rule_data: ConceptualMappingRuleCreateIn,
+        data: ConceptualMappingRuleCreateIn,
         user: User = Depends(current_active_user)
 ):
-    return await create_conceptual_mapping_rule(conceptual_mapping_rule_data=conceptual_mapping_rule_data, user=user)
+    return await create_conceptual_mapping_rule(data, user=user)
 
 
 @router.patch(
@@ -76,12 +88,11 @@ async def route_create_conceptual_mapping_rule(
     response_model=ConceptualMappingRuleOut
 )
 async def route_update_conceptual_mapping_rule(
-        id: PydanticObjectId,
-        conceptual_mapping_rule_data: ConceptualMappingRuleUpdateIn,
+        data: ConceptualMappingRuleUpdateIn,
+        conceptual_mapping_rule: ConceptualMappingRule = Depends(get_conceptual_mapping_rule),
         user: User = Depends(current_active_user)
 ):
-    await update_conceptual_mapping_rule(id=id, conceptual_mapping_rule_data=conceptual_mapping_rule_data, user=user)
-    return await get_conceptual_mapping_rule_out(id)
+    return await update_conceptual_mapping_rule(conceptual_mapping_rule, data, user=user)
 
 
 @router.get(
@@ -104,4 +115,31 @@ async def route_get_conceptual_mapping_rule(
 async def route_delete_conceptual_mapping_rule(
         conceptual_mapping_rule: ConceptualMappingRule = Depends(get_conceptual_mapping_rule)):
     await delete_conceptual_mapping_rule(conceptual_mapping_rule)
-    return APIEmptyContentWithIdResponse(_id=conceptual_mapping_rule.id)
+    return APIEmptyContentWithIdResponse(id=conceptual_mapping_rule.id)
+
+
+@router.post(
+    "/{id}/clone",
+    description=f"Clone {NAME_FOR_ONE}",
+    name=f"{NAME_FOR_MANY}:clone_{NAME_FOR_ONE}",
+    response_model=ConceptualMappingRuleOut
+)
+async def route_clone_conceptual_mapping_rule(
+        conceptual_mapping_rule: ConceptualMappingRule = Depends(get_conceptual_mapping_rule),
+        user: User = Depends(current_active_user)
+):
+    return await clone_conceptual_mapping_rule(conceptual_mapping_rule, user=user)
+
+
+@router.post(
+    "/tasks/generate_cm_assertions_queries",
+    description=f"Generate CM Assertions Queries",
+    name=f"generate_cm_assertions_queries"
+)
+async def route_generate_cm_assertions_queries(
+        filters: APIRequestForGenerateCMAssertionsQueries,
+        user: User = Depends(current_active_user)
+):
+    if filters.cleanup:
+        await clean_sparql_cm_assertions_queries_for_project(project_id=filters.project)
+    return await generate_and_save_cm_assertions_queries(project_id=filters.project, user=user)

mapping_workbench/backend/conceptual_mapping_rule/models/api_request.py

@@ -0,0 +1,9 @@
+from typing import Optional
+
+from beanie import PydanticObjectId
+from pydantic import BaseModel
+
+
+class APIRequestForGenerateCMAssertionsQueries(BaseModel):
+    project: Optional[PydanticObjectId] = None
+    cleanup: Optional[bool] = False

mapping_workbench/backend/conceptual_mapping_rule/models/entity.py

@@ -1,24 +1,54 @@
+from datetime import datetime
+from enum import Enum
 from typing import Optional, List
 
-from beanie import Link
+import pymongo
+from beanie import Link, Indexed
+from pydantic import BaseModel
+from pymongo import IndexModel
 
 from mapping_workbench.backend.core.models.base_project_resource_entity import BaseProjectResourceEntity, \
     BaseProjectResourceEntityInSchema, BaseProjectResourceEntityOutSchema
 from mapping_workbench.backend.mapping_package.models.entity import MappingPackage
+from mapping_workbench.backend.ontology.models.term import TermValidityResponse
 from mapping_workbench.backend.sparql_test_suite.models.entity import SPARQLTestFileResource
 from mapping_workbench.backend.triple_map_fragment.models.entity import GenericTripleMapFragment
+from mapping_workbench.backend.user.models.user import User
+
+
+class ConceptualMappingRuleTermsValidity(Enum):
+    VALID = "valid"
+    INVALID = "invalid"
+
+
+class ConceptualMappingRuleCommentPriority(Enum):
+    HIGH = "high"
+    NORMAL = "normal"
+    LOW = "low"
+
+
+class ConceptualMappingRuleComment(BaseModel):
+    title: Optional[str] = None
+    comment: Optional[str] = None
+    priority: Optional[ConceptualMappingRuleCommentPriority] = ConceptualMappingRuleCommentPriority.NORMAL
+    created_at: Optional[datetime] = None
+    created_by: Optional[Link[User]] = None
+    updated_at: Optional[datetime] = None
+    updated_by: Optional[Link[User]] = None
 
 
 class ConceptualMappingRuleIn(BaseProjectResourceEntityInSchema):
-    field_id: Optional[str]
-    field_title: Optional[str]
-    field_description: Optional[str]
-    source_xpath: Optional[List[str]]
-    target_class_path: Optional[str]
-    target_property_path: Optional[str]
-    mapping_packages: Optional[List[Link[MappingPackage]]]
-    triple_map_fragment: Optional[Link[GenericTripleMapFragment]]
-    sparql_assertions: Optional[List[Link[SPARQLTestFileResource]]]
+    field_id: Optional[str] = None
+    field_title: Optional[str] = None
+    field_description: Optional[str] = None
+    source_xpath: Optional[List[str]] = None
+    target_class_path: Optional[str] = None
+    target_property_path: Optional[str] = None
+    mapping_packages: Optional[List[Link[MappingPackage]]] = None
+    triple_map_fragment: Optional[Link[GenericTripleMapFragment]] = None
+    sparql_assertions: Optional[List[Link[SPARQLTestFileResource]]] = None
+    notes: Optional[List[ConceptualMappingRuleComment]] = None
+    comments: Optional[List[ConceptualMappingRuleComment]] = None
 
 
 class ConceptualMappingRuleCreateIn(ConceptualMappingRuleIn):
@@ -30,28 +60,50 @@ class ConceptualMappingRuleUpdateIn(ConceptualMappingRuleIn):
 
 
 class ConceptualMappingRuleOut(BaseProjectResourceEntityOutSchema):
-    field_id: Optional[str]
-    field_title: Optional[str]
-    field_description: Optional[str]
-    source_xpath: Optional[List[str]]
-    target_class_path: Optional[str]
-    target_property_path: Optional[str]
-    mapping_packages: Optional[List[Link[MappingPackage]]]
-    triple_map_fragment: Optional[Link[GenericTripleMapFragment]]
-    sparql_assertions: Optional[List[Link[SPARQLTestFileResource]]]
+    field_id: Optional[str] = None
+    field_title: Optional[str] = None
+    field_description: Optional[str] = None
+    source_xpath: Optional[List[str]] = None
+    target_class_path: Optional[str] = None
+    target_class_path_terms_validity: Optional[List[TermValidityResponse]] = None
+    target_property_path: Optional[str] = None
+    target_property_path_terms_validity: Optional[List[TermValidityResponse]] = None
+    terms_validity: Optional[ConceptualMappingRuleTermsValidity] = None
+    mapping_packages: Optional[List[Link[MappingPackage]]] = None
+    triple_map_fragment: Optional[Link[GenericTripleMapFragment]] = None
+    sparql_assertions: Optional[List[Link[SPARQLTestFileResource]]] = None
+    notes: Optional[List[ConceptualMappingRuleComment]] = None
+    comments: Optional[List[ConceptualMappingRuleComment]] = None
 
 
 class ConceptualMappingRule(BaseProjectResourceEntity):
-    field_id: Optional[str]
-    field_title: Optional[str]
-    field_description: Optional[str]
-    source_xpath: Optional[List[str]]
-    target_class_path: Optional[str]
-    target_property_path: Optional[str]
-    mapping_packages: Optional[List[Link[MappingPackage]]]
-    triple_map_fragment: Optional[Link[GenericTripleMapFragment]]
-    sparql_assertions: Optional[List[Link[SPARQLTestFileResource]]]
+    field_id: Optional[str] = None
+    field_title: Optional[str] = None
+    field_description: Optional[str] = None
+    source_xpath: Optional[List[str]] = None
+    target_class_path: Optional[str] = None
+    target_class_path_terms_validity: Optional[List[TermValidityResponse]] = None
+    target_property_path: Optional[str] = None
+    target_property_path_terms_validity: Optional[List[TermValidityResponse]] = None
+    terms_validity: Optional[ConceptualMappingRuleTermsValidity] = None
+    mapping_packages: Optional[List[Optional[Link[MappingPackage]]]] = None
+    triple_map_fragment: Optional[Link[GenericTripleMapFragment]] = None
+    sparql_assertions: Optional[List[Link[SPARQLTestFileResource]]] = None
+    notes: Optional[List[ConceptualMappingRuleComment]] = None
+    comments: Optional[List[ConceptualMappingRuleComment]] = None
 
     class Settings(BaseProjectResourceEntity.Settings):
         name = "conceptual_mapping_rules"
-
+        indexes = [
+            IndexModel(
+                [
+                    ("field_id", pymongo.TEXT),
+                    ("field_title", pymongo.TEXT),
+                    ("field_description", pymongo.TEXT),
+                    ("source_xpath", pymongo.TEXT),
+                    ("target_class_path", pymongo.TEXT),
+                    ("target_property_path", pymongo.TEXT)
+                ],
+                name="search_text_idx"
+            )
+        ]