Merge pull request #332 from meaningfy-ws/develop

release/2024-10-21-rc1

.github/workflows/production-ci-cd-pipeline.yml

@@ -29,7 +29,7 @@ jobs:
 
     - name: connect, pull and build
       run: |
-        ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "source ~/.bashrc && cd ${{ secrets.PROD_WORK_DIR }} && git checkout main && git pull && make prod-dotenv-file && make stop-frontend && make build-frontend && make stop-backend && make build-backend && exit"
+        ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "source ~/.bashrc && cd ${{ secrets.PROD_WORK_DIR }} && git checkout main && git pull && make deploy-prod-dotenv-file && make stop-frontend && make build-frontend && make stop-backend && make build-backend && exit"
 
     - name: cleanup unused containers, images, volumes, and networks
       run: |

.github/workflows/staging-ci-cd-pipeline.yml

@@ -24,7 +24,7 @@ jobs:
 
       - name: connect, pull and build
         run: |
-          ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "source ~/.bashrc && cd ${{ secrets.STAGING_WORK_DIR }} && git checkout develop && git pull && make staging-dotenv-file && make stop-frontend && make build-frontend && make stop-backend && make build-backend && exit"
+          ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "source ~/.bashrc && cd ${{ secrets.STAGING_WORK_DIR }} && git checkout develop && git pull && make deploy-staging-dotenv-file && make stop-frontend && make build-frontend && make stop-backend && make build-backend && exit"
 
       - name: cleanup unused containers, images, volumes, and networks
         run: |

Makefile

@@ -13,6 +13,8 @@ FRONTEND_INFRA_FOLDER := ${PROJECT_PATH}/${FRONTEND_HOME}
 
 RML_MAPPER_PATH = ${PROJECT_PATH}/.rmlmapper/rmlmapper.jar
 
+PYTHON := python3
+APP_VERSION_SCRIPT := ${BACKEND_INFRA_FOLDER}/core/scripts/get_app_version.py
 
 #-----------------------------------------------------------------------------
 # INSTALLING
@@ -245,3 +247,16 @@ init-rml-mapper:
 setup-env-paths:
 	@ perl -i -ne 'print unless /^RML_MAPPER_PATH/' ${ENV_FILE}
 	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> ${ENV_FILE}
+
+deploy-app-version:
+	@ perl -i -ne 'print unless /^MW_APP_VERSION/' ${ENV_FILE}
+	@ echo MW_APP_VERSION=$$($(PYTHON) $(APP_VERSION_SCRIPT)) >> ${ENV_FILE}
+
+deploy-env-app-settings: deploy-app-version
+	@ echo "Deployed ENV App Settings"
+
+deploy-prod-dotenv-file: prod-dotenv-file deploy-env-app-settings
+	@ echo "Deployed PROD ENV file"
+
+deploy-staging-dotenv-file: staging-dotenv-file deploy-env-app-settings
+	@ echo "Deployed STAGING ENV file"

mapping_workbench/backend/config/__init__.py

@@ -14,6 +14,10 @@ class AppSettings(BaseSettings):
     def APP_NAME(self, config_value: str) -> str:
         return config_value
 
+    @env_property(config_key='MW_APP_VERSION')
+    def APP_VERSION(self, config_value: str) -> str:
+        return config_value
+
     @env_property(config_key='MW_APP_DEBUG_MODE')
     def DEBUG_MODE(self, config_value: str) -> bool:
         return config_value == '1'

mapping_workbench/backend/core/entrypoints/api/routes.py

@@ -1,6 +1,9 @@
 from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
 
+from mapping_workbench.backend.config import settings
+from mapping_workbench.backend.core.models.api_response import AppSettingsResponse
+
 router = APIRouter()
 
 
@@ -12,3 +15,14 @@ async def index(request: Request) -> JSONResponse:
     return JSONResponse(content={
         "message": "Welcome to Mapping Workbench API!"
     })
+
+
+@router.get(
+    "/app/settings",
+    name="app:version",
+    response_model=AppSettingsResponse
+)
+async def app_settings() -> AppSettingsResponse:
+    return AppSettingsResponse(
+        version=settings.APP_VERSION
+    )

mapping_workbench/backend/core/models/api_response.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional
 
 from beanie import PydanticObjectId
 from pydantic import BaseModel
@@ -11,10 +11,15 @@ class APIEmptyContentResponse(BaseModel):
 class APIEmptyContentWithIdResponse(BaseModel):
     id: PydanticObjectId
 
+
 class APIEmptyContentWithStrIdResponse(BaseModel):
     id: str
 
 
 class APIListPaginatedResponse(BaseModel):
     items: List
     count: int
+
+
+class AppSettingsResponse(BaseModel):
+    version: Optional[str] = None

mapping_workbench/backend/core/scripts/get_app_version.py

@@ -0,0 +1,44 @@
+import subprocess
+
+
+def get_current_app_tag():
+    try:
+        # Get the tag of the current commit (if any)
+        current_tag = subprocess.check_output(
+            ["git", "describe", "--exact-match", "--tags"],
+            stderr=subprocess.DEVNULL
+        ).strip().decode('utf-8')
+        return current_tag
+    except subprocess.CalledProcessError:
+        return None
+
+
+def get_current_app_branch():
+    try:
+        return subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip().decode('utf-8')
+    except subprocess.CalledProcessError:
+        return None
+
+
+def get_current_app_tag_or_branch():
+    tag = get_current_app_tag()
+    if tag:
+        return tag
+    else:
+        branch = get_current_app_branch()
+        if branch:
+            return branch
+        else:
+            return None
+
+
+def get_current_app_version():
+    return get_current_app_tag_or_branch()
+
+
+def main():
+    return get_current_app_version() or ""
+
+
+if __name__ == "__main__":
+    print(main())

mapping_workbench/backend/core/services/project_initilisers.py

@@ -21,7 +21,7 @@
 from mapping_workbench.backend.triple_map_fragment.models.entity import SpecificTripleMapFragment, \
     GenericTripleMapFragment
 from mapping_workbench.backend.triple_map_registry.models.entity import TripleMapRegistry
-from mapping_workbench.backend.user.models.user import User
+from mapping_workbench.backend.user.models.user import User, Role
 from mapping_workbench.backend.xsd_schema.models.xsd_file_resource import XSDFileResource
 
 
@@ -30,13 +30,13 @@ async def init_admin_user() -> None:
         email=settings.DATABASE_ADMIN_NAME,
         hashed_password=settings.DATABASE_ADMIN_HASHED_PASSWORD,
         name="admin",
+        is_active=True,
         is_superuser=True,
-        is_verified=True
+        is_verified=True,
+        roles=[Role.ADMIN]
     )
 
-    if await User.find_one(
-            User.email == admin_user.email
-    ).count() == 0:
+    if await User.find_one(User.email == admin_user.email).count() == 0:
         await admin_user.create()
     return
 

mapping_workbench/backend/fields_registry/entrypoints/api/routes.py

@@ -8,13 +8,14 @@
     APIEmptyContentWithStrIdResponse
 from mapping_workbench.backend.fields_registry.models.field_registry import StructuralElement, \
     APIListStructuralElementsPaginatedResponse, APIListStructuralElementsVersionedViewPaginatedResponse, \
-    StructuralElementsVersionedView, StructuralElementLabelOut, StructuralElementIn
+    StructuralElementsVersionedView, StructuralElementLabelOut, BaseStructuralElementIn, StructuralElementOut, \
+    StructuralElementIn
 from mapping_workbench.backend.fields_registry.services import tasks
 from mapping_workbench.backend.fields_registry.services.api import list_structural_elements_versioned_view, \
     get_structural_elements_versioned_view, \
     delete_structural_elements_versioned_view, get_structural_elements_versioned_view_by_version, \
     list_structural_elements, get_structural_element, delete_structural_element, get_project_structural_elements, \
-    get_structural_element_label_list, insert_structural_element
+    get_structural_element_label_list, insert_structural_element, create_structural_element, update_structural_element
 from mapping_workbench.backend.fields_registry.services.data import tree_of_structural_elements
 from mapping_workbench.backend.fields_registry.services.generate_conceptual_mapping_rules import \
     generate_conceptual_mapping_rules
@@ -75,6 +76,34 @@ async def route_get_structural_element(structural_element: StructuralElement = D
     return structural_element
 
 
+@router.post(
+    "/elements/create",
+    description=f"Create element",
+    name=f"elements:create_element",
+    response_model=StructuralElementOut,
+    status_code=status.HTTP_201_CREATED
+)
+async def route_create_structural_element(
+        data: StructuralElementIn,
+        user: User = Depends(current_active_user)
+):
+    return await create_structural_element(data, user=user)
+
+
+@router.patch(
+    "/elements/{structural_element_id}",
+    description=f"Update element",
+    name=f"elements:update_element",
+    response_model=StructuralElementOut
+)
+async def route_update_structural_element(
+        data: StructuralElementIn,
+        element: StructuralElement = Depends(get_structural_element),
+        user: User = Depends(current_active_user)
+):
+    return await update_structural_element(element, data, user=user)
+
+
 @router.delete(
     "/elements/{structural_element_id}",
     description=f"Delete structural element by id",
@@ -226,7 +255,7 @@ async def route_get_structural_elements_label(
     tags=[TAG],
     status_code=status.HTTP_201_CREATED
 )
-async def route_post_structural_element(project_id: PydanticObjectId, structural_element_in: StructuralElementIn):
+async def route_post_structural_element(project_id: PydanticObjectId, structural_element_in: BaseStructuralElementIn):
     try:
         return await insert_structural_element(structural_element_in=structural_element_in, project_id=project_id)
     except (Exception,) as expected_exception:

mapping_workbench/backend/fields_registry/models/eforms_fields_structure.py

@@ -1,5 +1,6 @@
 import hashlib
 from typing import List, Optional
+
 from pydantic import BaseModel, Field
 
 
@@ -27,6 +28,19 @@ def generate_hash_id(self, project_id: str = None) -> str:
         return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest())
 
 
+def generate_eforms_node_hash_id(
+        id: str,
+        repeatable: bool,
+        parent_id: Optional[str] = None,
+        xpath_absolute: str = None,
+        xpath_relative: str = None,
+        project_id: str = None
+):
+    fields_to_hash = [project_id, id, xpath_absolute, xpath_relative, repeatable, parent_id]
+    str_content = "_".join(map(str, fields_to_hash))
+    return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest())
+
+
 class EFormsNode(BaseModel):
     id: str
     parent_id: Optional[str] = Field(default=None, alias='parentId')
@@ -35,11 +49,14 @@ class EFormsNode(BaseModel):
     repeatable: bool
 
     def generate_hash_id(self, project_id: str = None):
-        fields_to_hash = [
-            project_id, self.id, self.xpath_absolute, self.xpath_relative, self.repeatable, self.parent_id
-        ]
-        str_content = "_".join(map(str, fields_to_hash))
-        return str(hashlib.sha1(str_content.encode("utf-8")).hexdigest())
+        return generate_eforms_node_hash_id(
+            id=self.id,
+            repeatable=self.repeatable,
+            parent_id=self.parent_id,
+            xpath_absolute=self.xpath_absolute,
+            xpath_relative=self.xpath_relative,
+            project_id=project_id
+        )
 
 
 class EFormsSDKFields(BaseModel):

mapping_workbench/backend/fields_registry/models/field_registry.py

@@ -7,7 +7,7 @@
 
 from mapping_workbench.backend.core.models.api_response import APIListPaginatedResponse
 from mapping_workbench.backend.core.models.base_project_resource_entity import BaseProjectResourceEntity, \
-    BaseProjectResourceEntityOutSchema
+    BaseProjectResourceEntityOutSchema, BaseProjectResourceEntityInSchema
 from mapping_workbench.backend.state_manager.models.state_object import StatefulObjectABC, ObjectState
 
 
@@ -19,6 +19,18 @@ class StructuralElementLabelOut(BaseModel):
     sdk_element_id: str
 
 
+class StructuralElementIn(BaseProjectResourceEntityInSchema):
+    """
+        General input model for Structural Element
+    """
+    sdk_element_id: Optional[str] = None
+    absolute_xpath: Optional[str] = None
+    relative_xpath: Optional[str] = None
+    parent_node_id: Optional[str] = None
+    name: Optional[str] = None
+    bt_id: Optional[str] = None
+
+
 class StructuralElementOut(BaseProjectResourceEntityOutSchema):
     """
 
@@ -139,7 +151,7 @@ class APIListStructuralElementsVersionedViewPaginatedResponse(APIListPaginatedRe
     items: List[StructuralElementsVersionedView]
 
 
-class StructuralElementIn(BaseModel):
+class BaseStructuralElementIn(BaseModel):
     """
         General input model for Structural Element
     """

mapping_workbench/backend/fields_registry/services/api.py

@@ -2,14 +2,19 @@
 
 from beanie import PydanticObjectId
 from beanie.odm.operators.update.general import Set
+from pymongo.errors import DuplicateKeyError
 
 from mapping_workbench.backend.conceptual_mapping_rule.models.entity import ConceptualMappingRule
 from mapping_workbench.backend.core.models.base_entity import BaseEntityFiltersSchema
-from mapping_workbench.backend.core.services.exceptions import ResourceNotFoundException
-from mapping_workbench.backend.core.services.request import api_entity_is_found, prepare_search_param, pagination_params
+from mapping_workbench.backend.core.services.exceptions import ResourceNotFoundException, DuplicateKeyException
+from mapping_workbench.backend.core.services.request import api_entity_is_found, prepare_search_param, \
+    pagination_params, request_create_data, request_update_data
+from mapping_workbench.backend.fields_registry.models.eforms_fields_structure import generate_eforms_node_hash_id
 from mapping_workbench.backend.fields_registry.models.field_registry import StructuralElement, \
-    StructuralElementsVersionedView, StructuralElementOut, StructuralElementLabelOut, StructuralElementIn
+    StructuralElementsVersionedView, StructuralElementOut, StructuralElementLabelOut, BaseStructuralElementIn, \
+    StructuralElementIn
 from mapping_workbench.backend.project.models.entity import Project
+from mapping_workbench.backend.user.models.user import User
 
 
 async def list_structural_elements(filters: dict = None, page: int = None, limit: int = None) -> \
@@ -112,8 +117,42 @@ async def get_structural_element_label_list(project_id: PydanticObjectId) -> Lis
     ).to_list()
 
 
-async def insert_structural_element(structural_element_in: StructuralElementIn, project_id: PydanticObjectId) -> None:
+async def insert_structural_element(structural_element_in: BaseStructuralElementIn,
+                                    project_id: PydanticObjectId) -> None:
     structural_element_in_json = structural_element_in.model_dump()
     structural_element = StructuralElement.model_validate(structural_element_in_json)
     structural_element.project = Project.link_from_id(project_id)
     await structural_element.save()
+
+
+async def create_structural_element(
+        data: StructuralElementIn,
+        user: User
+) -> StructuralElementOut:
+    element: StructuralElement = \
+        StructuralElement(
+            **request_create_data(data, user=user)
+        )
+    element.id = generate_eforms_node_hash_id(
+        id=element.sdk_element_id,
+        parent_id=element.parent_node_id,
+        xpath_absolute=element.absolute_xpath,
+        xpath_relative=element.relative_xpath,
+        repeatable=element.repeatable,
+        project_id=data.project.to_ref().id
+    )
+    try:
+        await element.create()
+    except DuplicateKeyError as e:
+        raise DuplicateKeyException(e)
+    return StructuralElementOut(**element.model_dump())
+
+
+async def update_structural_element(
+        element: StructuralElement,
+        data: StructuralElementIn,
+        user: User
+) -> StructuralElementOut:
+    return StructuralElementOut(**(
+        await element.set(request_update_data(data, user=user))
+    ).model_dump())

mapping_workbench/backend/security/entrypoints/api/routes.py

@@ -35,7 +35,7 @@
                                    auth_backend,
                                    settings.JWT_SECRET,
                                    associate_by_email=True,
-                                   is_verified_by_default=True,
+                                   is_verified_by_default=False,
 
                                    # Redirect url to user according to the OAuth2.0 protocol flow
                                    # see: https://datatracker.ietf.org/doc/html/rfc6749#section-1.2