From e049ba3af31835d297cf39ac5466cd15aff60579 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Thu, 11 Jan 2024 14:49:01 +0100 Subject: [PATCH 01/99] Add models for RBAC --- .../agenta_backend/models/db_models.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index af8ab847b1..25b71a598a 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -1,3 +1,4 @@ +from enum import Enum from uuid import uuid4 from datetime import datetime from typing import Any, Dict, List, Optional, Union @@ -6,6 +7,35 @@ from beanie import Document, Link, PydanticObjectId +class WorkspaceRole(str, Enum): + OWNER = "owner" + ADMIN = "admin" + MEMBER = "member" + + +class Permission(str, Enum): + CREATE_APPLICATION = "create_application" + DELETE_APPLICATION = "delete_application" + MODIFY_CONFIGURATIONS = "modify_configurations" + DEPLOY_APPLICATION = "deploy_application" + RUN_EVALUATIONS = "run_evaluations" + CHANGE_USER_ROLES = "change_user_roles" + CREATE_NEW_WORKSPACE = "create_workspace" + ADD_NEW_USER_TO_ORGANIZATION = "add_new_user_to_organization" + ADD_NEW_USER_TO_WORKSPACE = "add_new_user_to_workspace" + MODIFY_USER_ROLES = "modify_user_roles" + + +class WorkspacePermissionDB(BaseModel): + role_name: WorkspaceRole + permissions: List[Permission] + + +class WorkspaceMemberDB(BaseModel): + user_id: PydanticObjectId + roles: List[WorkspacePermissionDB] + + class APIKeyDB(Document): prefix: str hashed_key: str @@ -34,6 +64,7 @@ class OrganizationDB(Document): owner: str # user id members: Optional[List[PydanticObjectId]] invitations: Optional[List[InvitationDB]] = [] + workspaces: Optional[List[PydanticObjectId]] = [] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -41,11 +72,52 @@ class Settings: name = "organizations" +class WorkspaceDB(Document): + name: str + type: Optional[str] + description: str = Field(default="") + organization: Link[OrganizationDB] + members: Optional[List[WorkspaceMemberDB]] = [] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + def get_member_roles(self, user_id: PydanticObjectId) -> Optional[List[WorkspacePermissionDB]]: + for member in self.members: + if member.user_id == user_id: + return member.roles + return None + + def get_member_role_names(self, user_id: PydanticObjectId) -> List[str]: + roles = self.get_member_roles(user_id) + return [role.role_name for role in roles] if roles else [] + + def get_all_members(self) -> List[PydanticObjectId]: + return [member.user_id for member in self.members] + + def has_permission(self, user_id: PydanticObjectId, permission: Permission) -> bool: + roles = self.get_member_roles(user_id) + if roles: + for role in roles: + if permission in role.permissions: + return True + return False + + def is_owner(self, user_id: PydanticObjectId) -> bool: + for member in self.members: + if member.user_id == user_id and WorkspaceRole.OWNER in self.get_member_role_names(user_id): + return True + return False + + class Settings: + name = "workspaces" + + class UserDB(Document): uid: str = Field(default="0", unique=True, index=True) username: str = Field(default="agenta") email: str = Field(default="demo@agenta.ai", unique=True) organizations: Optional[List[PydanticObjectId]] = [] + workspaces: Optional[List[PydanticObjectId]] = [] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -63,6 +135,7 @@ class ImageDB(Document): deletable: bool = Field(default=True) user: Link[UserDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) deletable: bool = Field(default=True) @@ -74,6 +147,7 @@ class Settings: class AppDB(Document): app_name: str organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] user: Link[UserDB] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -85,6 +159,7 @@ class Settings: class DeploymentDB(Document): app: Link[AppDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] user: Link[UserDB] container_name: Optional[str] container_id: Optional[str] @@ -100,6 +175,7 @@ class Settings: class VariantBaseDB(Document): app: Link[AppDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] user: Link[UserDB] base_name: str image: Link[ImageDB] @@ -136,6 +212,7 @@ class AppVariantDB(Document): image: Link[ImageDB] user: Link[UserDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] parameters: Dict[str, Any] = Field(default=dict) # TODO: deprecated. remove previous_variant_name: Optional[str] # TODO: deprecated. remove base_name: Optional[str] @@ -158,6 +235,7 @@ class AppEnvironmentDB(Document): name: str user: Link[UserDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] deployed_app_variant: Optional[PydanticObjectId] deployment: Optional[PydanticObjectId] # reference to deployment created_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -188,6 +266,7 @@ class TestSetDB(Document): csvdata: List[Dict[str, str]] user: Link[UserDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -198,6 +277,7 @@ class Settings: class EvaluatorConfigDB(Document): app: Link[AppDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] user: Link[UserDB] name: str evaluator_key: str @@ -248,6 +328,7 @@ class HumanEvaluationScenarioOutput(BaseModel): class HumanEvaluationDB(Document): app: Link[AppDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] user: Link[UserDB] status: str evaluation_type: str @@ -263,6 +344,7 @@ class Settings: class HumanEvaluationScenarioDB(Document): user: Link[UserDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] evaluation: Link[HumanEvaluationDB] inputs: List[HumanEvaluationScenarioInput] outputs: List[HumanEvaluationScenarioOutput] @@ -281,6 +363,7 @@ class Settings: class EvaluationDB(Document): app: Link[AppDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] user: Link[UserDB] status: str = Field(default="EVALUATION_INITIALIZED") testset: Link[TestSetDB] @@ -297,6 +380,7 @@ class Settings: class EvaluationScenarioDB(Document): user: Link[UserDB] organization: Link[OrganizationDB] + workspace: Link[WorkspaceDB] evaluation: Link[EvaluationDB] variant_id: PydanticObjectId inputs: List[EvaluationScenarioInputDB] From d3efdf40bbd38b357782d5eea1f75c14b4b0493c Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 09:18:27 +0100 Subject: [PATCH 02/99] Add Workspace to Database Engine --- agenta-backend/agenta_backend/models/db_engine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index b951be23a0..b2cf48c3e2 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -10,6 +10,7 @@ APIKeyDB, AppEnvironmentDB, OrganizationDB, + WorkspaceDB, UserDB, ImageDB, AppDB, @@ -37,6 +38,7 @@ APIKeyDB, AppEnvironmentDB, OrganizationDB, + WorkspaceDB, UserDB, ImageDB, AppDB, From a92cdce4ee92714925cbb24ac4679a7663203f30 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 09:18:51 +0100 Subject: [PATCH 03/99] Set description to be optional --- agenta-backend/agenta_backend/models/db_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 25b71a598a..7905896f87 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -59,7 +59,7 @@ class InvitationDB(BaseModel): class OrganizationDB(Document): name: str = Field(default="agenta") - description: str = Field(default="") + description: Optional[str] = Field(default="") type: Optional[str] owner: str # user id members: Optional[List[PydanticObjectId]] @@ -75,7 +75,7 @@ class Settings: class WorkspaceDB(Document): name: str type: Optional[str] - description: str = Field(default="") + description: Optional[str] = Field(default="") organization: Link[OrganizationDB] members: Optional[List[WorkspaceMemberDB]] = [] created_at: Optional[datetime] = Field(default=datetime.utcnow()) From c7063460474918ec75371638b4f05efdea3b600f Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 09:19:13 +0100 Subject: [PATCH 04/99] Define CreateOrganization --- .../agenta_backend/models/api/organization_models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/agenta-backend/agenta_backend/models/api/organization_models.py b/agenta-backend/agenta_backend/models/api/organization_models.py index c4151d697d..e7388e49ea 100644 --- a/agenta-backend/agenta_backend/models/api/organization_models.py +++ b/agenta-backend/agenta_backend/models/api/organization_models.py @@ -17,11 +17,19 @@ class Organization(BaseModel): owner: str members: Optional[List[str]] invitations: Optional[List] + + +class CreateOrganization(BaseModel): + name: str + description: Optional[str] + type: Optional[str] + owner: Optional[str] class OrganizationUpdate(BaseModel): name: Optional[str] description: Optional[str] + updated_at: Optional[datetime] class OrganizationOutput(BaseModel): From 0a0b6e0e2bfc44d855573ef7b3aef3e171001740 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 09:19:43 +0100 Subject: [PATCH 05/99] Create pydantic basemodel for Workspace --- .../models/api/workspace_models.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 agenta-backend/agenta_backend/models/api/workspace_models.py diff --git a/agenta-backend/agenta_backend/models/api/workspace_models.py b/agenta-backend/agenta_backend/models/api/workspace_models.py new file mode 100644 index 0000000000..8008b65dd0 --- /dev/null +++ b/agenta-backend/agenta_backend/models/api/workspace_models.py @@ -0,0 +1,25 @@ +from datetime import datetime +from bson import ObjectId +from typing import Optional, List +from pydantic import BaseModel, Field + + +class Workspace(BaseModel): + id: Optional[str] + name: str + description: Optional[str] + type: Optional[str] + + +class CreateWorkspace(BaseModel): + name: str + description: Optional[str] + type: Optional[str] + organization_id: str + + +class WorkspaceUpdate(BaseModel): + name: Optional[str] + description: Optional[str] + updated_at: Optional[datetime] + From f990e6b2b9b7e7ff8ae4599ea1eb783804bfc3f0 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 09:20:16 +0100 Subject: [PATCH 06/99] Feat: create workspace --- .../agenta_backend/services/db_manager.py | 82 ++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index adbac48879..964d15fa31 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -11,6 +11,10 @@ ImageExtended, Template, ) +from agenta_backend.models.api.workspace_models import ( + Workspace, + CreateWorkspace, +) from agenta_backend.models.converters import ( app_db_to_pydantic, image_db_to_pydantic, @@ -40,6 +44,11 @@ TemplateDB, TestSetDB, UserDB, + Permission, + WorkspaceDB, + WorkspaceRole, + WorkspaceMemberDB, + WorkspacePermissionDB ) from agenta_backend.utils.common import check_user_org_access from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum @@ -568,6 +577,55 @@ async def list_variants_for_base( return app_variants_db +async def get_workspace(workspace_id: str) -> Workspace: + """ + Retrieve a workspace. + + Args: + workspace_id (str): The workspace id. + + Returns: + Workspace: The retrieved workspace. + """ + workspace = await WorkspaceDB.find_one(WorkspaceDB.id == ObjectId(workspace_id)) + return workspace + + +async def create_workspace(payload: CreateWorkspace, user) -> WorkspaceDB: + """Create a new workspace. + + Args: + payload (Workspace): The workspace payload. + user (UserDB): The user that the workspace belongs to. + + Returns: + Workspace: The created workspace. + """ + organization = await get_organization_object(payload.organization_id) + + # create default workspace + workspace = WorkspaceDB( + name=payload.name, + type=payload.type if payload.type else "", + description=payload.description if payload.description else "", + organization=organization + ) + + # Assign the creator as the owner with all permissions + workspace.members = WorkspaceMemberDB( + user_id = user.id, + roles = [WorkspacePermissionDB( + role_name=WorkspaceRole.OWNER, + permissions=list(Permission)) + ] + ) + await workspace.create() + logger.info(f"Created workspace {workspace} for organization {organization.id}") + + return workspace + + + async def get_user(user_uid: str) -> UserDB: """Get the user object from the database. @@ -581,15 +639,33 @@ async def get_user(user_uid: str) -> UserDB: user = await UserDB.find_one(UserDB.uid == user_uid) if user is None: if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"]: + + # create user user_db = UserDB(uid="0") user = await user_db.create() + # create default organization for user org_db = OrganizationDB(type="default", owner=str(user.id)) org = await org_db.create() - + + # create default workspace for user + workspace_payload = CreateWorkspace( + name=org_db.name, + type=org_db.type, + description="My Default Workspace", + organization_id=str(org_db.id) + ) + workspace = await create_workspace(workspace_payload, user) + + # update organization with default workspace id + org_db.workspaces = [workspace.id] + await org_db.update({"$set": org_db.dict(exclude_unset=True)}) + + # update user with organization and workspace user_db.organizations.append(org.id) - await user_db.save() - + user_db.workspaces.append(workspace.id) + await user_db.update({"$set": user_db.dict(exclude_unset=True)}) + return user raise Exception("Please login or signup") return user From 7de5b3502171a0148e332a1be68290cb6dbb5b25 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 12:01:24 +0100 Subject: [PATCH 07/99] Add new permissions and method to check for role access --- .../agenta_backend/models/db_models.py | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 7905896f87..2221e52a40 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -9,21 +9,49 @@ class WorkspaceRole(str, Enum): OWNER = "owner" - ADMIN = "admin" MEMBER = "member" + WORKSPACE_ADMIN = "workspace_admin" class Permission(str, Enum): + # general + MODIFY_CONFIGURATIONS = "modify_configurations" + CHANGE_USER_ROLES = "change_user_roles" + READ_SYSTEM = "read_system" + + VIEW_APPLICATION = "view_application" CREATE_APPLICATION = "create_application" + EDIT_APPLICATION = "edit_application" DELETE_APPLICATION = "delete_application" - MODIFY_CONFIGURATIONS = "modify_configurations" - DEPLOY_APPLICATION = "deploy_application" + + # Testset + VIEW_TESTSET = "view_testset" + CREATE_TESTSET = "create_testset" + EDIT_TESTSET = "edit_testset" + DELETE_TESTSET = "delete_testset" + + # Evaluation + VIEW_EVALUATION = "view_evaluation" + CREATE_EVALUATION = "create_evaluation" RUN_EVALUATIONS = "run_evaluations" - CHANGE_USER_ROLES = "change_user_roles" - CREATE_NEW_WORKSPACE = "create_workspace" - ADD_NEW_USER_TO_ORGANIZATION = "add_new_user_to_organization" - ADD_NEW_USER_TO_WORKSPACE = "add_new_user_to_workspace" + EDIT_EVALUATION = "edit_evaluation" + DELETE_EVALUATION = "delete_evaluation" + + # Deployment + DEPLOY_APPLICATION = "deploy_application" + + # Workspace + VIEW_WORKSPACE = "view_workspace" + CREATE_WORKSPACE = "create_workspace" + EDIT_WORKSPACE = "edit_workspace" + DELETE_WORKSPACE = "delete_workspace" MODIFY_USER_ROLES = "modify_user_roles" + ADD_NEW_USER_TO_WORKSPACE = "add_new_user_to_workspace" + + # Organization + ADD_NEW_USER_TO_ORGANIZATION = "add_new_user_to_organization" + EDIT_ORGANIZATION = "edit_organizayion" + class WorkspacePermissionDB(BaseModel): @@ -101,6 +129,14 @@ def has_permission(self, user_id: PydanticObjectId, permission: Permission) -> b if permission in role.permissions: return True return False + + def has_role(self, user_id: PydanticObjectId, role_to_check: WorkspaceRole) -> bool: + roles = self.get_member_roles(user_id) + if roles: + for role in roles: + if role.role_name == role_to_check: + return True + return False def is_owner(self, user_id: PydanticObjectId) -> bool: for member in self.members: From a978d3fb5d968c6275f56a6296b78ca66f4f9ad4 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 12:02:40 +0100 Subject: [PATCH 08/99] include user id in get_user_and_org_id --- agenta-backend/agenta_backend/services/selectors.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/agenta-backend/agenta_backend/services/selectors.py b/agenta-backend/agenta_backend/services/selectors.py index bae85cc8b5..2e111a31b2 100644 --- a/agenta-backend/agenta_backend/services/selectors.py +++ b/agenta-backend/agenta_backend/services/selectors.py @@ -15,8 +15,8 @@ async def get_user_and_org_id(user_uid_id) -> Dict[str, List]: Returns: A dictionary containing the user_id and a list of the user's organization_ids. """ - user_id, org_ids = await get_user_objectid(user_uid_id) - return {"uid": user_id, "organization_ids": org_ids} + user, org_ids = await get_user_objectid(user_uid_id) + return {"uid": str(user.uid), "id": str(user.id), "organization_ids": org_ids} async def get_user_objectid(user_uid: str) -> Tuple[str, List]: @@ -33,11 +33,10 @@ async def get_user_objectid(user_uid: str) -> Tuple[str, List]: user = await UserDB.find_one(UserDB.uid == user_uid) if user is not None: - user_id = str(user.uid) organization_ids: List = ( [org for org in user.organizations] if user.organizations else [] ) - return user_id, organization_ids + return user, organization_ids return None, [] From fb514cea15620d97fc799e7b814a6db3790fa564 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 12:03:18 +0100 Subject: [PATCH 09/99] define fine-grained rbac function check --- agenta-backend/agenta_backend/utils/common.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index 4ed0f23c82..90aa8f81fa 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -10,6 +10,9 @@ OrganizationDB, AppDB, VariantBaseDB, + WorkspaceDB, + Permission, + WorkspaceRole ) from beanie import PydanticObjectId as ObjectId @@ -166,3 +169,66 @@ async def check_access_to_base( return False organization_id = base.organization.id return await check_user_org_access(user_org_data, str(organization_id), check_owner) + + +async def check_rbac_permission( + user_org_data: Dict[str, str, Union[str, list]], + workspace_id: ObjectId, + provided_organization_id: ObjectId, + permission: Permission = None, + role: str = None, +) -> bool: + """ + Check if a user belongs to a workspace and has a certain permission. + + Args: + workspace_id (ObjectId): The ID of the workspace to check. + user_id (ObjectId): The user's ID. + permission (Permission): The permission to check. + + Returns: + bool: True if the user belongs to the workspace and has the specified permission, False otherwise. + """ + if permission is None and role is None: + raise Exception("Either permission or role must be provided") + elif permission is not None and role is not None: + raise Exception("Only one of permission or role must be provided") + + # Retrieve the workspace object using the provided workspace_id + workspace = await WorkspaceDB.find_one(WorkspaceDB.id == workspace_id) + if workspace is None: + raise Exception("Workspace not found") + + provided_organization = await get_organization(str(provided_organization_id)) + if provided_organization is None: + raise Exception("Organization not found") + + # confirm that the workspace belongs to the provided organization + if str(workspace.organization.id) != provided_organization_id: + raise Exception("Workspace does not belong to the provided organization") + + # get workspace organization and check if user belongs to it + workspace_organization_id = workspace.organization.id + if not check_user_org_access(user_org_data, str(workspace_organization_id)): + return False + + # Check if the user belongs to the workspace + if user_id not in workspace.get_all_members(): + return False + + # Check if user is the owner of the workspace ( they have all permissions ) + user_id = user_org_data["id"] + if workspace.is_owner(user_id): + return True + + # Check if the user has the specified permission or role + if role is not None: + # validate role exists + if role not in list(WorkspaceRole): + raise Exception("Invalid role specified") + return workspace.has_role(user_id, role) + else: + # validate permission exists + if permission not in list(Permission): + raise Exception("Invalid permission specified") + return workspace.has_permission(user_id, permission) From b755b015233836ba6d151acd541af194f539edf3 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 13:08:59 +0100 Subject: [PATCH 10/99] Add Workspace when creating dm_manager objects --- .../agenta_backend/services/db_manager.py | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 964d15fa31..108d1e2738 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -202,8 +202,7 @@ async def fetch_app_variant_by_id( async def fetch_base_by_id( - base_id: str, - user_org_data: dict, + base_id: str ) -> Optional[VariantBaseDB]: """ Fetches a base by its ID. @@ -218,13 +217,6 @@ async def fetch_base_by_id( if base is None: logger.error("Base not found") return False - organization_id = base.organization.id - access = await check_user_org_access( - user_org_data, str(organization_id), check_owner=False - ) - if not access: - logger.error("User does not have access to this base") - return False return base @@ -252,6 +244,7 @@ async def fetch_app_variant_by_name_and_appid( async def create_new_variant_base( app: AppDB, organization: OrganizationDB, + workspace: WorkspaceDB, user: UserDB, base_name: str, image: ImageDB, @@ -260,6 +253,10 @@ async def create_new_variant_base( Args: base_name (str): The name of the base. image (ImageDB): The image of the base. + user (UserDB): The User Object creating the variant. + app (AppDB): The associated App Object. + organization (OrganizationDB): The Organization the variant belongs to. + workspace (WorkspaceDB): The Workspace the variant belongs to. Returns: VariantBaseDB: The created base. """ @@ -267,6 +264,7 @@ async def create_new_variant_base( base = VariantBaseDB( app=app, organization=organization, + workspace=workspace, user=user, base_name=base_name, image=image, @@ -303,6 +301,7 @@ async def create_new_config( async def create_new_app_variant( app: AppDB, organization: OrganizationDB, + workspace: WorkspaceDB, user: UserDB, variant_name: str, image: ImageDB, @@ -324,6 +323,7 @@ async def create_new_app_variant( variant = AppVariantDB( app=app, organization=organization, + workspace=workspace, user=user, variant_name=variant_name, image=image, @@ -342,6 +342,7 @@ async def create_image( user: UserDB, deletable: bool, organization: OrganizationDB, + workspace: WorkspaceDB, template_uri: str = None, docker_id: str = None, tags: str = None, @@ -353,6 +354,7 @@ async def create_image( user (UserDB): The user that the image belongs to. deletable (bool): Whether the image can be deleted. organization (OrganizationDB): The organization that the image belongs to. + workspace (WorkspaceDB): The workspace that the image belongs to. Returns: ImageDB: The created image. """ @@ -388,6 +390,7 @@ async def create_image( deletable=deletable, user=user, organization=organization, + workspace=workspace, ) await image.create() return image @@ -396,6 +399,7 @@ async def create_image( async def create_deployment( app: AppVariantDB, organization: OrganizationDB, + workspace: WorkspaceDB, user: UserDB, container_name: str, container_id: str, @@ -406,6 +410,7 @@ async def create_deployment( Args: app (AppVariantDB): The app variant to create the deployment for. organization (OrganizationDB): The organization that the deployment belongs to. + workspace (WorkspaceDB): The Workspace that the deployment belongs to. user (UserDB): The user that the deployment belongs to. container_name (str): The name of the container. container_id (str): The ID of the container. @@ -417,6 +422,7 @@ async def create_deployment( deployment = DeploymentDB( app=app, organization=organization, + workspace=workspace, user=user, container_name=container_name, container_id=container_id, @@ -461,22 +467,6 @@ async def create_app_and_envs( return app -async def create_user_organization(user_uid: str) -> OrganizationDB: - """Create a default organization for a user. - - Args: - user_uid (str): The uid of the user - - Returns: - OrganizationDB: Instance of OrganizationDB - """ - - user = await UserDB.find_one(UserDB.uid == user_uid) - org_db = OrganizationDB(owner=str(user.id), type="default") - await org_db.create() - return org_db - - async def get_deployment_by_objectid( deployment_id: ObjectId, ) -> DeploymentDB: @@ -1680,6 +1670,7 @@ async def fetch_app_by_name_and_organization( async def create_new_evaluation( app: AppDB, organization: OrganizationDB, + workspace: WorkspaceDB, user: UserDB, testset: TestSetDB, status: str, @@ -1693,6 +1684,7 @@ async def create_new_evaluation( evaluation = EvaluationDB( app=app, organization=organization, + workspace=workspace, user=user, testset=testset, status=status, @@ -1709,6 +1701,7 @@ async def create_new_evaluation( async def create_new_evaluation_scenario( user: UserDB, organization: OrganizationDB, + workspace: WorkspaceDB, evaluation: EvaluationDB, variant_id: str, inputs: List[EvaluationScenarioInputDB], @@ -1726,6 +1719,7 @@ async def create_new_evaluation_scenario( evaluation_scenario = EvaluationScenarioDB( user=user, organization=organization, + workspace=workspace, evaluation=evaluation, variant_id=ObjectId(variant_id), inputs=inputs, @@ -1844,6 +1838,7 @@ async def create_evaluator_config( app: AppDB, user: UserDB, organization: OrganizationDB, + workspace: WorkspaceDB, name: str, evaluator_key: str, settings_values: Optional[Dict[str, Any]] = None, @@ -1854,6 +1849,7 @@ async def create_evaluator_config( app=app, user=user, organization=organization, + workspace=workspace, name=name, evaluator_key=evaluator_key, settings_values=settings_values, From 3bd8c7165db7776674d18faa7de82179ee6077bd Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 12 Jan 2024 13:10:11 +0100 Subject: [PATCH 11/99] define function to get workspace and organization id --- .../agenta_backend/services/db_manager.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 108d1e2738..0f0e7f2746 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -1922,3 +1922,76 @@ async def update_evaluation( setattr(evaluation, key, value) await evaluation.save() return evaluation + + +async def get_object_workspace_org_id(object_id: str, type: str) -> str: + """ + Get the organization and workspace id of the object. + + Args: + object_id (str): The ID of the object. + type (str): The type of the object. + + Returns: + dict: The organization and workspace id of the object. + """ + try: + if type == "app": + app = await fetch_app_by_id(object_id) + organization_id = str(app.organization.id) + workspace_id = str(app.workspace.id) + + elif type == "app_variant": + app_variant = await fetch_app_variant_by_id(object_id) + organization_id = str(app_variant.organization.id) + workspace_id = str(app_variant.workspace.id) + + elif type == "base": + base = await fetch_base_by_id(object_id) + organization_id = str(base.organization.id) + workspace_id = str(base.workspace.id) + + elif type == "deployment": + deployment = await get_deployment_by_objectid(object_id) + organization_id = str(deployment.organization.id) + workspace_id = str(deployment.workspace.id) + + elif type == "testset": + testset = await fetch_testset_by_id(object_id) + organization_id = str(testset.organization.id) + workspace_id = str(testset.workspace.id) + + elif type == "evaluation": + evaluation = await fetch_evaluation_by_id(object_id) + organization_id = str(evaluation.organization.id) + workspace_id = str(evaluation.workspace.id) + + elif type == "evaluation_scenario": + evaluation_scenario = await fetch_evaluation_scenario_by_id(object_id) + organization_id = str(evaluation_scenario.organization.id) + workspace_id = str(evaluation_scenario.workspace.id) + + elif type == "evaluator_config": + evaluator_config = await fetch_evaluator_config(object_id) + organization_id = str(evaluator_config.organization.id) + workspace_id = str(evaluator_config.workspace.id) + + elif type == "human_evaluation": + human_evaluation = await fetch_human_evaluation_by_id(object_id) + organization_id = str(human_evaluation.human_evaluation.id) + workspace_id = str(human_evaluation.human_evaluation.id) + + elif type == "human_evaluation_scenario": + human_evaluation_scenario = await fetch_human_evaluation_scenario_by_id( + object_id + ) + organization_id = str(human_evaluation_scenario.organization.id) + workspace_id = str(human_evaluation_scenario.workspace.id) + + else: + raise ValueError(f"Unknown object type: {type}") + + return {"organization_id": organization_id, "workspace_id": workspace_id} + except Exception as e: + raise e + From 0de3b88ae56df9b7bb1b31284d5815c2fcb52101 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 14:14:13 +0100 Subject: [PATCH 12/99] add methods to get member roles and permissions --- .../agenta_backend/models/db_models.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 2221e52a40..75d56ac3cb 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -50,7 +50,7 @@ class Permission(str, Enum): # Organization ADD_NEW_USER_TO_ORGANIZATION = "add_new_user_to_organization" - EDIT_ORGANIZATION = "edit_organizayion" + EDIT_ORGANIZATION = "edit_organization" @@ -81,6 +81,9 @@ class Settings: class InvitationDB(BaseModel): token: str = Field(unique=True) email: str + organization: str + workspace = str + workspace_role: Optional[List[WorkspacePermissionDB]] expiration_date: datetime = Field(default="0") used: bool = False @@ -118,9 +121,23 @@ def get_member_roles(self, user_id: PydanticObjectId) -> Optional[List[Workspace def get_member_role_names(self, user_id: PydanticObjectId) -> List[str]: roles = self.get_member_roles(user_id) return [role.role_name for role in roles] if roles else [] - + def get_all_members(self) -> List[PydanticObjectId]: return [member.user_id for member in self.members] + + def get_member_with_roles(self, user_id: PydanticObjectId) -> Optional[WorkspaceMemberDB]: + for member in self.members: + if member.user_id == user_id: + return member + return None + + def get_member_permissions(self, user_id: PydanticObjectId, role_to_check: WorkspaceRole) -> List[Permission]: + roles = self.get_member_roles(user_id) + if roles: + for role in roles: + if role.role_name == role_to_check: + return role.permissions + return [] def has_permission(self, user_id: PydanticObjectId, permission: Permission) -> bool: roles = self.get_member_roles(user_id) From 4f1c7ac296e5af3cafa1847220f5e181517f25e8 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 14:16:04 +0100 Subject: [PATCH 13/99] Add workspace --- .../agenta_backend/models/api/api_models.py | 7 ++ .../agenta_backend/services/app_manager.py | 8 +++ .../agenta_backend/services/db_manager.py | 64 +++++++++++-------- .../services/deployment_manager.py | 8 +-- .../services/evaluator_manager.py | 2 + 5 files changed, 58 insertions(+), 31 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index 5319788dd9..91ee121f2d 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -36,6 +36,7 @@ class VariantAction(BaseModel): class CreateApp(BaseModel): app_name: str organization_id: Optional[str] = None + workspace_id: Optional[str] = None class CreateAppOutput(BaseModel): @@ -58,6 +59,7 @@ class AppVariant(BaseModel): parameters: Optional[Dict[str, Any]] previous_variant_name: Optional[str] organization_id: Optional[str] = None + workspace_id: Optional[str] = None base_name: Optional[str] config_name: Optional[str] @@ -74,6 +76,7 @@ class AppVariantOutput(BaseModel): parameters: Optional[Dict[str, Any]] previous_variant_name: Optional[str] organization_id: str + workspace_id: Optional[str] user_id: str base_name: str base_id: str @@ -107,6 +110,7 @@ class AppVariantFromImage(BaseModel): parameters: Optional[Dict[str, Any]] previous_variant_name: Optional[str] organization_id: Optional[str] = None + workspace_id: Optional[str] = None class RestartAppContainer(BaseModel): @@ -118,6 +122,7 @@ class Image(BaseModel): docker_id: str tags: str organization_id: Optional[str] = None + workspace_id: Optional[str] = None class AddVariantFromImagePayload(BaseModel): @@ -171,6 +176,7 @@ class CreateAppVariant(BaseModel): template_id: str env_vars: Dict[str, str] organization_id: Optional[str] = None + workspace_id: Optional[str] = None class InviteRequest(BaseModel): @@ -187,6 +193,7 @@ class Environment(BaseModel): deployed_base_name: Optional[str] deployed_config_name: Optional[str] organization_id: Optional[str] = None + workspace_id: Optional[str] = None class DeployToEnvironmentPayload(BaseModel): diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 03313277a2..fd056c9700 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -71,6 +71,7 @@ async def start_variant( db_app_variant.image.tags, db_app_variant.app.app_name, db_app_variant.organization, + db_app_variant.workspace, ) logger.debug("App name is %s", db_app_variant.app.app_name) # update the env variables @@ -140,6 +141,7 @@ async def update_variant_image( user=app_variant_db.user, deletable=True, organization=app_variant_db.organization, + workspace=app_variant_db.workspace, ) # Update base with new image await db_manager.update_base(app_variant_db.base, image=db_image) @@ -412,11 +414,13 @@ async def add_variant_based_on_image( if parsed_url.scheme and parsed_url.netloc: db_image = await db_manager.get_orga_image_instance_by_uri( organization_id=str(app.organization.id), + workspace_id=str(app.workspace.id), template_uri=docker_id_or_template_uri, ) else: db_image = await db_manager.get_orga_image_instance_by_docker_id( organization_id=str(app.organization.id), + workspace_id=str(app.workspace.id), docker_id=docker_id_or_template_uri, ) @@ -430,6 +434,7 @@ async def add_variant_based_on_image( deletable=not (is_template_image), user=user_instance, organization=app.organization, + workspace=app.workspace ) else: docker_id = docker_id_or_template_uri @@ -440,6 +445,7 @@ async def add_variant_based_on_image( deletable=not (is_template_image), user=user_instance, organization=app.organization, + workspace=app.workspace ) # Create config @@ -457,6 +463,7 @@ async def add_variant_based_on_image( db_base = await db_manager.create_new_variant_base( app=app, organization=app.organization, + workspace=app.workspace, user=user_instance, base_name=base_name, # the first variant always has default base image=db_image, @@ -470,6 +477,7 @@ async def add_variant_based_on_image( image=db_image, user=user_instance, organization=app.organization, + workspace=app.workspace, parameters={}, base_name=base_name, config_name=config_name, diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 0f0e7f2746..3ceb1cd6f2 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -69,7 +69,7 @@ async def add_testset_to_app_variant( - app_id: str, org_id: str, template_name: str, app_name: str, **kwargs: dict + app_id: str, org_id: str, workspace_id: str, template_name: str, app_name: str, **kwargs: dict ): """Add testset to app variant. Args: @@ -83,6 +83,7 @@ async def add_testset_to_app_variant( try: app_db = await get_app_instance_by_id(app_id) org_db = await get_organization_object(org_id) + workspace_db = await get_workspace(workspace_id) user_db = await get_user(user_uid=kwargs["uid"]) json_path = os.path.join( @@ -101,7 +102,7 @@ async def add_testset_to_app_variant( "csvdata": csvdata, } testset_db = TestSetDB( - **testset, app=app_db, user=user_db, organization=org_db + **testset, app=app_db, user=user_db, organization=org_db, workspace=workspace_db ) await testset_db.create() @@ -124,6 +125,7 @@ async def get_image(app_variant: AppVariant, **kwargs: dict) -> ImageExtended: AppVariantDB.app.id == app_variant.app_id, AppVariantDB.variant_name == app_variant.variant_name, AppVariantDB.organization.id == app_variant.organization, + AppVariantDB.workspace.id == app_variant.workspace, ) db_app_variant = await AppVariantDB.find_one(query_expression) @@ -160,7 +162,7 @@ async def fetch_app_by_id(app_id: str, **kwargs: dict) -> AppDB: async def fetch_app_by_name( - app_name: str, organization_id: Optional[str] = None, **user_org_data: dict + app_name: str, organization_id: Optional[str] = None, workspace_id: Optional[str] = None, **user_org_data: dict ) -> Optional[AppDB]: """Fetches an app by its name. @@ -171,13 +173,14 @@ async def fetch_app_by_name( AppDB: the instance of the app """ - if not organization_id: + if not organization_id and not workspace_id: user = await get_user(user_uid=user_org_data["uid"]) app = await AppDB.find_one(AppDB.app_name == app_name, AppDB.user.id == user.id) else: app = await AppDB.find_one( AppDB.app_name == app_name, AppDB.organization.id == ObjectId(organization_id), + AppDB.workspace.id == ObjectId(workspace_id), ) return app @@ -434,7 +437,7 @@ async def create_deployment( async def create_app_and_envs( - app_name: str, organization_id: str, **user_org_data + app_name: str, organization_id: str, workspace_id: str, **user_org_data ) -> AppDB: """ Create a new app with the given name and organization ID. @@ -442,6 +445,7 @@ async def create_app_and_envs( Args: app_name (str): The name of the app to create. organization_id (str): The ID of the organization that the app belongs to. + workspace_id (str): The ID of the workspace that the app belongs to. **user_org_data: Additional keyword arguments. Returns: @@ -452,14 +456,16 @@ async def create_app_and_envs( """ user_instance = await get_user(user_uid=user_org_data["uid"]) - app = await fetch_app_by_name(app_name, organization_id, **user_org_data) + app = await fetch_app_by_name(app_name, organization_id, workspace_id, **user_org_data) if app is not None: raise ValueError("App with the same name already exists") organization_db = await get_organization_object(organization_id) + workspace_db = await get_workspace(workspace_id) app = AppDB( app_name=app_name, organization=organization_db, + workspace=workspace_db, user=user_instance, ) await app.create() @@ -567,7 +573,7 @@ async def list_variants_for_base( return app_variants_db -async def get_workspace(workspace_id: str) -> Workspace: +async def get_workspace(workspace_id: str): """ Retrieve a workspace. @@ -577,21 +583,22 @@ async def get_workspace(workspace_id: str) -> Workspace: Returns: Workspace: The retrieved workspace. """ - workspace = await WorkspaceDB.find_one(WorkspaceDB.id == ObjectId(workspace_id)) + logger.debug(f"workspace_id: {workspace_id}") + workspace = await WorkspaceDB.find_one(WorkspaceDB.id == ObjectId(workspace_id), fetch_links=True) return workspace -async def create_workspace(payload: CreateWorkspace, user) -> WorkspaceDB: +async def create_workspace(payload: CreateWorkspace, organization: OrganizationDB, user: UserDB) -> WorkspaceDB: """Create a new workspace. Args: payload (Workspace): The workspace payload. + organization (OrganizationDB): The organization that the workspace belongs to. user (UserDB): The user that the workspace belongs to. Returns: Workspace: The created workspace. """ - organization = await get_organization_object(payload.organization_id) # create default workspace workspace = WorkspaceDB( @@ -602,20 +609,19 @@ async def create_workspace(payload: CreateWorkspace, user) -> WorkspaceDB: ) # Assign the creator as the owner with all permissions - workspace.members = WorkspaceMemberDB( + workspace.members = [WorkspaceMemberDB( user_id = user.id, roles = [WorkspacePermissionDB( role_name=WorkspaceRole.OWNER, permissions=list(Permission)) ] - ) + )] await workspace.create() logger.info(f"Created workspace {workspace} for organization {organization.id}") return workspace - async def get_user(user_uid: str) -> UserDB: """Get the user object from the database. @@ -645,7 +651,7 @@ async def get_user(user_uid: str) -> UserDB: description="My Default Workspace", organization_id=str(org_db.id) ) - workspace = await create_workspace(workspace_payload, user) + workspace = await create_workspace(workspace_payload, org, user) # update organization with default workspace id org_db.workspaces = [workspace.id] @@ -726,7 +732,7 @@ async def get_users_by_ids(user_ids: List) -> List: async def get_orga_image_instance_by_docker_id( - organization_id: str, docker_id: str + organization_id: str, workspace_id: str, docker_id: str ) -> ImageDB: """Get the image object from the database with the provided id. @@ -741,12 +747,13 @@ async def get_orga_image_instance_by_docker_id( image = await ImageDB.find_one( ImageDB.docker_id == docker_id, ImageDB.organization.id == ObjectId(organization_id), + ImageDB.workspace.id == ObjectId(workspace_id), ) return image async def get_orga_image_instance_by_uri( - organization_id: str, template_uri: str + organization_id: str, workspace_id: str, template_uri: str ) -> ImageDB: """Get the image object from the database with the provided id. @@ -765,6 +772,7 @@ async def get_orga_image_instance_by_uri( image = await ImageDB.find_one( ImageDB.template_uri == template_uri, ImageDB.organization.id == ObjectId(organization_id), + ImageDB.workspace.id == ObjectId(workspace_id) ) return image @@ -832,6 +840,7 @@ async def add_variant_from_base_and_config( image=base_db.image, user=user_db, organization=previous_app_variant_db.organization, + workspace=previous_app_variant_db.workspace, parameters=parameters, previous_variant_name=previous_app_variant_db.variant_name, # TODO: Remove in future base_name=base_db.base_name, @@ -845,7 +854,7 @@ async def add_variant_from_base_and_config( async def list_apps( - app_name: str = None, org_id: str = None, **user_org_data: dict + app_name: str = None, org_id: str = None, workspace_id: str = None, **user_org_data: dict ) -> List[App]: """ Lists all the unique app names and their IDs from the database @@ -861,13 +870,14 @@ async def list_apps( assert user is not None, "User is None" if app_name is not None: - app_db = await fetch_app_by_name(app_name, org_id, **user_org_data) + app_db = await fetch_app_by_name(app_name, org_id, workspace_id, **user_org_data) return [app_db_to_pydantic(app_db)] elif org_id is not None: organization_access = await check_user_org_access(user_org_data, org_id) if organization_access: apps: List[AppDB] = await AppDB.find( - AppDB.organization.id == ObjectId(org_id) + AppDB.organization.id == ObjectId(org_id), + AppDB.workspace.id == ObjectId(workspace_id) ).to_list() return [app_db_to_pydantic(app) for app in apps] @@ -912,6 +922,7 @@ async def check_is_last_variant_for_image(db_app_variant: AppVariantDB) -> bool: count_variants = await AppVariantDB.find( AppVariantDB.organization.id == db_app_variant.organization.id, + AppVariantDB.workspace.id == db_app_variant.workspace.id, AppVariantDB.base.id == db_app_variant.base.id, ).count() return count_variants == 1 @@ -947,7 +958,7 @@ async def remove_app_variant_from_db(app_variant_db: AppVariantDB, **kwargs: dic ) for environment in environments: environment.deployed_app_variant = None - await environment.create() + await environment.delete() # removing the config config = app_variant_db.config await config.delete() @@ -1055,6 +1066,7 @@ async def create_environment( name=name, user=app_db.user, organization=app_db.organization, + workspace=app_db.workspace, ) await environment_db.create() return environment_db @@ -1540,7 +1552,7 @@ async def update_app_variant( if hasattr(app_variant, key): setattr(app_variant, key, value) - await app_variant.update() + await app_variant.save() return app_variant @@ -1648,21 +1660,22 @@ async def fetch_app_variant_and_check_access( return app_variant -async def fetch_app_by_name_and_organization( - app_name: str, organization_id: str, **user_org_data: dict +async def fetch_app_by_name_and_organization_and_workspace( + app_name: str, organization_id: str, workspace_id: str, **user_org_data: dict ): - """Fetch an app by it's name and organization id. + """Fetch an app by it's name, organization id and workspace id. Args: app_name (str): The name of the app organization_id (str): The ID of the app organization + workspace_id (str): The ID of the app workspace Returns: AppDB: the instance of the app """ app_db = await AppDB.find_one( - {"app_name": app_name, "organization": ObjectId(organization_id)} + {"app_name": app_name, "organization": ObjectId(organization_id), "workspace": ObjectId(workspace_id)}, fetch_links=True ) return app_db @@ -1994,4 +2007,3 @@ async def get_object_workspace_org_id(object_id: str, type: str) -> str: return {"organization_id": organization_id, "workspace_id": workspace_id} except Exception as e: raise e - diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index a43e302a98..489ccb7ab9 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -19,11 +19,8 @@ async def start_service( Start a service. Args: - image_name: List of image tags. - app_name: Name of the app. - base_name: Base name for the container. - env_vars: Environment variables. - organization_id: ID of the organization. + app_variant_db (AppVariantDB): The app variant to start. + env_vars (Dict[str, str]): The environment variables to pass to the container. Returns: True if successful, False otherwise. @@ -55,6 +52,7 @@ async def start_service( deployment = await db_manager.create_deployment( app=app_variant_db.app, organization=app_variant_db.organization, + workspace=app_variant_db.workspace, user=app_variant_db.user, container_name=container_name, container_id=container_id, diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index c2fd5a6055..b15131f913 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -86,6 +86,7 @@ async def create_evaluator_config( evaluator_config = await db_manager.create_evaluator_config( app=app, organization=app.organization, + workspace=app.workspace, user=app.user, name=name, evaluator_key=evaluator_key, @@ -152,6 +153,7 @@ async def create_ready_to_use_evaluators(app: AppDB): await db_manager.create_evaluator_config( app=app, organization=app.organization, + workspace=app.workspace, user=app.user, name=evaluator["name"], evaluator_key=evaluator["key"], From 3ef2fb7c05cfd9f562a6c66bed8c6aceadb28cae Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 14:16:29 +0100 Subject: [PATCH 14/99] Define Workspace Pydantic Models --- .../models/api/workspace_models.py | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/workspace_models.py b/agenta-backend/agenta_backend/models/api/workspace_models.py index 8008b65dd0..9dff9ade6e 100644 --- a/agenta-backend/agenta_backend/models/api/workspace_models.py +++ b/agenta-backend/agenta_backend/models/api/workspace_models.py @@ -1,7 +1,23 @@ from datetime import datetime -from bson import ObjectId -from typing import Optional, List -from pydantic import BaseModel, Field +from pydantic import BaseModel +from typing import Optional, List, Dict +from agenta_backend.models.api.user_models import TimestampModel +from agenta_backend.models.db_models import WorkspaceRole, Permission + + +class WorkspacePermission(BaseModel): + role_name: WorkspaceRole + permissions: List[Permission] + + +class WorkspaceMember(BaseModel): + user_id: str + roles: List[WorkspacePermission] + + +class WorkspaceMemberOutput(BaseModel): + user: Dict + roles: List[WorkspacePermission] class Workspace(BaseModel): @@ -9,16 +25,25 @@ class Workspace(BaseModel): name: str description: Optional[str] type: Optional[str] - + members: Optional[List[WorkspaceMember]] + + +class WorkspaceOutput(TimestampModel): + id: str + name: str + description: Optional[str] + type: Optional[str] + organization: str + members: Optional[List[WorkspaceMemberOutput]] + class CreateWorkspace(BaseModel): name: str description: Optional[str] type: Optional[str] - organization_id: str -class WorkspaceUpdate(BaseModel): +class UpdateWorkspace(BaseModel): name: Optional[str] description: Optional[str] updated_at: Optional[datetime] From a680b01f347d225cc3ab43ef095a52b02a080e1b Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 14:18:46 +0100 Subject: [PATCH 15/99] define get organization default workspace --- .../agenta_backend/routers/user_profile.py | 3 +-- .../agenta_backend/services/selectors.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/user_profile.py b/agenta-backend/agenta_backend/routers/user_profile.py index a751ae72d7..c34774cddc 100644 --- a/agenta-backend/agenta_backend/routers/user_profile.py +++ b/agenta-backend/agenta_backend/routers/user_profile.py @@ -1,9 +1,8 @@ import os -from agenta_backend.models.db_models import UserDB from fastapi import HTTPException, Request -from agenta_backend.models.api.user_models import User from agenta_backend.services import db_manager from agenta_backend.utils.common import APIRouter +from agenta_backend.models.api.user_models import User router = APIRouter() diff --git a/agenta-backend/agenta_backend/services/selectors.py b/agenta-backend/agenta_backend/services/selectors.py index 2e111a31b2..212057684d 100644 --- a/agenta-backend/agenta_backend/services/selectors.py +++ b/agenta-backend/agenta_backend/services/selectors.py @@ -3,6 +3,7 @@ from agenta_backend.models.db_models import ( UserDB, OrganizationDB, + WorkspaceDB, ) @@ -58,3 +59,20 @@ async def get_user_own_org(user_uid: str) -> OrganizationDB: return org else: return None + + +async def get_org_default_workspace(organization: OrganizationDB) -> WorkspaceDB: + """Get's the default workspace for an organization from the database. + + Arguments: + organization (OrganizationDB): The organization + + Returns: + WorkspaceDB: Instance of WorkspaceDB + """ + + workspace: WorkspaceDB = await WorkspaceDB.find_one( + WorkspaceDB.organization == organization.id, WorkspaceDB.type == "default" + ) + return workspace + From 53ff76e74dd00a1fc7bc378bbf6bb7b9d2ecad98 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 14:19:19 +0100 Subject: [PATCH 16/99] add workspace to evaluations --- agenta-backend/agenta_backend/services/evaluation_service.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 3182b8d393..963be715da 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -192,6 +192,7 @@ async def prepare_csvdata_and_create_evaluation_scenario( **evaluation_scenario_payload, user=user, organization=app.organization, + workspace=app.workspace, evaluation=new_evaluation, inputs=list_of_scenario_input, outputs=[], @@ -228,6 +229,7 @@ async def create_evaluation_scenario( new_eval_scenario = EvaluationScenarioDB( user=evaluation.user, organization=evaluation.organization, + workspace=evaluation.workspace, evaluation=evaluation, inputs=scenario_inputs, outputs=[], @@ -630,6 +632,7 @@ async def create_new_human_evaluation( eval_instance = HumanEvaluationDB( app=app, organization=app.organization, # Assuming user has an organization_id attribute + workspace=app.workspace, user=user, status=payload.status, evaluation_type=payload.evaluation_type, @@ -682,6 +685,7 @@ async def create_new_evaluation( evaluation_db = await db_manager.create_new_evaluation( app=app, organization=app.organization, + workspace=app.workspace, user=app.user, testset=testset, status=EvaluationStatusEnum.EVALUATION_STARTED, From 23f49e14d7b62538da3596c500be4e9c62aa08a7 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 15:34:53 +0100 Subject: [PATCH 17/99] define check_user_workspace_access --- agenta-backend/agenta_backend/utils/common.py | 78 +++++++++++++++++-- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index 90aa8f81fa..1cbe8cd48a 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -171,10 +171,69 @@ async def check_access_to_base( return await check_user_org_access(user_org_data, str(organization_id), check_owner) +async def check_user_workspace_access( + user_org_data: Dict[str, Union[str, list]], + workspace_id: str, + org_id: str = None, +) -> bool: + """ + Check if a user has access to a specific workspace. + + Args: + user_org_data (Dict[str, Union[str, list]]): User-specific information. + workspace_id (str): The ID of the workspace. + + Returns: + bool: True if the user has access, False otherwise. + + Raises: + Exception: If neither or both `app` and `app_id` are provided. + + """ + workspace = await WorkspaceDB.find_one( + WorkspaceDB.id == ObjectId(workspace_id), fetch_links=True + ) + if workspace is None: + raise Exception("Workspace not found") + + if org_id is not None: + organization_id = ObjectId(org_id) + + # validate organization exists + organization = await get_organization(str(organization_id)) + if organization is None: + raise Exception("Organization not found") + + # check that workspace belongs to the organization + if workspace.organization.id != organization_id: + raise Exception("Workspace does not belong to the provided organization") + else: + organization_id = workspace.organization.id + + # check that user belongs to the organization + if not await check_user_org_access(user_org_data, str(organization_id)): + logger.error("User does not belong to the organization") + return False + + # check that user belongs to the workspace + user_id = user_org_data["id"] + if ObjectId(user_id) not in workspace.get_all_members(): + logger.error("User does not belong to the workspace") + return False + + # check that workspace is in the user's workspaces + user = await UserDB.find_one(UserDB.id == ObjectId(user_id)) + if ObjectId(workspace_id) not in user.workspaces: + logger.error("Workspace not in user's workspaces") + return False + + return True + + async def check_rbac_permission( - user_org_data: Dict[str, str, Union[str, list]], + user_org_data: Dict[str, Union[str, list]], workspace_id: ObjectId, - provided_organization_id: ObjectId, + organization_id: ObjectId, permission: Permission = None, role: str = None, ) -> bool: @@ -182,9 +241,12 @@ async def check_rbac_permission( Check if a user belongs to a workspace and has a certain permission. Args: - workspace_id (ObjectId): The ID of the workspace to check. + user_org_data (Dict[str, Union[str, list]]): User-specific information. user_id (ObjectId): The user's ID. + organization_id (ObjectId): The ID of the organization to which the workspace belongs. + workspace_id (ObjectId): The ID of the workspace to check. permission (Permission): The permission to check. + role (str): The role to check. Returns: bool: True if the user belongs to the workspace and has the specified permission, False otherwise. @@ -195,29 +257,29 @@ async def check_rbac_permission( raise Exception("Only one of permission or role must be provided") # Retrieve the workspace object using the provided workspace_id - workspace = await WorkspaceDB.find_one(WorkspaceDB.id == workspace_id) + workspace = await WorkspaceDB.find_one(WorkspaceDB.id == workspace_id, fetch_links=True) if workspace is None: raise Exception("Workspace not found") - provided_organization = await get_organization(str(provided_organization_id)) + provided_organization = await get_organization(str(organization_id)) if provided_organization is None: raise Exception("Organization not found") # confirm that the workspace belongs to the provided organization - if str(workspace.organization.id) != provided_organization_id: + if workspace.organization.id != organization_id: raise Exception("Workspace does not belong to the provided organization") # get workspace organization and check if user belongs to it workspace_organization_id = workspace.organization.id - if not check_user_org_access(user_org_data, str(workspace_organization_id)): + if not await check_user_org_access(user_org_data, str(workspace_organization_id)): return False + user_id = ObjectId(user_org_data["id"]) # Check if the user belongs to the workspace if user_id not in workspace.get_all_members(): return False # Check if user is the owner of the workspace ( they have all permissions ) - user_id = user_org_data["id"] if workspace.is_owner(user_id): return True From 545ee3cf2c5225cc7859e5cde5cba8ad083f0947 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 15:35:34 +0100 Subject: [PATCH 18/99] Define RBAC Permission for App Router --- .../models/api/organization_models.py | 1 - .../agenta_backend/routers/app_router.py | 188 ++++++++++++------ .../agenta_backend/services/db_manager.py | 59 +++--- .../agenta_backend/services/selectors.py | 5 +- 4 files changed, 163 insertions(+), 90 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/organization_models.py b/agenta-backend/agenta_backend/models/api/organization_models.py index e7388e49ea..11bf43d915 100644 --- a/agenta-backend/agenta_backend/models/api/organization_models.py +++ b/agenta-backend/agenta_backend/models/api/organization_models.py @@ -1,5 +1,4 @@ from datetime import datetime -from bson import ObjectId from typing import Optional, List from pydantic import BaseModel, Field diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index b77e114acd..10f5824819 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -1,32 +1,35 @@ import os import logging +from typing import List, Optional from docker.errors import DockerException from fastapi.responses import JSONResponse from agenta_backend.config import settings -from typing import List, Optional from fastapi import HTTPException, Request +from agenta_backend.models import converters +from beanie import PydanticObjectId as ObjectId from agenta_backend.utils.common import APIRouter -from agenta_backend.services.selectors import get_user_own_org +from agenta_backend.utils.common import check_rbac_permission +from agenta_backend.models.db_models import Permission, WorkspaceRole + +from agenta_backend.services.selectors import ( + get_user_own_org, + get_org_default_workspace +) from agenta_backend.services import ( - app_manager, db_manager, + app_manager, evaluator_manager, ) -from agenta_backend.utils.common import ( - check_access_to_app, - check_user_org_access, -) from agenta_backend.models.api.api_models import ( App, + Image, CreateApp, CreateAppOutput, CreateAppVariant, AppVariantOutput, - AddVariantFromImagePayload, EnvironmentOutput, - Image, + AddVariantFromImagePayload, ) -from agenta_backend.models import converters if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: from agenta_backend.commons.services.selectors import ( @@ -72,12 +75,15 @@ async def list_app_variants( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - - access_app = await check_access_to_app( - user_org_data=user_org_data, app_id=app_id + workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=WorkspaceRole.MEMBER, ) - if not access_app: - error_msg = f"You cannot access app: {app_id}" + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) return JSONResponse( {"detail": error_msg}, @@ -122,8 +128,21 @@ async def get_variant_by_env( """ try: # Retrieve the user and organization ID based on the session token - user_org_data = await get_user_and_org_id(request.state.user_id) - await check_access_to_app(user_org_data, app_id=app_id) + user_org_data: dict = await get_user_and_org_id(request.state.user_id) + workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=WorkspaceRole.MEMBER, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) # Fetch the app variant using the provided app_id and environment app_variant_db = await db_manager.get_app_variant_by_app_name_and_environment( @@ -164,26 +183,42 @@ async def create_app( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) + if payload.organization_id: - access = await check_user_org_access(user_org_data, payload.organization_id) - if not access: - raise HTTPException( - status_code=403, - detail="You do not have permission to access this app", - ) organization_id = payload.organization_id else: # Retrieve or create user organization organization = await get_user_own_org(user_org_data["uid"]) - if organization is None: # TODO: Check whether we need this - logger.error("Organization for user not found.") - organization = await db_manager.create_user_organization( - user_org_data["uid"] - ) organization_id = str(organization.id) + if not organization_id: + raise HTTPException( + status_code=403, + detail="User Organization not found", + ) + + if payload.workspace_id: + workspace_id = payload.workspace_id + else: + workspace = await get_org_default_workspace(organization) + workspace_id = str(workspace.id) + + user_org_data: dict = await get_user_and_org_id(request.state.user_id) + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=ObjectId(workspace_id), + organization_id=ObjectId(organization_id), + permission=Permission.CREATE_APPLICATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) app_db = await db_manager.create_app_and_envs( - payload.app_name, organization_id, **user_org_data + payload.app_name, organization_id, workspace_id, **user_org_data ) return CreateAppOutput(app_id=str(app_db.id), app_name=str(app_db.app_name)) except Exception as e: @@ -195,6 +230,7 @@ async def list_apps( request: Request, app_name: Optional[str] = None, org_id: Optional[str] = None, + workspace_id: Optional[str] = None, ) -> List[App]: """ Retrieve a list of apps filtered by app_name and org_id. @@ -212,7 +248,7 @@ async def list_apps( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - apps = await db_manager.list_apps(app_name, org_id, **user_org_data) + apps = await db_manager.list_apps(app_name, org_id, workspace_id, **user_org_data) return apps except Exception as e: logger.error(f"list_apps exception ===> {e}") @@ -256,9 +292,15 @@ async def add_variant_from_image( try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_app = await check_access_to_app(user_org_data, app_id=app_id) - if not access_app: - error_msg = f"You cannot access app: {app_id}" + workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + permission=Permission.CREATE_APPLICATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) return JSONResponse( {"detail": error_msg}, @@ -295,17 +337,21 @@ async def remove_app(app_id: str, request: Request): """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_app = await check_access_to_app( - user_org_data, app_id=app_id, check_owner=True + workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + permission=Permission.DELETE_APPLICATION, ) - - if not access_app: - error_msg = f"You do not have permission to delete app: {app_id}" + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) + else: await app_manager.remove_app(app_id=app_id, **user_org_data) except DockerException as e: @@ -347,33 +393,55 @@ async def create_app_and_variant_from_template( logger.debug("Step 2: Setting organization ID") if payload.organization_id is None: organization = await get_user_own_org(user_org_data["uid"]) - organization_id = organization.id + organization_id = str(organization.id) else: organization_id = payload.organization_id + + logger.debug("Step 3: Setting workspace ID") + if payload.workspace_id is None: + workspace = await get_org_default_workspace(organization) + workspace_id = str(workspace.id) + else: + workspace_id = payload.workspace_id + + logger.debug("Step 4: Checking user has permission to create app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_id, + organization_id=organization_id, + permission=Permission.CREATE_APPLICATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) - logger.debug(f"Step 3 Checking if app {payload.app_name} already exists") + logger.debug(f"Step 5: Checking if app {payload.app_name} already exists") app_name = payload.app_name.lower() - app = await db_manager.fetch_app_by_name_and_organization( - app_name, organization_id, **user_org_data + app = await db_manager.fetch_app_by_name_and_organization_and_workspace( + app_name, organization_id, workspace_id, **user_org_data ) if app is not None: raise Exception( f"App with name {app_name} already exists", ) - logger.debug("Step 4: Creating new app and initializing environments") + logger.debug("Step 6: Creating new app and initializing environments") if app is None: app = await db_manager.create_app_and_envs( - app_name, organization_id, **user_org_data + app_name, organization_id, workspace_id, **user_org_data ) - logger.debug("Step 5: Retrieve template from db") + logger.debug("Step 7: Retrieve template from db") template_db = await db_manager.get_template(payload.template_id) repo_name = os.environ.get("AGENTA_TEMPLATE_REPO", "agentaai/templates_v2") image_name = f"{repo_name}:{template_db.name}" logger.debug( - "Step 6: Creating image instance and adding variant based on image" + "Step 8: Creating image instance and adding variant based on image" ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, @@ -390,19 +458,20 @@ async def create_app_and_variant_from_template( **user_org_data, ) - logger.debug("Step 7: Creating testset for app variant") + logger.debug("Step 9: Creating testset for app variant") await db_manager.add_testset_to_app_variant( app_id=str(app.id), org_id=organization_id, + workspace_id=workspace_id, template_name=template_db.name, app_name=app.app_name, **user_org_data, ) - logger.debug("Step 8: We create ready-to use evaluators") + logger.debug("Step 10: We create ready-to use evaluators") await evaluator_manager.create_ready_to_use_evaluators(app=app) - logger.debug("Step 9: Starting variant and injecting environment variables") + logger.debug("Step 11: Starting variant and injecting environment variables") if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: if not os.environ["OPENAI_API_KEY"]: raise Exception( @@ -451,17 +520,22 @@ async def list_environments( logger.debug("get user and org data") user_and_org_data: dict = await get_user_and_org_id(request.state.user_id) - # Check if has app access + # Check if user has access to app logger.debug("check_access_to_app") - access_app = await check_access_to_app( - user_org_data=user_and_org_data, app_id=app_id + workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_and_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=WorkspaceRole.MEMBER, ) - logger.debug(f"access_app: {access_app}") - if not access_app: - error_msg = f"You do not have access to this app: {app_id}" + logger.debug(f"access_app: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) else: environments_db = await db_manager.list_environments( diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 3ceb1cd6f2..83ede4ce38 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -50,8 +50,8 @@ WorkspaceMemberDB, WorkspacePermissionDB ) -from agenta_backend.utils.common import check_user_org_access from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum +from agenta_backend.utils.common import check_user_org_access, check_user_workspace_access from fastapi import HTTPException from fastapi.responses import JSONResponse @@ -216,7 +216,7 @@ async def fetch_base_by_id( """ if base_id is None: raise Exception("No base_id provided") - base = await VariantBaseDB.find_one(VariantBaseDB.id == ObjectId(base_id)) + base = await VariantBaseDB.find_one(VariantBaseDB.id == ObjectId(base_id), fetch_links=True) if base is None: logger.error("Base not found") return False @@ -868,13 +868,16 @@ async def list_apps( user = await get_user(user_uid=user_org_data["uid"]) assert user is not None, "User is None" + + # assert that if org_id is provided, workspace_id is also provided, and vice versa + assert org_id is not None and workspace_id is not None, "org_id and workspace_id must be provided together" if app_name is not None: app_db = await fetch_app_by_name(app_name, org_id, workspace_id, **user_org_data) return [app_db_to_pydantic(app_db)] - elif org_id is not None: - organization_access = await check_user_org_access(user_org_data, org_id) - if organization_access: + elif org_id is not None and workspace_id is not None: + action_access = await check_user_workspace_access(user_org_data, workspace_id, org_id) + if action_access: apps: List[AppDB] = await AppDB.find( AppDB.organization.id == ObjectId(org_id), AppDB.workspace.id == ObjectId(workspace_id) @@ -1359,7 +1362,7 @@ async def fetch_evaluation_scenario_by_id( """ assert evaluation_scenario_id is not None, "evaluation_scenario_id cannot be None" evaluation_scenario = await EvaluationScenarioDB.find_one( - EvaluationScenarioDB.id == ObjectId(evaluation_scenario_id) + EvaluationScenarioDB.id == ObjectId(evaluation_scenario_id, fetch_links=True) ) return evaluation_scenario @@ -1791,7 +1794,7 @@ async def fetch_evaluator_config(evaluator_config_id: str): try: evaluator_config: EvaluatorConfigDB = await EvaluatorConfigDB.find_one( - EvaluatorConfigDB.id == ObjectId(evaluator_config_id) + EvaluatorConfigDB.id == ObjectId(evaluator_config_id, fetch_links=True) ) return evaluator_config except Exception as e: @@ -1937,7 +1940,7 @@ async def update_evaluation( return evaluation -async def get_object_workspace_org_id(object_id: str, type: str) -> str: +async def get_object_workspace_org_id(object_id: str, type: str) -> dict: """ Get the organization and workspace id of the object. @@ -1951,55 +1954,55 @@ async def get_object_workspace_org_id(object_id: str, type: str) -> str: try: if type == "app": app = await fetch_app_by_id(object_id) - organization_id = str(app.organization.id) - workspace_id = str(app.workspace.id) + organization_id = app.organization.id + workspace_id = app.workspace.id elif type == "app_variant": app_variant = await fetch_app_variant_by_id(object_id) - organization_id = str(app_variant.organization.id) - workspace_id = str(app_variant.workspace.id) + organization_id = app_variant.organization.id + workspace_id = app_variant.workspace.id elif type == "base": base = await fetch_base_by_id(object_id) - organization_id = str(base.organization.id) - workspace_id = str(base.workspace.id) + organization_id = base.organization.id + workspace_id = base.workspace.id elif type == "deployment": deployment = await get_deployment_by_objectid(object_id) - organization_id = str(deployment.organization.id) - workspace_id = str(deployment.workspace.id) + organization_id = deployment.organization.id + workspace_id = deployment.workspace.id elif type == "testset": testset = await fetch_testset_by_id(object_id) - organization_id = str(testset.organization.id) - workspace_id = str(testset.workspace.id) + organization_id = testset.organization.id + workspace_id = testset.workspace.id elif type == "evaluation": evaluation = await fetch_evaluation_by_id(object_id) - organization_id = str(evaluation.organization.id) - workspace_id = str(evaluation.workspace.id) + organization_id = evaluation.organization.id + workspace_id = evaluation.workspace.id elif type == "evaluation_scenario": evaluation_scenario = await fetch_evaluation_scenario_by_id(object_id) - organization_id = str(evaluation_scenario.organization.id) - workspace_id = str(evaluation_scenario.workspace.id) + organization_id = evaluation_scenario.organization.id + workspace_id = evaluation_scenario.workspace.id elif type == "evaluator_config": evaluator_config = await fetch_evaluator_config(object_id) - organization_id = str(evaluator_config.organization.id) - workspace_id = str(evaluator_config.workspace.id) + organization_id = evaluator_config.organization.id + workspace_id = evaluator_config.workspace.id elif type == "human_evaluation": human_evaluation = await fetch_human_evaluation_by_id(object_id) - organization_id = str(human_evaluation.human_evaluation.id) - workspace_id = str(human_evaluation.human_evaluation.id) + organization_id = human_evaluation.human_evaluation.id + workspace_id = human_evaluation.human_evaluation.id elif type == "human_evaluation_scenario": human_evaluation_scenario = await fetch_human_evaluation_scenario_by_id( object_id ) - organization_id = str(human_evaluation_scenario.organization.id) - workspace_id = str(human_evaluation_scenario.workspace.id) + organization_id = human_evaluation_scenario.organization.id + workspace_id = human_evaluation_scenario.workspace.id else: raise ValueError(f"Unknown object type: {type}") diff --git a/agenta-backend/agenta_backend/services/selectors.py b/agenta-backend/agenta_backend/services/selectors.py index 212057684d..646e8d1755 100644 --- a/agenta-backend/agenta_backend/services/selectors.py +++ b/agenta-backend/agenta_backend/services/selectors.py @@ -55,10 +55,7 @@ async def get_user_own_org(user_uid: str) -> OrganizationDB: org: OrganizationDB = await OrganizationDB.find_one( OrganizationDB.owner == str(user.id), OrganizationDB.type == "default" ) - if org is not None: - return org - else: - return None + return org async def get_org_default_workspace(organization: OrganizationDB) -> WorkspaceDB: From c131e528f8b1adde52c1d4f3154e6a67198ad239 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 15:36:55 +0100 Subject: [PATCH 19/99] format --- .../models/api/organization_models.py | 4 +- .../models/api/workspace_models.py | 7 +- .../agenta_backend/models/db_models.py | 40 +++--- .../agenta_backend/routers/app_router.py | 16 ++- .../agenta_backend/services/app_manager.py | 4 +- .../agenta_backend/services/db_manager.py | 133 +++++++++++------- .../agenta_backend/services/selectors.py | 1 - agenta-backend/agenta_backend/utils/common.py | 32 +++-- 8 files changed, 142 insertions(+), 95 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/organization_models.py b/agenta-backend/agenta_backend/models/api/organization_models.py index 11bf43d915..6036cf32da 100644 --- a/agenta-backend/agenta_backend/models/api/organization_models.py +++ b/agenta-backend/agenta_backend/models/api/organization_models.py @@ -16,8 +16,8 @@ class Organization(BaseModel): owner: str members: Optional[List[str]] invitations: Optional[List] - - + + class CreateOrganization(BaseModel): name: str description: Optional[str] diff --git a/agenta-backend/agenta_backend/models/api/workspace_models.py b/agenta-backend/agenta_backend/models/api/workspace_models.py index 9dff9ade6e..3491364243 100644 --- a/agenta-backend/agenta_backend/models/api/workspace_models.py +++ b/agenta-backend/agenta_backend/models/api/workspace_models.py @@ -13,8 +13,8 @@ class WorkspacePermission(BaseModel): class WorkspaceMember(BaseModel): user_id: str roles: List[WorkspacePermission] - - + + class WorkspaceMemberOutput(BaseModel): user: Dict roles: List[WorkspacePermission] @@ -36,7 +36,7 @@ class WorkspaceOutput(TimestampModel): organization: str members: Optional[List[WorkspaceMemberOutput]] - + class CreateWorkspace(BaseModel): name: str description: Optional[str] @@ -47,4 +47,3 @@ class UpdateWorkspace(BaseModel): name: Optional[str] description: Optional[str] updated_at: Optional[datetime] - diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 75d56ac3cb..7193ab1ea0 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -18,28 +18,28 @@ class Permission(str, Enum): MODIFY_CONFIGURATIONS = "modify_configurations" CHANGE_USER_ROLES = "change_user_roles" READ_SYSTEM = "read_system" - + VIEW_APPLICATION = "view_application" CREATE_APPLICATION = "create_application" EDIT_APPLICATION = "edit_application" DELETE_APPLICATION = "delete_application" - + # Testset VIEW_TESTSET = "view_testset" CREATE_TESTSET = "create_testset" EDIT_TESTSET = "edit_testset" DELETE_TESTSET = "delete_testset" - + # Evaluation VIEW_EVALUATION = "view_evaluation" CREATE_EVALUATION = "create_evaluation" RUN_EVALUATIONS = "run_evaluations" EDIT_EVALUATION = "edit_evaluation" DELETE_EVALUATION = "delete_evaluation" - + # Deployment DEPLOY_APPLICATION = "deploy_application" - + # Workspace VIEW_WORKSPACE = "view_workspace" CREATE_WORKSPACE = "create_workspace" @@ -47,12 +47,11 @@ class Permission(str, Enum): DELETE_WORKSPACE = "delete_workspace" MODIFY_USER_ROLES = "modify_user_roles" ADD_NEW_USER_TO_WORKSPACE = "add_new_user_to_workspace" - + # Organization ADD_NEW_USER_TO_ORGANIZATION = "add_new_user_to_organization" EDIT_ORGANIZATION = "edit_organization" - - + class WorkspacePermissionDB(BaseModel): role_name: WorkspaceRole @@ -112,7 +111,9 @@ class WorkspaceDB(Document): created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) - def get_member_roles(self, user_id: PydanticObjectId) -> Optional[List[WorkspacePermissionDB]]: + def get_member_roles( + self, user_id: PydanticObjectId + ) -> Optional[List[WorkspacePermissionDB]]: for member in self.members: if member.user_id == user_id: return member.roles @@ -124,14 +125,18 @@ def get_member_role_names(self, user_id: PydanticObjectId) -> List[str]: def get_all_members(self) -> List[PydanticObjectId]: return [member.user_id for member in self.members] - - def get_member_with_roles(self, user_id: PydanticObjectId) -> Optional[WorkspaceMemberDB]: + + def get_member_with_roles( + self, user_id: PydanticObjectId + ) -> Optional[WorkspaceMemberDB]: for member in self.members: if member.user_id == user_id: return member return None - - def get_member_permissions(self, user_id: PydanticObjectId, role_to_check: WorkspaceRole) -> List[Permission]: + + def get_member_permissions( + self, user_id: PydanticObjectId, role_to_check: WorkspaceRole + ) -> List[Permission]: roles = self.get_member_roles(user_id) if roles: for role in roles: @@ -146,18 +151,21 @@ def has_permission(self, user_id: PydanticObjectId, permission: Permission) -> b if permission in role.permissions: return True return False - + def has_role(self, user_id: PydanticObjectId, role_to_check: WorkspaceRole) -> bool: roles = self.get_member_roles(user_id) if roles: for role in roles: if role.role_name == role_to_check: return True - return False + return False def is_owner(self, user_id: PydanticObjectId) -> bool: for member in self.members: - if member.user_id == user_id and WorkspaceRole.OWNER in self.get_member_role_names(user_id): + if ( + member.user_id == user_id + and WorkspaceRole.OWNER in self.get_member_role_names(user_id) + ): return True return False diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 10f5824819..2e6bc07199 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -13,7 +13,7 @@ from agenta_backend.services.selectors import ( get_user_own_org, - get_org_default_workspace + get_org_default_workspace, ) from agenta_backend.services import ( db_manager, @@ -183,7 +183,7 @@ async def create_app( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - + if payload.organization_id: organization_id = payload.organization_id else: @@ -195,13 +195,13 @@ async def create_app( status_code=403, detail="User Organization not found", ) - + if payload.workspace_id: workspace_id = payload.workspace_id else: workspace = await get_org_default_workspace(organization) workspace_id = str(workspace.id) - + user_org_data: dict = await get_user_and_org_id(request.state.user_id) has_permission = await check_rbac_permission( user_org_data=user_org_data, @@ -248,7 +248,9 @@ async def list_apps( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - apps = await db_manager.list_apps(app_name, org_id, workspace_id, **user_org_data) + apps = await db_manager.list_apps( + app_name, org_id, workspace_id, **user_org_data + ) return apps except Exception as e: logger.error(f"list_apps exception ===> {e}") @@ -396,14 +398,14 @@ async def create_app_and_variant_from_template( organization_id = str(organization.id) else: organization_id = payload.organization_id - + logger.debug("Step 3: Setting workspace ID") if payload.workspace_id is None: workspace = await get_org_default_workspace(organization) workspace_id = str(workspace.id) else: workspace_id = payload.workspace_id - + logger.debug("Step 4: Checking user has permission to create app") has_permission = await check_rbac_permission( user_org_data=user_org_data, diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index fd056c9700..587050006a 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -434,7 +434,7 @@ async def add_variant_based_on_image( deletable=not (is_template_image), user=user_instance, organization=app.organization, - workspace=app.workspace + workspace=app.workspace, ) else: docker_id = docker_id_or_template_uri @@ -445,7 +445,7 @@ async def add_variant_based_on_image( deletable=not (is_template_image), user=user_instance, organization=app.organization, - workspace=app.workspace + workspace=app.workspace, ) # Create config diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 83ede4ce38..95169d6a32 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -48,10 +48,13 @@ WorkspaceDB, WorkspaceRole, WorkspaceMemberDB, - WorkspacePermissionDB + WorkspacePermissionDB, ) from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum -from agenta_backend.utils.common import check_user_org_access, check_user_workspace_access +from agenta_backend.utils.common import ( + check_user_org_access, + check_user_workspace_access, +) from fastapi import HTTPException from fastapi.responses import JSONResponse @@ -69,7 +72,12 @@ async def add_testset_to_app_variant( - app_id: str, org_id: str, workspace_id: str, template_name: str, app_name: str, **kwargs: dict + app_id: str, + org_id: str, + workspace_id: str, + template_name: str, + app_name: str, + **kwargs: dict, ): """Add testset to app variant. Args: @@ -102,7 +110,11 @@ async def add_testset_to_app_variant( "csvdata": csvdata, } testset_db = TestSetDB( - **testset, app=app_db, user=user_db, organization=org_db, workspace=workspace_db + **testset, + app=app_db, + user=user_db, + organization=org_db, + workspace=workspace_db, ) await testset_db.create() @@ -162,7 +174,10 @@ async def fetch_app_by_id(app_id: str, **kwargs: dict) -> AppDB: async def fetch_app_by_name( - app_name: str, organization_id: Optional[str] = None, workspace_id: Optional[str] = None, **user_org_data: dict + app_name: str, + organization_id: Optional[str] = None, + workspace_id: Optional[str] = None, + **user_org_data: dict, ) -> Optional[AppDB]: """Fetches an app by its name. @@ -204,9 +219,7 @@ async def fetch_app_variant_by_id( return app_variant -async def fetch_base_by_id( - base_id: str -) -> Optional[VariantBaseDB]: +async def fetch_base_by_id(base_id: str) -> Optional[VariantBaseDB]: """ Fetches a base by its ID. Args: @@ -216,7 +229,9 @@ async def fetch_base_by_id( """ if base_id is None: raise Exception("No base_id provided") - base = await VariantBaseDB.find_one(VariantBaseDB.id == ObjectId(base_id), fetch_links=True) + base = await VariantBaseDB.find_one( + VariantBaseDB.id == ObjectId(base_id), fetch_links=True + ) if base is None: logger.error("Base not found") return False @@ -456,7 +471,9 @@ async def create_app_and_envs( """ user_instance = await get_user(user_uid=user_org_data["uid"]) - app = await fetch_app_by_name(app_name, organization_id, workspace_id, **user_org_data) + app = await fetch_app_by_name( + app_name, organization_id, workspace_id, **user_org_data + ) if app is not None: raise ValueError("App with the same name already exists") @@ -584,11 +601,15 @@ async def get_workspace(workspace_id: str): Workspace: The retrieved workspace. """ logger.debug(f"workspace_id: {workspace_id}") - workspace = await WorkspaceDB.find_one(WorkspaceDB.id == ObjectId(workspace_id), fetch_links=True) + workspace = await WorkspaceDB.find_one( + WorkspaceDB.id == ObjectId(workspace_id), fetch_links=True + ) return workspace -async def create_workspace(payload: CreateWorkspace, organization: OrganizationDB, user: UserDB) -> WorkspaceDB: +async def create_workspace( + payload: CreateWorkspace, organization: OrganizationDB, user: UserDB +) -> WorkspaceDB: """Create a new workspace. Args: @@ -599,26 +620,29 @@ async def create_workspace(payload: CreateWorkspace, organization: OrganizationD Returns: Workspace: The created workspace. """ - + # create default workspace workspace = WorkspaceDB( name=payload.name, type=payload.type if payload.type else "", description=payload.description if payload.description else "", - organization=organization + organization=organization, ) - + # Assign the creator as the owner with all permissions - workspace.members = [WorkspaceMemberDB( - user_id = user.id, - roles = [WorkspacePermissionDB( - role_name=WorkspaceRole.OWNER, - permissions=list(Permission)) - ] - )] + workspace.members = [ + WorkspaceMemberDB( + user_id=user.id, + roles=[ + WorkspacePermissionDB( + role_name=WorkspaceRole.OWNER, permissions=list(Permission) + ) + ], + ) + ] await workspace.create() logger.info(f"Created workspace {workspace} for organization {organization.id}") - + return workspace @@ -635,7 +659,6 @@ async def get_user(user_uid: str) -> UserDB: user = await UserDB.find_one(UserDB.uid == user_uid) if user is None: if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"]: - # create user user_db = UserDB(uid="0") user = await user_db.create() @@ -643,25 +666,25 @@ async def get_user(user_uid: str) -> UserDB: # create default organization for user org_db = OrganizationDB(type="default", owner=str(user.id)) org = await org_db.create() - + # create default workspace for user workspace_payload = CreateWorkspace( name=org_db.name, type=org_db.type, description="My Default Workspace", - organization_id=str(org_db.id) + organization_id=str(org_db.id), ) workspace = await create_workspace(workspace_payload, org, user) - + # update organization with default workspace id org_db.workspaces = [workspace.id] await org_db.update({"$set": org_db.dict(exclude_unset=True)}) - + # update user with organization and workspace user_db.organizations.append(org.id) user_db.workspaces.append(workspace.id) await user_db.update({"$set": user_db.dict(exclude_unset=True)}) - + return user raise Exception("Please login or signup") return user @@ -772,7 +795,7 @@ async def get_orga_image_instance_by_uri( image = await ImageDB.find_one( ImageDB.template_uri == template_uri, ImageDB.organization.id == ObjectId(organization_id), - ImageDB.workspace.id == ObjectId(workspace_id) + ImageDB.workspace.id == ObjectId(workspace_id), ) return image @@ -854,7 +877,10 @@ async def add_variant_from_base_and_config( async def list_apps( - app_name: str = None, org_id: str = None, workspace_id: str = None, **user_org_data: dict + app_name: str = None, + org_id: str = None, + workspace_id: str = None, + **user_org_data: dict, ) -> List[App]: """ Lists all the unique app names and their IDs from the database @@ -868,19 +894,25 @@ async def list_apps( user = await get_user(user_uid=user_org_data["uid"]) assert user is not None, "User is None" - + # assert that if org_id is provided, workspace_id is also provided, and vice versa - assert org_id is not None and workspace_id is not None, "org_id and workspace_id must be provided together" + assert ( + org_id is not None and workspace_id is not None + ), "org_id and workspace_id must be provided together" if app_name is not None: - app_db = await fetch_app_by_name(app_name, org_id, workspace_id, **user_org_data) + app_db = await fetch_app_by_name( + app_name, org_id, workspace_id, **user_org_data + ) return [app_db_to_pydantic(app_db)] elif org_id is not None and workspace_id is not None: - action_access = await check_user_workspace_access(user_org_data, workspace_id, org_id) + action_access = await check_user_workspace_access( + user_org_data, workspace_id, org_id + ) if action_access: apps: List[AppDB] = await AppDB.find( AppDB.organization.id == ObjectId(org_id), - AppDB.workspace.id == ObjectId(workspace_id) + AppDB.workspace.id == ObjectId(workspace_id), ).to_list() return [app_db_to_pydantic(app) for app in apps] @@ -1678,7 +1710,12 @@ async def fetch_app_by_name_and_organization_and_workspace( """ app_db = await AppDB.find_one( - {"app_name": app_name, "organization": ObjectId(organization_id), "workspace": ObjectId(workspace_id)}, fetch_links=True + { + "app_name": app_name, + "organization": ObjectId(organization_id), + "workspace": ObjectId(workspace_id), + }, + fetch_links=True, ) return app_db @@ -1956,57 +1993,57 @@ async def get_object_workspace_org_id(object_id: str, type: str) -> dict: app = await fetch_app_by_id(object_id) organization_id = app.organization.id workspace_id = app.workspace.id - + elif type == "app_variant": app_variant = await fetch_app_variant_by_id(object_id) organization_id = app_variant.organization.id workspace_id = app_variant.workspace.id - + elif type == "base": base = await fetch_base_by_id(object_id) organization_id = base.organization.id workspace_id = base.workspace.id - + elif type == "deployment": deployment = await get_deployment_by_objectid(object_id) organization_id = deployment.organization.id workspace_id = deployment.workspace.id - + elif type == "testset": testset = await fetch_testset_by_id(object_id) organization_id = testset.organization.id workspace_id = testset.workspace.id - + elif type == "evaluation": evaluation = await fetch_evaluation_by_id(object_id) organization_id = evaluation.organization.id workspace_id = evaluation.workspace.id - + elif type == "evaluation_scenario": evaluation_scenario = await fetch_evaluation_scenario_by_id(object_id) organization_id = evaluation_scenario.organization.id workspace_id = evaluation_scenario.workspace.id - + elif type == "evaluator_config": evaluator_config = await fetch_evaluator_config(object_id) organization_id = evaluator_config.organization.id workspace_id = evaluator_config.workspace.id - + elif type == "human_evaluation": human_evaluation = await fetch_human_evaluation_by_id(object_id) organization_id = human_evaluation.human_evaluation.id workspace_id = human_evaluation.human_evaluation.id - + elif type == "human_evaluation_scenario": human_evaluation_scenario = await fetch_human_evaluation_scenario_by_id( object_id ) organization_id = human_evaluation_scenario.organization.id workspace_id = human_evaluation_scenario.workspace.id - + else: raise ValueError(f"Unknown object type: {type}") - + return {"organization_id": organization_id, "workspace_id": workspace_id} except Exception as e: raise e diff --git a/agenta-backend/agenta_backend/services/selectors.py b/agenta-backend/agenta_backend/services/selectors.py index 646e8d1755..8c5bac351f 100644 --- a/agenta-backend/agenta_backend/services/selectors.py +++ b/agenta-backend/agenta_backend/services/selectors.py @@ -72,4 +72,3 @@ async def get_org_default_workspace(organization: OrganizationDB) -> WorkspaceDB WorkspaceDB.organization == organization.id, WorkspaceDB.type == "default" ) return workspace - diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index 1cbe8cd48a..39027e3334 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -12,7 +12,7 @@ VariantBaseDB, WorkspaceDB, Permission, - WorkspaceRole + WorkspaceRole, ) from beanie import PydanticObjectId as ObjectId @@ -195,38 +195,38 @@ async def check_user_workspace_access( ) if workspace is None: raise Exception("Workspace not found") - + if org_id is not None: organization_id = ObjectId(org_id) - + # validate organization exists organization = await get_organization(str(organization_id)) if organization is None: raise Exception("Organization not found") - + # check that workspace belongs to the organization if workspace.organization.id != organization_id: raise Exception("Workspace does not belong to the provided organization") else: organization_id = workspace.organization.id - + # check that user belongs to the organization if not await check_user_org_access(user_org_data, str(organization_id)): logger.error("User does not belong to the organization") return False - + # check that user belongs to the workspace user_id = user_org_data["id"] if ObjectId(user_id) not in workspace.get_all_members(): logger.error("User does not belong to the workspace") return False - + # check that workspace is in the user's workspaces user = await UserDB.find_one(UserDB.id == ObjectId(user_id)) if ObjectId(workspace_id) not in user.workspaces: logger.error("Workspace not in user's workspaces") return False - + return True @@ -255,30 +255,32 @@ async def check_rbac_permission( raise Exception("Either permission or role must be provided") elif permission is not None and role is not None: raise Exception("Only one of permission or role must be provided") - + # Retrieve the workspace object using the provided workspace_id - workspace = await WorkspaceDB.find_one(WorkspaceDB.id == workspace_id, fetch_links=True) + workspace = await WorkspaceDB.find_one( + WorkspaceDB.id == workspace_id, fetch_links=True + ) if workspace is None: raise Exception("Workspace not found") - + provided_organization = await get_organization(str(organization_id)) if provided_organization is None: raise Exception("Organization not found") - + # confirm that the workspace belongs to the provided organization if workspace.organization.id != organization_id: raise Exception("Workspace does not belong to the provided organization") - + # get workspace organization and check if user belongs to it workspace_organization_id = workspace.organization.id if not await check_user_org_access(user_org_data, str(workspace_organization_id)): return False - + user_id = ObjectId(user_org_data["id"]) # Check if the user belongs to the workspace if user_id not in workspace.get_all_members(): return False - + # Check if user is the owner of the workspace ( they have all permissions ) if workspace.is_owner(user_id): return True From 557bf10d017d34f7032b44918b87787b9a0da203 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Sun, 14 Jan 2024 16:45:20 +0100 Subject: [PATCH 20/99] fix bugs with creating an app from workspace assignment --- .../agenta_backend/models/converters.py | 3 ++ .../agenta_backend/routers/app_router.py | 17 +++++------ .../agenta_backend/services/db_manager.py | 28 ++++++++++++------- .../agenta_backend/services/selectors.py | 2 +- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index f74acebe8e..5c2faad350 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -199,6 +199,7 @@ def app_variant_db_to_pydantic( parameters=app_variant_db.config.parameters, previous_variant_name=app_variant_db.previous_variant_name, organization_id=str(app_variant_db.organization.id), + workspace_id=str(app_variant_db.workspace.id), base_name=app_variant_db.base_name, config_name=app_variant_db.config_name, ) @@ -221,6 +222,7 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantOu variant_id=str(app_variant_db.id), user_id=str(app_variant_db.user.id), organization_id=str(app_variant_db.organization.id), + workspace_id=str(app_variant_db.workspace.id), parameters=app_variant_db.config.parameters, previous_variant_name=app_variant_db.previous_variant_name, base_name=app_variant_db.base_name, @@ -264,6 +266,7 @@ def app_db_to_pydantic(app_db: AppDB) -> App: def image_db_to_pydantic(image_db: ImageDB) -> ImageExtended: return ImageExtended( organization_id=str(image_db.organization.id), + workspace_id=str(image_db.workspace.id), docker_id=image_db.docker_id, tags=image_db.tags, id=str(image_db.id), diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 2e6bc07199..9cd5d5c762 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -75,7 +75,7 @@ async def list_app_variants( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") has_permission = await check_rbac_permission( user_org_data=user_org_data, workspace_id=workspace_org_data["workspace_id"], @@ -129,7 +129,7 @@ async def get_variant_by_env( try: # Retrieve the user and organization ID based on the session token user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") has_permission = await check_rbac_permission( user_org_data=user_org_data, workspace_id=workspace_org_data["workspace_id"], @@ -186,8 +186,8 @@ async def create_app( if payload.organization_id: organization_id = payload.organization_id + organization = await db_manager.get_organization_object(organization_id) else: - # Retrieve or create user organization organization = await get_user_own_org(user_org_data["uid"]) organization_id = str(organization.id) if not organization_id: @@ -294,7 +294,7 @@ async def add_variant_from_image( try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") has_permission = await check_rbac_permission( user_org_data=user_org_data, workspace_id=workspace_org_data["workspace_id"], @@ -339,7 +339,7 @@ async def remove_app(app_id: str, request: Request): """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") has_permission = await check_rbac_permission( user_org_data=user_org_data, workspace_id=workspace_org_data["workspace_id"], @@ -398,6 +398,7 @@ async def create_app_and_variant_from_template( organization_id = str(organization.id) else: organization_id = payload.organization_id + organization = await db_manager.get_organization_object(organization_id) logger.debug("Step 3: Setting workspace ID") if payload.workspace_id is None: @@ -409,8 +410,8 @@ async def create_app_and_variant_from_template( logger.debug("Step 4: Checking user has permission to create app") has_permission = await check_rbac_permission( user_org_data=user_org_data, - workspace_id=workspace_id, - organization_id=organization_id, + workspace_id=ObjectId(workspace_id), + organization_id=ObjectId(organization_id), permission=Permission.CREATE_APPLICATION, ) if not has_permission: @@ -524,7 +525,7 @@ async def list_environments( # Check if user has access to app logger.debug("check_access_to_app") - workspace_org_data = db_manager.get_object_workspace_org_id(app_id, "app") + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") has_permission = await check_rbac_permission( user_org_data=user_and_org_data, workspace_id=workspace_org_data["workspace_id"], diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 95169d6a32..aeea4a4c7e 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -399,6 +399,7 @@ async def create_image( deletable=deletable, user=user, organization=organization, + workspace=workspace, ) elif image_type == "image": image = ImageDB( @@ -896,23 +897,30 @@ async def list_apps( assert user is not None, "User is None" # assert that if org_id is provided, workspace_id is also provided, and vice versa - assert ( - org_id is not None and workspace_id is not None - ), "org_id and workspace_id must be provided together" + # assert ( # TODO: Enable this check when workspace_id is provided + # org_id is not None and workspace_id is not None + # ), "org_id and workspace_id must be provided together" if app_name is not None: app_db = await fetch_app_by_name( app_name, org_id, workspace_id, **user_org_data ) return [app_db_to_pydantic(app_db)] - elif org_id is not None and workspace_id is not None: - action_access = await check_user_workspace_access( - user_org_data, workspace_id, org_id - ) - if action_access: + # elif org_id is not None and workspace_id is not None: # TODO: Enable this check when workspace_id is provided + # action_access = await check_user_workspace_access( + # user_org_data, workspace_id, org_id + # ) + # if action_access: + # apps: List[AppDB] = await AppDB.find( + # AppDB.organization.id == ObjectId(org_id), + # AppDB.workspace.id == ObjectId(workspace_id), + # ).to_list() + # return [app_db_to_pydantic(app) for app in apps] + elif org_id is not None: + organization_access = await check_user_org_access(user_org_data, org_id) + if organization_access: apps: List[AppDB] = await AppDB.find( - AppDB.organization.id == ObjectId(org_id), - AppDB.workspace.id == ObjectId(workspace_id), + AppDB.organization.id == ObjectId(org_id) ).to_list() return [app_db_to_pydantic(app) for app in apps] diff --git a/agenta-backend/agenta_backend/services/selectors.py b/agenta-backend/agenta_backend/services/selectors.py index 8c5bac351f..0d42e36642 100644 --- a/agenta-backend/agenta_backend/services/selectors.py +++ b/agenta-backend/agenta_backend/services/selectors.py @@ -69,6 +69,6 @@ async def get_org_default_workspace(organization: OrganizationDB) -> WorkspaceDB """ workspace: WorkspaceDB = await WorkspaceDB.find_one( - WorkspaceDB.organization == organization.id, WorkspaceDB.type == "default" + WorkspaceDB.organization.id == organization.id, WorkspaceDB.type == "default" ) return workspace From be721582a132c9282e243f9b6fd0a39431868ff2 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Thu, 18 Jan 2024 09:16:03 +0100 Subject: [PATCH 21/99] register models for both oss and cloud --- .../agenta_backend/models/db_engine.py | 69 ++++++++++++------- 1 file changed, 45 insertions(+), 24 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index b2cf48c3e2..32ff21bd78 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -6,39 +6,52 @@ from beanie import init_beanie, Document from motor.motor_asyncio import AsyncIOMotorClient +if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import ( + APIKeyDB, + OrganizationDB_ as OrganizationDB, + WorkspaceDB, + AppEnvironmentDB_ as AppEnvironmentDB, + UserDB_ as UserDB, + ImageDB_ as ImageDB, + AppDB_ as AppDB, + DeploymentDB_ as DeploymentDB, + VariantBaseDB_ as VariantBaseDB, + AppVariantDB_ as AppVariantDB, + TestSetDB_ as TestSetDB, + EvaluatorConfigDB_ as EvaluatorConfigDB, + HumanEvaluationDB_ as HumanEvaluationDB, + HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, + EvaluationDB_ as EvaluationDB, + EvaluationScenarioDB_ as EvaluationScenarioDB, + ) +else: + from agenta_backend.models.db_models import ( + AppEnvironmentDB, + UserDB, + ImageDB, + AppDB, + DeploymentDB, + VariantBaseDB, + AppVariantDB, + TestSetDB, + EvaluatorConfigDB, + HumanEvaluationDB, + HumanEvaluationScenarioDB, + EvaluationDB, + EvaluationScenarioDB, + ) + from agenta_backend.models.db_models import ( - APIKeyDB, - AppEnvironmentDB, - OrganizationDB, - WorkspaceDB, - UserDB, - ImageDB, - AppDB, - DeploymentDB, - VariantBaseDB, - ConfigDB, - AppVariantDB, TemplateDB, - TestSetDB, - EvaluatorConfigDB, - HumanEvaluationDB, - HumanEvaluationScenarioDB, - EvaluationDB, - EvaluationScenarioDB, + ConfigDB, SpanDB, TraceDB, ) -# Configure and set logging level -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - # Define Document Models document_models: List[Document] = [ - APIKeyDB, AppEnvironmentDB, - OrganizationDB, - WorkspaceDB, UserDB, ImageDB, AppDB, @@ -57,6 +70,14 @@ TraceDB, ] +if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: + document_models = document_models + [OrganizationDB, WorkspaceDB, APIKeyDB] + + +# Configure and set logging level +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + class DBEngine: """ From f211f955d99a089bea749e0e68ace1c9c619215c Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Thu, 18 Jan 2024 18:26:07 +0100 Subject: [PATCH 22/99] Feat [ draft ]: remove Organization features, refactor and conditionally implement RBAC and use of org & workspace --- .../agenta_backend/models/api/api_models.py | 29 - .../models/api/organization_models.py | 36 - .../models/api/workspace_models.py | 49 -- .../agenta_backend/models/db_models.py | 192 ----- .../agenta_backend/routers/app_router.py | 417 +++++----- .../agenta_backend/routers/bases_router.py | 52 +- .../agenta_backend/routers/configs_router.py | 64 +- .../routers/container_router.py | 79 +- .../routers/environment_router.py | 61 +- .../routers/evaluation_router.py | 221 ++++-- .../routers/evaluators_router.py | 79 +- .../agenta_backend/routers/testset_router.py | 173 +++-- .../agenta_backend/routers/variants_router.py | 118 +-- .../agenta_backend/services/app_manager.py | 96 ++- .../services/container_manager.py | 17 +- .../agenta_backend/services/db_manager.py | 723 ++++++++---------- .../services/deployment_manager.py | 6 +- .../services/evaluation_service.py | 127 ++- .../services/evaluator_manager.py | 17 +- .../agenta_backend/services/selectors.py | 90 +-- .../agenta_backend/tasks/evaluations.py | 7 +- agenta-backend/agenta_backend/utils/common.py | 127 --- 22 files changed, 1237 insertions(+), 1543 deletions(-) delete mode 100644 agenta-backend/agenta_backend/models/api/organization_models.py delete mode 100644 agenta-backend/agenta_backend/models/api/workspace_models.py diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index 91ee121f2d..1d68dbf79e 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -35,8 +35,6 @@ class VariantAction(BaseModel): class CreateApp(BaseModel): app_name: str - organization_id: Optional[str] = None - workspace_id: Optional[str] = None class CreateAppOutput(BaseModel): @@ -58,8 +56,6 @@ class AppVariant(BaseModel): variant_name: str parameters: Optional[Dict[str, Any]] previous_variant_name: Optional[str] - organization_id: Optional[str] = None - workspace_id: Optional[str] = None base_name: Optional[str] config_name: Optional[str] @@ -75,8 +71,6 @@ class AppVariantOutput(BaseModel): variant_name: str parameters: Optional[Dict[str, Any]] previous_variant_name: Optional[str] - organization_id: str - workspace_id: Optional[str] user_id: str base_name: str base_id: str @@ -109,8 +103,6 @@ class AppVariantFromImage(BaseModel): variant_name: str parameters: Optional[Dict[str, Any]] previous_variant_name: Optional[str] - organization_id: Optional[str] = None - workspace_id: Optional[str] = None class RestartAppContainer(BaseModel): @@ -121,8 +113,6 @@ class Image(BaseModel): type: Optional[str] docker_id: str tags: str - organization_id: Optional[str] = None - workspace_id: Optional[str] = None class AddVariantFromImagePayload(BaseModel): @@ -175,16 +165,6 @@ class CreateAppVariant(BaseModel): app_name: str template_id: str env_vars: Dict[str, str] - organization_id: Optional[str] = None - workspace_id: Optional[str] = None - - -class InviteRequest(BaseModel): - email: str - - -class InviteToken(BaseModel): - token: str class Environment(BaseModel): @@ -192,8 +172,6 @@ class Environment(BaseModel): deployed_app_variant: Optional[str] deployed_base_name: Optional[str] deployed_config_name: Optional[str] - organization_id: Optional[str] = None - workspace_id: Optional[str] = None class DeployToEnvironmentPayload(BaseModel): @@ -217,13 +195,6 @@ class PostVariantConfigPayload(BaseModel): overwrite: bool -class ListAPIKeysOutput(BaseModel): - prefix: str - created_at: datetime - last_used_at: datetime = None - expiration_date: datetime = None - - class BaseOutput(BaseModel): base_id: str base_name: str diff --git a/agenta-backend/agenta_backend/models/api/organization_models.py b/agenta-backend/agenta_backend/models/api/organization_models.py deleted file mode 100644 index 6036cf32da..0000000000 --- a/agenta-backend/agenta_backend/models/api/organization_models.py +++ /dev/null @@ -1,36 +0,0 @@ -from datetime import datetime -from typing import Optional, List -from pydantic import BaseModel, Field - - -class TimestampModel(BaseModel): - created_at: datetime = Field(datetime.utcnow()) - updated_at: datetime = Field(datetime.utcnow()) - - -class Organization(BaseModel): - id: Optional[str] - name: str - description: Optional[str] - type: Optional[str] - owner: str - members: Optional[List[str]] - invitations: Optional[List] - - -class CreateOrganization(BaseModel): - name: str - description: Optional[str] - type: Optional[str] - owner: Optional[str] - - -class OrganizationUpdate(BaseModel): - name: Optional[str] - description: Optional[str] - updated_at: Optional[datetime] - - -class OrganizationOutput(BaseModel): - id: str - name: str diff --git a/agenta-backend/agenta_backend/models/api/workspace_models.py b/agenta-backend/agenta_backend/models/api/workspace_models.py deleted file mode 100644 index 3491364243..0000000000 --- a/agenta-backend/agenta_backend/models/api/workspace_models.py +++ /dev/null @@ -1,49 +0,0 @@ -from datetime import datetime -from pydantic import BaseModel -from typing import Optional, List, Dict -from agenta_backend.models.api.user_models import TimestampModel -from agenta_backend.models.db_models import WorkspaceRole, Permission - - -class WorkspacePermission(BaseModel): - role_name: WorkspaceRole - permissions: List[Permission] - - -class WorkspaceMember(BaseModel): - user_id: str - roles: List[WorkspacePermission] - - -class WorkspaceMemberOutput(BaseModel): - user: Dict - roles: List[WorkspacePermission] - - -class Workspace(BaseModel): - id: Optional[str] - name: str - description: Optional[str] - type: Optional[str] - members: Optional[List[WorkspaceMember]] - - -class WorkspaceOutput(TimestampModel): - id: str - name: str - description: Optional[str] - type: Optional[str] - organization: str - members: Optional[List[WorkspaceMemberOutput]] - - -class CreateWorkspace(BaseModel): - name: str - description: Optional[str] - type: Optional[str] - - -class UpdateWorkspace(BaseModel): - name: Optional[str] - description: Optional[str] - updated_at: Optional[datetime] diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 7193ab1ea0..6b3ab5e3a3 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -7,178 +7,10 @@ from beanie import Document, Link, PydanticObjectId -class WorkspaceRole(str, Enum): - OWNER = "owner" - MEMBER = "member" - WORKSPACE_ADMIN = "workspace_admin" - - -class Permission(str, Enum): - # general - MODIFY_CONFIGURATIONS = "modify_configurations" - CHANGE_USER_ROLES = "change_user_roles" - READ_SYSTEM = "read_system" - - VIEW_APPLICATION = "view_application" - CREATE_APPLICATION = "create_application" - EDIT_APPLICATION = "edit_application" - DELETE_APPLICATION = "delete_application" - - # Testset - VIEW_TESTSET = "view_testset" - CREATE_TESTSET = "create_testset" - EDIT_TESTSET = "edit_testset" - DELETE_TESTSET = "delete_testset" - - # Evaluation - VIEW_EVALUATION = "view_evaluation" - CREATE_EVALUATION = "create_evaluation" - RUN_EVALUATIONS = "run_evaluations" - EDIT_EVALUATION = "edit_evaluation" - DELETE_EVALUATION = "delete_evaluation" - - # Deployment - DEPLOY_APPLICATION = "deploy_application" - - # Workspace - VIEW_WORKSPACE = "view_workspace" - CREATE_WORKSPACE = "create_workspace" - EDIT_WORKSPACE = "edit_workspace" - DELETE_WORKSPACE = "delete_workspace" - MODIFY_USER_ROLES = "modify_user_roles" - ADD_NEW_USER_TO_WORKSPACE = "add_new_user_to_workspace" - - # Organization - ADD_NEW_USER_TO_ORGANIZATION = "add_new_user_to_organization" - EDIT_ORGANIZATION = "edit_organization" - - -class WorkspacePermissionDB(BaseModel): - role_name: WorkspaceRole - permissions: List[Permission] - - -class WorkspaceMemberDB(BaseModel): - user_id: PydanticObjectId - roles: List[WorkspacePermissionDB] - - -class APIKeyDB(Document): - prefix: str - hashed_key: str - user_id: str - rate_limit: int = Field(default=0) - hidden: Optional[bool] = Field(default=False) - expiration_date: Optional[datetime] - created_at: Optional[datetime] = datetime.utcnow() - updated_at: Optional[datetime] - - class Settings: - name = "api_keys" - - -class InvitationDB(BaseModel): - token: str = Field(unique=True) - email: str - organization: str - workspace = str - workspace_role: Optional[List[WorkspacePermissionDB]] - expiration_date: datetime = Field(default="0") - used: bool = False - - -class OrganizationDB(Document): - name: str = Field(default="agenta") - description: Optional[str] = Field(default="") - type: Optional[str] - owner: str # user id - members: Optional[List[PydanticObjectId]] - invitations: Optional[List[InvitationDB]] = [] - workspaces: Optional[List[PydanticObjectId]] = [] - created_at: Optional[datetime] = Field(default=datetime.utcnow()) - updated_at: Optional[datetime] = Field(default=datetime.utcnow()) - - class Settings: - name = "organizations" - - -class WorkspaceDB(Document): - name: str - type: Optional[str] - description: Optional[str] = Field(default="") - organization: Link[OrganizationDB] - members: Optional[List[WorkspaceMemberDB]] = [] - created_at: Optional[datetime] = Field(default=datetime.utcnow()) - updated_at: Optional[datetime] = Field(default=datetime.utcnow()) - - def get_member_roles( - self, user_id: PydanticObjectId - ) -> Optional[List[WorkspacePermissionDB]]: - for member in self.members: - if member.user_id == user_id: - return member.roles - return None - - def get_member_role_names(self, user_id: PydanticObjectId) -> List[str]: - roles = self.get_member_roles(user_id) - return [role.role_name for role in roles] if roles else [] - - def get_all_members(self) -> List[PydanticObjectId]: - return [member.user_id for member in self.members] - - def get_member_with_roles( - self, user_id: PydanticObjectId - ) -> Optional[WorkspaceMemberDB]: - for member in self.members: - if member.user_id == user_id: - return member - return None - - def get_member_permissions( - self, user_id: PydanticObjectId, role_to_check: WorkspaceRole - ) -> List[Permission]: - roles = self.get_member_roles(user_id) - if roles: - for role in roles: - if role.role_name == role_to_check: - return role.permissions - return [] - - def has_permission(self, user_id: PydanticObjectId, permission: Permission) -> bool: - roles = self.get_member_roles(user_id) - if roles: - for role in roles: - if permission in role.permissions: - return True - return False - - def has_role(self, user_id: PydanticObjectId, role_to_check: WorkspaceRole) -> bool: - roles = self.get_member_roles(user_id) - if roles: - for role in roles: - if role.role_name == role_to_check: - return True - return False - - def is_owner(self, user_id: PydanticObjectId) -> bool: - for member in self.members: - if ( - member.user_id == user_id - and WorkspaceRole.OWNER in self.get_member_role_names(user_id) - ): - return True - return False - - class Settings: - name = "workspaces" - - class UserDB(Document): uid: str = Field(default="0", unique=True, index=True) username: str = Field(default="agenta") email: str = Field(default="demo@agenta.ai", unique=True) - organizations: Optional[List[PydanticObjectId]] = [] - workspaces: Optional[List[PydanticObjectId]] = [] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -195,8 +27,6 @@ class ImageDB(Document): tags: Optional[str] deletable: bool = Field(default=True) user: Link[UserDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) deletable: bool = Field(default=True) @@ -207,8 +37,6 @@ class Settings: class AppDB(Document): app_name: str - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] user: Link[UserDB] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -219,8 +47,6 @@ class Settings: class DeploymentDB(Document): app: Link[AppDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] user: Link[UserDB] container_name: Optional[str] container_id: Optional[str] @@ -235,8 +61,6 @@ class Settings: class VariantBaseDB(Document): app: Link[AppDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] user: Link[UserDB] base_name: str image: Link[ImageDB] @@ -272,8 +96,6 @@ class AppVariantDB(Document): variant_name: str image: Link[ImageDB] user: Link[UserDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] parameters: Dict[str, Any] = Field(default=dict) # TODO: deprecated. remove previous_variant_name: Optional[str] # TODO: deprecated. remove base_name: Optional[str] @@ -295,8 +117,6 @@ class AppEnvironmentDB(Document): app: Link[AppDB] name: str user: Link[UserDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] deployed_app_variant: Optional[PydanticObjectId] deployment: Optional[PydanticObjectId] # reference to deployment created_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -326,8 +146,6 @@ class TestSetDB(Document): app: Link[AppDB] csvdata: List[Dict[str, str]] user: Link[UserDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -337,8 +155,6 @@ class Settings: class EvaluatorConfigDB(Document): app: Link[AppDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] user: Link[UserDB] name: str evaluator_key: str @@ -388,8 +204,6 @@ class HumanEvaluationScenarioOutput(BaseModel): class HumanEvaluationDB(Document): app: Link[AppDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] user: Link[UserDB] status: str evaluation_type: str @@ -404,8 +218,6 @@ class Settings: class HumanEvaluationScenarioDB(Document): user: Link[UserDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] evaluation: Link[HumanEvaluationDB] inputs: List[HumanEvaluationScenarioInput] outputs: List[HumanEvaluationScenarioOutput] @@ -423,8 +235,6 @@ class Settings: class EvaluationDB(Document): app: Link[AppDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] user: Link[UserDB] status: str = Field(default="EVALUATION_INITIALIZED") testset: Link[TestSetDB] @@ -440,8 +250,6 @@ class Settings: class EvaluationScenarioDB(Document): user: Link[UserDB] - organization: Link[OrganizationDB] - workspace: Link[WorkspaceDB] evaluation: Link[EvaluationDB] variant_id: PydanticObjectId inputs: List[EvaluationScenarioInputDB] diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 9cd5d5c762..858ff666ff 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -8,13 +8,7 @@ from agenta_backend.models import converters from beanie import PydanticObjectId as ObjectId from agenta_backend.utils.common import APIRouter -from agenta_backend.utils.common import check_rbac_permission -from agenta_backend.models.db_models import Permission, WorkspaceRole -from agenta_backend.services.selectors import ( - get_user_own_org, - get_org_default_workspace, -) from agenta_backend.services import ( db_manager, app_manager, @@ -22,27 +16,48 @@ ) from agenta_backend.models.api.api_models import ( App, - Image, - CreateApp, CreateAppOutput, - CreateAppVariant, - AppVariantOutput, EnvironmentOutput, AddVariantFromImagePayload, ) -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.api.api_models import ( + ImageDB_ as Image, + CreateApp_ as CreateApp, + AppVariantOutput_ as AppVariantOutput, + CreateAppVariant_ as CreateAppVariant, + ) +else: + from agenta_backend.models.api.api_models import ( + Image, + CreateApp, + AppVariantOutput, + CreateAppVariant, + ) +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.services import db_manager_ee from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, + get_user_own_org, + get_user_org_and_workspace_id, + get_org_default_workspace, ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id + from agenta_backend.commons.utils.permissions import ( + check_action_access, + check_rbac_permission + ) + from agenta_backend.commons.models.db_models import ( + Permission, + WorkspaceRole + ) -if os.environ["FEATURE_FLAG"] in ["cloud"]: + +if FEATURE_FLAG in ["cloud"]: from agenta_backend.cloud.services import ( lambda_deployment_manager as deployment_manager, ) # noqa pylint: disable-all -elif os.environ["FEATURE_FLAG"] in ["ee"]: +elif FEATURE_FLAG in ["ee"]: from agenta_backend.ee.services import ( deployment_manager, ) # noqa pylint: disable-all @@ -74,25 +89,22 @@ async def list_app_variants( List[AppVariantOutput]: A list of app variants for the given app ID. """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=WorkspaceRole.MEMBER, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + role=WorkspaceRole.VIEWER, ) + logger.debug(f"User has Permission to list app variants: {has_permission}") + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) - app_variants = await db_manager.list_app_variants( - app_id=app_id, **user_org_data - ) + app_variants = await db_manager.list_app_variants(app_id=app_id) return [ await converters.app_variant_db_to_output(app_variant) for app_variant in app_variants @@ -127,26 +139,24 @@ async def get_variant_by_env( AppVariantOutput: The retrieved app variant. """ try: - # Retrieve the user and organization ID based on the session token - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=WorkspaceRole.MEMBER, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + role=WorkspaceRole.VIEWER, ) + logger.debug(f"user has Permission to get variant by environment: {has_permission}") + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) # Fetch the app variant using the provided app_id and environment app_variant_db = await db_manager.get_app_variant_by_app_name_and_environment( - app_id=app_id, environment=environment, **user_org_data + app_id=app_id, environment=environment ) # Check if the fetched app variant is None and raise exception if it is @@ -182,43 +192,62 @@ async def create_app( HTTPException: If there is an error creating the app or the user does not have permission to access the app. """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - - if payload.organization_id: - organization_id = payload.organization_id - organization = await db_manager.get_organization_object(organization_id) - else: - organization = await get_user_own_org(user_org_data["uid"]) - organization_id = str(organization.id) - if not organization_id: - raise HTTPException( - status_code=403, - detail="User Organization not found", + if FEATURE_FLAG in ["cloud", "ee"]: + + try: + user_org_workspace_data = await get_user_org_and_workspace_id(request.state.user_id) + if user_org_workspace_data is None: + raise HTTPException( + status_code=400, + detail="Failed to get user org and workspace data", + ) + + if payload.organization_id: + organization_id = payload.organization_id + organization = await db_manager_ee.get_organization(organization_id) + else: + organization = await get_user_own_org(user_org_workspace_data["uid"]) + organization_id = str(organization.id) + + if not organization: + raise HTTPException( + status_code=400, + detail="User Organization not found", + ) + + if payload.workspace_id: + workspace_id = payload.workspace_id + workspace = db_manager_ee.get_workspace(workspace_id) + else: + workspace = await get_org_default_workspace(organization) + + if not workspace: + raise HTTPException( + status_code=400, + detail="User Organization not found", + ) + + has_permission = await check_rbac_permission( + user_org_workspace_data=user_org_workspace_data, + workspace_id=ObjectId(workspace_id), + organization=organization, + permission=Permission.CREATE_APPLICATION, ) - - if payload.workspace_id: - workspace_id = payload.workspace_id - else: - workspace = await get_org_default_workspace(organization) - workspace_id = str(workspace.id) - - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=ObjectId(workspace_id), - organization_id=ObjectId(organization_id), - permission=Permission.CREATE_APPLICATION, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, - ) + logger.debug(f"User has Permission to Create Application: {has_permission}") + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) app_db = await db_manager.create_app_and_envs( - payload.app_name, organization_id, workspace_id, **user_org_data + payload.app_name, + request.state.user_id, + organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, ) return CreateAppOutput(app_id=str(app_db.id), app_name=str(app_db.app_name)) except Exception as e: @@ -246,10 +275,9 @@ async def list_apps( Raises: HTTPException: If there was an error retrieving the list of apps. """ - try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) + try: apps = await db_manager.list_apps( - app_name, org_id, workspace_id, **user_org_data + app_name, org_id, workspace_id, request.state.user_id ) return apps except Exception as e: @@ -278,7 +306,7 @@ async def add_variant_from_image( dict: The newly added variant. """ - if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"]: + if FEATURE_FLAG not in ["cloud", "ee"]: image = Image( type="image", docker_id=payload.docker_id, @@ -293,21 +321,22 @@ async def add_variant_from_image( raise HTTPException(status_code=404, detail="Image not found") try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - permission=Permission.CREATE_APPLICATION, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + role=WorkspaceRole.VIEWER, ) + logger.debug(f"User has Permission to create app from image: {has_permission}") + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + app = await db_manager.fetch_app_by_id(app_id) variant_db = await app_manager.add_variant_based_on_image( @@ -318,7 +347,7 @@ async def add_variant_from_image( base_name=payload.base_name, config_name=payload.config_name, is_template_image=False, - **user_org_data, + user_uid=request.state.user_id, ) app_variant_db = await db_manager.fetch_app_variant_by_id(str(variant_db.id)) @@ -338,24 +367,24 @@ async def remove_app(app_id: str, request: Request): app -- App to remove """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - permission=Permission.DELETE_APPLICATION, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + permission=Permission.DELETE_APPLICATION, ) + logger.debug(f"User has Permission to delete app: {has_permission}") + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) else: - await app_manager.remove_app(app_id=app_id, **user_org_data) + await app_manager.remove_app(app_id=app_id, user_uid=request.state.user_id) except DockerException as e: detail = f"Docker error while trying to remove the app: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -388,94 +417,102 @@ async def create_app_and_variant_from_template( try: logger.debug("Start: Creating app and variant from template") - # Get user and org id - logger.debug("Step 1: Getting user and organization ID") - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - - logger.debug("Step 2: Setting organization ID") - if payload.organization_id is None: - organization = await get_user_own_org(user_org_data["uid"]) - organization_id = str(organization.id) - else: - organization_id = payload.organization_id - organization = await db_manager.get_organization_object(organization_id) - - logger.debug("Step 3: Setting workspace ID") - if payload.workspace_id is None: - workspace = await get_org_default_workspace(organization) - workspace_id = str(workspace.id) - else: - workspace_id = payload.workspace_id - - logger.debug("Step 4: Checking user has permission to create app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=ObjectId(workspace_id), - organization_id=ObjectId(organization_id), - permission=Permission.CREATE_APPLICATION, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + + # Get user and org id + logger.debug("Step 1: Getting user and organization ID") + user_org_data: dict = await get_user_org_and_workspace_id(request.state.user_id) + + logger.debug("Step 2: Setting organization ID") + if payload.organization_id is None: + organization = await get_user_own_org(user_org_data["uid"]) + organization_id = str(organization.id) + else: + organization_id = payload.organization_id + organization = await db_manager_ee.get_organization(organization_id) + + logger.debug("Step 3: Setting workspace ID") + if payload.workspace_id is None: + workspace = await get_org_default_workspace(organization) + workspace_id = str(workspace.id) + else: + workspace_id = payload.workspace_id + + logger.debug("Step 4: Checking user has permission to create app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=ObjectId(workspace_id), + organization_id=ObjectId(organization_id), + permission=Permission.CREATE_APPLICATION, ) + logger.debug(f"User has Permission to create app from template: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) - logger.debug(f"Step 5: Checking if app {payload.app_name} already exists") + logger.debug(f"Step 5: Checking if app {payload.app_name} already exists" if FEATURE_FLAG in ["cloud", "ee"] else f"Step 1: Checking if app {payload.app_name} already exists") app_name = payload.app_name.lower() - app = await db_manager.fetch_app_by_name_and_organization_and_workspace( - app_name, organization_id, workspace_id, **user_org_data + app = await db_manager.fetch_app_by_name_and_parameters( + app_name, + request.state.user_id, + organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, ) if app is not None: raise Exception( f"App with name {app_name} already exists", ) - logger.debug("Step 6: Creating new app and initializing environments") + logger.debug("Step 6: Creating new app and initializing environments" if FEATURE_FLAG in ["cloud", "ee"] else "Step 2: Creating new app and initializing environments") if app is None: app = await db_manager.create_app_and_envs( - app_name, organization_id, workspace_id, **user_org_data + app_name, + request.state.user_id, + organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, ) - logger.debug("Step 7: Retrieve template from db") + logger.debug("Step 7: Retrieve template from db" if FEATURE_FLAG in ["cloud", "ee"] else "Step 3: Retrieve template from db") template_db = await db_manager.get_template(payload.template_id) repo_name = os.environ.get("AGENTA_TEMPLATE_REPO", "agentaai/templates_v2") image_name = f"{repo_name}:{template_db.name}" logger.debug( - "Step 8: Creating image instance and adding variant based on image" + "Step 8: Creating image instance and adding variant based on image" if FEATURE_FLAG in ["cloud", "ee"] else "Step 4: Creating image instance and adding variant based on image" ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, variant_name="app.default", docker_id_or_template_uri=template_db.template_uri - if os.environ["FEATURE_FLAG"] in ["cloud", "ee"] + if FEATURE_FLAG in ["cloud", "ee"] else template_db.digest, tags=f"{image_name}" - if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"] + if FEATURE_FLAG not in ["cloud", "ee"] else None, base_name="app", config_name="default", is_template_image=True, - **user_org_data, ) - logger.debug("Step 9: Creating testset for app variant") + logger.debug("Step 9: Creating testset for app variant" if FEATURE_FLAG in ["cloud", "ee"] else "Step 5: Creating testset for app variant") await db_manager.add_testset_to_app_variant( app_id=str(app.id), - org_id=organization_id, - workspace_id=workspace_id, + org_id=organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace_id=workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, template_name=template_db.name, app_name=app.app_name, - **user_org_data, + user_uid=request.state.user_id, ) - logger.debug("Step 10: We create ready-to use evaluators") + logger.debug("Step 10: We create ready-to use evaluators" if FEATURE_FLAG in ["cloud", "ee"] else "Step 6: We create ready-to use evaluators") await evaluator_manager.create_ready_to_use_evaluators(app=app) - logger.debug("Step 11: Starting variant and injecting environment variables") - if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: + logger.debug("Step 11: Starting variant and injecting environment variables" if FEATURE_FLAG in ["cloud", "ee"] else "Step 7: Starting variant and injecting environment variables") + if FEATURE_FLAG in ["cloud", "ee"]: if not os.environ["OPENAI_API_KEY"]: raise Exception( "Unable to start app container. Please file an issue by clicking on the button below.", @@ -520,34 +557,26 @@ async def list_environments( """ logger.debug(f"Listing environments for app: {app_id}") try: - logger.debug("get user and org data") - user_and_org_data: dict = await get_user_and_org_id(request.state.user_id) - - # Check if user has access to app - logger.debug("check_access_to_app") - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_and_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=WorkspaceRole.MEMBER, - ) - logger.debug(f"access_app: {has_permission}") - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, - ) - else: - environments_db = await db_manager.list_environments( - app_id=app_id, **user_and_org_data + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + role=WorkspaceRole.VIEWER, ) - logger.debug(f"environments_db: {environments_db}") - return [ - await converters.environment_db_to_output(env) - for env in environments_db - ] + logger.debug(f"User has Permission to list environments: {has_permission}") + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + + environments_db = await db_manager.list_environments(app_id=app_id) + logger.debug(f"environments_db: {environments_db}") + return [ + await converters.environment_db_to_output(env) + for env in environments_db + ] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index ae58937800..4a13c6caed 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -1,27 +1,25 @@ import os +import logging + from typing import List, Optional +from fastapi.responses import JSONResponse from fastapi import Request, HTTPException + +from agenta_backend.models import converters +from agenta_backend.services import db_manager from agenta_backend.utils.common import APIRouter from agenta_backend.models.api.api_models import BaseOutput -from fastapi.responses import JSONResponse -from agenta_backend.services import db_manager -from agenta_backend.models import converters -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id -from agenta_backend.utils.common import check_access_to_app +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import WorkspaceRole + from agenta_backend.cmmons.utils.permissions import check_action_access -import logging +router = APIRouter() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -router = APIRouter() - @router.get("/", response_model=List[BaseOutput], operation_id="list_bases") async def list_bases( @@ -44,20 +42,22 @@ async def list_bases( HTTPException: If there was an error retrieving the bases. """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_app = await check_access_to_app( - user_org_data=user_org_data, app_id=app_id - ) - if not access_app: - error_msg = f"You cannot access app: {app_id}" - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if os.environ["FEATURE_FLAG"] in ["cloud", "ee"] and app_id is not None: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + role=WorkspaceRole.VIEWER, ) - bases = await db_manager.list_bases_for_app_id( - app_id, base_name, **user_org_data - ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + bases = await db_manager.list_bases_for_app_id(app_id, base_name) return [converters.base_db_to_pydantic(base) for base in bases] except Exception as e: logger.error(f"list_bases exception ===> {e}") diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index 0f0704b762..cc1395b27f 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -1,8 +1,10 @@ import os +import logging + from typing import Optional +from fastapi.responses import JSONResponse from fastapi import Request, HTTPException from agenta_backend.utils.common import APIRouter -import logging from agenta_backend.models.api.api_models import ( SaveConfigPayload, @@ -13,19 +15,16 @@ app_manager, ) -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import Permission + from agenta_backend.cmmons.utils.permissions import check_action_access +router = APIRouter() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -router = APIRouter() - @router.post("/", operation_id="save_config") async def save_config( @@ -33,11 +32,23 @@ async def save_config( request: Request, ): try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - base_db = await db_manager.fetch_base_and_check_access( - payload.base_id, user_org_data - ) - variants_db = await db_manager.list_variants_for_base(base_db, **user_org_data) + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=payload.base_id, + object_type="base", + permission=Permission.MODIFY_CONFIGURATIONS, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + base_db = await db_manager.fetch_base_by_id(payload.base_id) + variants_db = await db_manager.list_variants_for_base(base_db) variant_to_overwrite = None for variant_db in variants_db: if variant_db.config_name == payload.config_name: @@ -49,7 +60,6 @@ async def save_config( await app_manager.update_variant_parameters( app_variant_id=str(variant_to_overwrite.id), parameters=payload.parameters, - **user_org_data, ) else: raise HTTPException( @@ -64,7 +74,7 @@ async def save_config( base_db=base_db, new_config_name=payload.config_name, parameters=payload.parameters, - **user_org_data, + user_uid=request.state.user_id, ) except HTTPException as e: logger.error(f"save_config http exception ===> {e.detail}") @@ -83,8 +93,22 @@ async def get_config( ): try: # detemine whether the user has access to the base - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - base_db = await db_manager.fetch_base_and_check_access(base_id, user_org_data) + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=base_id, + object_type="base", + permission=Permission.MODIFY_CONFIGURATIONS, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + base_db = await db_manager.fetch_base_by_id(base_id) # in case environment_name is provided, find the variant deployed if environment_name: app_environments = await db_manager.list_environments( @@ -109,9 +133,7 @@ async def get_config( ) config = found_variant.config elif config_name: - variants_db = await db_manager.list_variants_for_base( - base_db, **user_org_data - ) + variants_db = await db_manager.list_variants_for_base(base_db) found_variant = None for variant_db in variants_db: if variant_db.config_name == config_name: diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 0f71250fa0..7ce307be1b 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -1,21 +1,22 @@ import os import logging -from typing import List, Optional, Union +from typing import List, Optional, Union from fastapi.responses import JSONResponse from fastapi import Request, UploadFile, HTTPException -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id +from agenta_backend.services import db_manager +from agenta_backend.utils.common import APIRouter +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import WorkspaceRole, Permission + from agenta_backend.commons.utils.permissions import check_action_access -if os.environ["FEATURE_FLAG"] in ["cloud"]: + +if FEATURE_FLAG in ["cloud"]: from agenta_backend.cloud.services import container_manager -elif os.environ["FEATURE_FLAG"] in ["ee"]: +elif FEATURE_FLAG in ["ee"]: from agenta_backend.ee.services import container_manager else: from agenta_backend.services import container_manager @@ -26,15 +27,12 @@ RestartAppContainer, Template, ) -from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter +router = APIRouter() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -router = APIRouter() - # TODO: We need to improve this to use the introduced abstraction to also use start and stop service # * Edit: someone remind me (abram) to work on this. @@ -57,13 +55,23 @@ async def build_image( Returns: Image: The Docker image that was built. """ - # Get user and org id - user_org_data: dict = await get_user_and_org_id(request.state.user_id) + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + permission=Permission.CREATE_APPLICATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) # Check app access - app_db = await db_manager.fetch_app_and_check_access( - app_id=app_id, user_org_data=user_org_data - ) + app_db = await db_manager.fetch_app_by_id(app_id) image_result = await container_manager.build_image( app_db=app_db, @@ -84,12 +92,8 @@ async def restart_docker_container( Args: payload (RestartAppContainer) -- the required data (app_name and variant_name) """ - logger.debug(f"Restarting container for variant {payload.variant_id}") - # Get user and org id - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - app_variant_db = await db_manager.fetch_app_variant_and_check_access( - app_variant_id=payload.variant_id, user_org_data=user_org_data - ) + logger.debug(f"Restarting container for variant {payload.variant_id}") + app_variant_db = await db_manager.fetch_app_variant_by_id(payload.variant_id) try: deployment = await db_manager.get_deployment_by_objectid( app_variant_db.base.deployment @@ -144,11 +148,26 @@ async def construct_app_container_url( Raises: HTTPException: If the base or variant cannot be found or the user does not have access. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - if base_id: - base_db = await db_manager.fetch_base_and_check_access( - base_id=base_id, user_org_data=user_org_data + # assert that one of base_id or variant_id is provided + assert base_id or variant_id, "Please provide either base_id or variant_id" + + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=base_id if base_id else variant_id, + object_type="base" if base_id else "app_variant", + role=WorkspaceRole.VIEWER, ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + if base_id: + base_db = await db_manager.fetch_base_by_id(base_id) # TODO: Add status check if base_db.status == "running" if base_db.deployment: deployment = await db_manager.get_deployment_by_objectid(base_db.deployment) @@ -161,9 +180,7 @@ async def construct_app_container_url( return URI(uri=uri) elif variant_id: - variant_db = await db_manager.fetch_app_variant_and_check_access( - app_variant_id=variant_id, user_org_data=user_org_data - ) + variant_db = await db_manager.fetch_app_variant_by_id(variant_id) deployment = await db_manager.get_deployment_by_objectid( variant_db.base.deployment ) diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index a64b11027e..653d948375 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -1,29 +1,22 @@ import os -from typing import List +import logging from fastapi.responses import JSONResponse -from agenta_backend.services import db_manager from fastapi import Request, HTTPException + +from agenta_backend.services import db_manager from agenta_backend.utils.common import APIRouter -from agenta_backend.utils.common import check_access_to_app, check_access_to_variant -from agenta_backend.models.api.api_models import ( - EnvironmentOutput, - DeployToEnvironmentPayload, -) +from agenta_backend.models.api.api_models import DeployToEnvironmentPayload -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id -import logging +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import Permission + from agenta_backend.commons.utils.permissions import check_action_access +router = APIRouter() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -router = APIRouter() - @router.post("/deploy/", operation_id="deploy_to_environment") async def deploy_to_environment( @@ -41,24 +34,24 @@ async def deploy_to_environment( HTTPException: If the deployment fails. """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - - # Check if has app access - access_app = await check_access_to_variant( - user_org_data, variant_id=payload.variant_id - ) - - if not access_app: - error_msg = f"You do not have access to this variant: {payload.variant_id}" - return JSONResponse( - {"detail": error_msg}, - status_code=400, - ) - else: - await db_manager.deploy_to_environment( - environment_name=payload.environment_name, - variant_id=payload.variant_id, - **user_org_data, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=payload.variant_id, + object_type="app_variant", + permission=Permission.DEPLOY_APPLICATION, ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + await db_manager.deploy_to_environment( + environment_name=payload.environment_name, + variant_id=payload.variant_id, + ) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 3cc5a572fc..23dee0c531 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -1,12 +1,15 @@ import os import secrets -from typing import Any, List +import logging +from typing import Any, List from fastapi.responses import JSONResponse -from fastapi.encoders import jsonable_encoder from fastapi import HTTPException, Request, status, Response from agenta_backend.utils.common import APIRouter +from agenta_backend.tasks.evaluations import evaluate +from agenta_backend.services import evaluation_service, db_manager + from agenta_backend.models.api.evaluation_model import ( Evaluation, EvaluationScenario, @@ -15,25 +18,19 @@ DeleteEvaluation, EvaluationWebhook, ) -from agenta_backend.services import db_manager -from agenta_backend.tasks.evaluations import evaluate -from agenta_backend.services import evaluation_service -from agenta_backend.utils.common import check_access_to_app from agenta_backend.services.evaluator_manager import ( check_ai_critique_inputs, ) -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( # noqa pylint: disable-all - get_user_and_org_id, - ) -else: - from agenta_backend.services.selectors import get_user_and_org_id - +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import Permission + from agenta_backend.commons.utils.permissions import check_action_access -# Initialize api router router = APIRouter() +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) @router.post("/", response_model=List[Evaluation], operation_id="create_evaluation") @@ -48,18 +45,21 @@ async def create_evaluation( _description_ """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_app = await check_access_to_app( - user_org_data=user_org_data, - app_id=payload.app_id, - check_owner=False, - ) - if not access_app: - error_msg = f"You do not have access to this app: {payload.app_id}" - return JSONResponse( - {"detail": error_msg}, - status_code=400, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=payload.app_id, + object_type="app", + permission=Permission.CREATE_EVALUATION, ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: raise HTTPException(status_code=404, detail="App not found") @@ -112,11 +112,22 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): """ try: - # Get user and organization id - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - evaluation = await evaluation_service.fetch_evaluation( - evaluation_id, **user_org_data - ) + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=evaluation_id, + object_type="evaluation", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + evaluation = await evaluation_service.fetch_evaluation(evaluation_id) return {"status": evaluation.status} except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -135,11 +146,22 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): """ try: - # Get user and organization id - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - results = await evaluation_service.retrieve_evaluation_results( - evaluation_id, **user_org_data - ) + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=evaluation_id, + object_type="evaluation", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + results = await evaluation_service.retrieve_evaluation_results(evaluation_id) return {"results": results, "evaluation_id": evaluation_id} except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -166,12 +188,27 @@ async def fetch_evaluation_scenarios( List[EvaluationScenario]: A list of evaluation scenarios. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - eval_scenarios = await evaluation_service.fetch_evaluation_scenarios_for_evaluation( - evaluation_id, **user_org_data - ) - - return eval_scenarios + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=evaluation_id, + object_type="evaluation", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + eval_scenarios = await evaluation_service.fetch_evaluation_scenarios_for_evaluation(evaluation_id) + return eval_scenarios + + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) @router.get("/", response_model=List[Evaluation]) @@ -187,10 +224,25 @@ async def fetch_list_evaluations( Returns: List[Evaluation]: A list of evaluations. """ - user_org_data = await get_user_and_org_id(request.state.user_id) - return await evaluation_service.fetch_list_evaluations( - app_id=app_id, **user_org_data - ) + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + return await evaluation_service.fetch_list_evaluations(app_id) + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) @router.get( @@ -208,8 +260,25 @@ async def fetch_evaluation( Returns: Evaluation: The fetched evaluation. """ - user_org_data = await get_user_and_org_id(request.state.user_id) - return await evaluation_service.fetch_evaluation(evaluation_id, **user_org_data) + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=evaluation_id, + object_type="evaluation", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + return await evaluation_service.fetch_evaluation(evaluation_id) + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) @router.delete("/", response_model=List[str], operation_id="delete_evaluations") @@ -227,12 +296,27 @@ async def delete_evaluations( A list of the deleted comparison tables' IDs. """ - # Get user and organization id - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - await evaluation_service.delete_evaluations( - delete_evaluations.evaluations_ids, **user_org_data - ) - return Response(status_code=status.HTTP_204_NO_CONTENT) + try: + if FEATURE_FLAG in ["cloud", "ee"]: + for evaluation_id in delete_evaluations.evaluations_ids: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=evaluation_id, + object_type="evaluation", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + await evaluation_service.delete_evaluations(delete_evaluations.evaluations_ids) + return Response(status_code=status.HTTP_204_NO_CONTENT) + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) @router.post( @@ -272,10 +356,27 @@ async def fetch_evaluation_scenarios( Returns: List[EvaluationScenario]: A list of evaluation scenarios. """ - evaluations_ids_list = evaluations_ids.split(",") - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - eval_scenarios = await evaluation_service.compare_evaluations_scenarios( - evaluations_ids_list, **user_org_data - ) - - return eval_scenarios + try: + evaluations_ids_list = evaluations_ids.split(",") + + if FEATURE_FLAG in ["cloud", "ee"]: + for evaluation_id in evaluations_ids_list: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=evaluation_id, + object_type="evaluation", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + eval_scenarios = await evaluation_service.compare_evaluations_scenarios(evaluations_ids_list) + + return eval_scenarios + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index ef6d25670c..ce0e372fc5 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -1,11 +1,14 @@ import os import json -from typing import List import logging +from typing import List -from fastapi import HTTPException, APIRouter, Query +from fastapi import HTTPException, Request from fastapi.responses import JSONResponse +from agenta_backend.utils.common import APIRouter +from agenta_backend.services import evaluator_manager + from agenta_backend.models.api.evaluation_model import ( Evaluator, EvaluatorConfig, @@ -13,20 +16,10 @@ UpdateEvaluatorConfig, ) -from agenta_backend.services import ( - db_manager, -) - -from agenta_backend.services import evaluator_manager - -from agenta_backend.utils.common import check_access_to_app - -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( # noqa pylint: disable-all - get_user_and_org_id, - ) -else: - from agenta_backend.services.selectors import get_user_and_org_id +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import Permission + from agenta_backend.commons.utils.permissions import check_action_access router = APIRouter() logger = logging.getLogger(__name__) @@ -41,15 +34,18 @@ async def get_evaluators_endpoint(): List[Evaluator]: A list of evaluator objects. """ - evaluators = evaluator_manager.get_evaluators() + try: + evaluators = evaluator_manager.get_evaluators() - if evaluators is None: - raise HTTPException(status_code=500, detail="Error processing evaluators file") + if evaluators is None: + raise HTTPException(status_code=500, detail="Error processing evaluators file") - if not evaluators: - raise HTTPException(status_code=404, detail="No evaluators found") + if not evaluators: + raise HTTPException(status_code=404, detail="No evaluators found") - return evaluators + return evaluators + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) @router.get("/configs/", response_model=List[EvaluatorConfig]) @@ -63,22 +59,47 @@ async def get_evaluator_configs(app_id: str): List[EvaluatorConfigDB]: A list of evaluator configuration objects. """ - evaluators_configs = await evaluator_manager.get_evaluators_configs(app_id) - return evaluators_configs + try: + evaluators_configs = await evaluator_manager.get_evaluators_configs(app_id) + return evaluators_configs + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Error fetching evaluator configurations: {str(e)}" + ) @router.get("/configs/{evaluator_config_id}/", response_model=EvaluatorConfig) -async def get_evaluator_config(evaluator_config_id: str): +async def get_evaluator_config(evaluator_config_id: str, request: Request): """Endpoint to fetch evaluator configurations for a specific app. Returns: List[EvaluatorConfigDB]: A list of evaluator configuration objects. """ - evaluators_configs = await evaluator_manager.get_evaluator_config( - evaluator_config_id - ) - return evaluators_configs + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=evaluator_config_id, + object_type="evaluator_config", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + evaluators_configs = await evaluator_manager.get_evaluator_config( + evaluator_config_id + ) + return evaluators_configs + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Error fetching evaluator configuration: {str(e)}" + ) @router.post("/configs/", response_model=EvaluatorConfig) diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index fca1f2b89b..964b2489fb 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -2,29 +2,35 @@ import os import csv import json +import logging import requests + from bson import ObjectId from datetime import datetime from typing import Optional, List from pydantic import ValidationError - from fastapi.responses import JSONResponse +from agenta_backend.services import db_manager +from agenta_backend.utils.common import APIRouter +from agenta_backend.models.db_models import TestSetDB +from agenta_backend.models.db_models import Permission +from agenta_backend.services.db_manager import get_user from fastapi import HTTPException, UploadFile, File, Form, Request +from agenta_backend.models.converters import testset_db_to_pydantic +from agenta_backend.utils.common import APIRouter, check_rbac_permission + from agenta_backend.models.api.testset_model import ( - TestSetSimpleResponse, - DeleteTestsets, NewTestset, + DeleteTestsets, + TestSetSimpleResponse, TestSetOutputResponse, ) -from agenta_backend.services import db_manager -from agenta_backend.models.db_models import TestSetDB -from agenta_backend.services.db_manager import get_user -from agenta_backend.models.converters import testset_db_to_pydantic -from agenta_backend.utils.common import APIRouter, check_access_to_app - router = APIRouter() +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + upload_folder = "./path/to/upload/folder" @@ -59,15 +65,21 @@ async def upload_file( """ user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_app = await check_access_to_app( - user_org_data=user_org_data, app_id=app_id, check_owner=False + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.CREATE_TESTSET, ) - if not access_app: - error_msg = f"You do not have access to this app: {app_id}" + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) + app = await db_manager.fetch_app_by_id(app_id=app_id) # Create a document document = { @@ -75,6 +87,7 @@ async def upload_file( "name": testset_name if testset_name else file.filename, "app": app, "organization": app.organization, + "workspace": app.workspace, "csvdata": [], } @@ -136,15 +149,21 @@ async def import_testset( dict: The result of the import process. """ user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_app = await check_access_to_app( - user_org_data=user_org_data, app_id=app_id, check_owner=False + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.CREATE_TESTSET, ) - if not access_app: - error_msg = f"You do not have access to this app: {app_id}" + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) + app = await db_manager.fetch_app_by_id(app_id=app_id) try: @@ -161,6 +180,7 @@ async def import_testset( "name": testset_name, "app": app, "organization": app.organization, + "workspace": app.workspace, "csvdata": [], } @@ -216,22 +236,29 @@ async def create_testset( """ user_org_data: dict = await get_user_and_org_id(request.state.user_id) - user = await get_user(user_uid=user_org_data["uid"]) - access_app = await check_access_to_app( - user_org_data=user_org_data, app_id=app_id, check_owner=False + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.CREATE_TESTSET, ) - if not access_app: - error_msg = f"You do not have access to this app: {app_id}" + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) + + user = await get_user(user_uid=user_org_data["uid"]) app = await db_manager.fetch_app_by_id(app_id=app_id) testset = { "created_at": datetime.now().isoformat(), "name": csvdata.name, "app": app, "organization": app.organization, + "workspace": app.workspace, "csvdata": csvdata.csvdata, "user": user, } @@ -267,25 +294,33 @@ async def update_testset( Returns: str: The id of the test set updated. """ + user_org_data: dict = await get_user_and_org_id(request.state.user_id) + workspace_org_data = await db_manager.get_object_workspace_org_id(testset_id, "testset") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.EDIT_TESTSET, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + testset_update = { "name": csvdata.name, "csvdata": csvdata.csvdata, "updated_at": datetime.now().isoformat(), } - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - + test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: raise HTTPException(status_code=404, detail="testset not found") - access_app = await check_access_to_app( - user_org_data=user_org_data, app_id=str(test_set.app.id), check_owner=False - ) - if not access_app: - error_msg = f"You do not have access to this app: {test_set.app.id}" - return JSONResponse( - {"detail": error_msg}, - status_code=400, - ) + try: await test_set.update({"$set": testset_update}) if isinstance(test_set.id, ObjectId): @@ -316,15 +351,21 @@ async def get_testsets( - `HTTPException` with status code 404 if no testsets are found. """ user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_app = await check_access_to_app( - user_org_data=user_org_data, app_id=app_id, check_owner=False + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.VIEW_TESTSET, ) - if not access_app: - error_msg = f"You do not have access to this app: {app_id}" + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) + app = await db_manager.fetch_app_by_id(app_id=app_id) if app is None: @@ -356,18 +397,25 @@ async def get_single_testset( The requested testset if found, else an HTTPException. """ user_org_data: dict = await get_user_and_org_id(request.state.user_id) - test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) - if test_set is None: - raise HTTPException(status_code=404, detail="testset not found") - access_app = await check_access_to_app( - user_org_data=user_org_data, app_id=str(test_set.app.id), check_owner=False + workspace_org_data = await db_manager.get_object_workspace_org_id(testset_id, "testset") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.VIEW_TESTSET, ) - if not access_app: - error_msg = "You do not have access to this test set" + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) + + test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) + if test_set is None: + raise HTTPException(status_code=404, detail="testset not found") + return testset_db_to_pydantic(test_set) @@ -386,24 +434,27 @@ async def delete_testsets( A list of the deleted testsets' IDs. """ user_org_data: dict = await get_user_and_org_id(request.state.user_id) + workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.DELETE_TESTSET, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) deleted_ids = [] - for testset_id in delete_testsets.testset_ids: test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: raise HTTPException(status_code=404, detail="testset not found") - access_app = await check_access_to_app( - user_org_data=user_org_data, - app_id=str(test_set.app.id), - check_owner=False, - ) - if not access_app: - error_msg = "You do not have access to this test set" - return JSONResponse( - {"detail": error_msg}, - status_code=400, - ) + await test_set.delete() deleted_ids.append(testset_id) diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 5cf9f8908f..7c956b77ee 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -1,17 +1,17 @@ import os import logging + +from typing import Any, Optional, Union from docker.errors import DockerException from fastapi.responses import JSONResponse -from typing import Any, Optional, Union from fastapi import HTTPException, Request, Body -from agenta_backend.utils.common import APIRouter +from agenta_backend.models.db_models import Permission +from agenta_backend.utils.common import APIRouter, check_rbac_permission + from agenta_backend.services import ( app_manager, db_manager, ) -from agenta_backend.utils.common import ( - check_access_to_variant, -) from agenta_backend.models import converters from agenta_backend.models.api.api_models import ( @@ -58,10 +58,25 @@ async def add_variant_from_base_and_config( try: logger.debug("Initiating process to add a variant based on a previous one.") logger.debug(f"Received payload: {payload}") + + # Check user has permission to add variant user_org_data: dict = await get_user_and_org_id(request.state.user_id) - base_db = await db_manager.fetch_base_and_check_access( - payload.base_id, user_org_data + workspace_org_data = await db_manager.get_object_workspace_org_id(payload.base_id, "base") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.CREATE_APPLICATION, ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + base_db = await db_manager.fetch_base_by_id(payload.base_id) # Find the previous variant in the database db_app_variant = await db_manager.add_variant_from_base_and_config( @@ -100,25 +115,24 @@ async def remove_variant( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - - # Check app access - access_app = await check_access_to_variant( - user_org_data, variant_id=variant_id, check_owner=True + workspace_org_data = await db_manager.get_object_workspace_org_id(variant_id, "app_variant") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.DELETE_APPLICATION_VARIANT, ) - - if not access_app: - error_msg = ( - f"You do not have permission to delete app variant: {variant_id}" - ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, - ) - else: - await app_manager.terminate_and_remove_app_variant( - app_variant_id=variant_id, **user_org_data + status_code=403, ) + + await app_manager.terminate_and_remove_app_variant( + app_variant_id=variant_id, **user_org_data + ) except DockerException as e: detail = f"Docker error while trying to remove the app variant: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -149,25 +163,26 @@ async def update_variant_parameters( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_variant = await check_access_to_variant( - user_org_data=user_org_data, variant_id=variant_id + workspace_org_data = await db_manager.get_object_workspace_org_id(variant_id, "app_variant") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.MODIFY_CONFIGURATIONS, ) - - if not access_variant: - error_msg = ( - f"You do not have permission to update app variant: {variant_id}" - ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, - ) - else: - await app_manager.update_variant_parameters( - app_variant_id=variant_id, - parameters=payload.parameters, - **user_org_data, + status_code=403, ) + + await app_manager.update_variant_parameters( + app_variant_id=variant_id, + parameters=payload.parameters, + **user_org_data, + ) except ValueError as e: detail = f"Error while trying to update the app variant: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -197,18 +212,21 @@ async def update_variant_image( """ try: user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_variant = await check_access_to_variant( - user_org_data=user_org_data, variant_id=variant_id + workspace_org_data = await db_manager.get_object_workspace_org_id(variant_id, "app_variant") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.CREATE_APPLICATION, ) - if not access_variant: - error_msg = ( - f"You do not have permission to update app variant: {variant_id}" - ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) + db_app_variant = await db_manager.fetch_app_variant_by_id( app_variant_id=variant_id ) @@ -264,16 +282,22 @@ async def start_variant( else: envvars = {} if env_vars is None else env_vars.env_vars - access = await check_access_to_variant( - user_org_data=user_org_data, variant_id=variant_id + # Check user has permission to start variant + workspace_org_data = await db_manager.get_object_workspace_org_id(variant_id, "app_variant") + has_permission = await check_rbac_permission( + user_org_data=user_org_data, + workspace_id=workspace_org_data["workspace_id"], + organization_id=workspace_org_data["organization_id"], + role=Permission.CREATE_APPLICATION, ) - if not access: - error_msg = f"You do not have access to this variant: {variant_id}" + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) + app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id=variant_id) if action.action == VariantActionEnum.START: url: URI = await app_manager.start_variant( diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 587050006a..4522ef72d2 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -21,18 +21,20 @@ evaluator_manager, ) -if os.environ["FEATURE_FLAG"] in ["cloud"]: +FEATURE_FLAG = s.environ["FEATURE_FLAG"] + +if FEATURE_FLAG in ["cloud"]: from agenta_backend.cloud.services import ( lambda_deployment_manager as deployment_manager, ) # noqa pylint: disable-all -elif os.environ["FEATURE_FLAG"] in ["ee"]: +elif FEATURE_FLAG in ["ee"]: from agenta_backend.ee.services import ( deployment_manager, ) # noqa pylint: disable-all else: from agenta_backend.services import deployment_manager -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: +if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.services import ( api_key_service, ) # noqa pylint: disable-all @@ -86,7 +88,7 @@ async def start_variant( env_vars.update( {"AGENTA_BASE_ID": str(db_app_variant.base.id), "AGENTA_HOST": domain_name} ) - if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: + if FEATURE_FLAG in ["cloud", "ee"]: api_key = await api_key_service.create_api_key( str(db_app_variant.user.uid), expiration_date=None, hidden=True ) @@ -129,7 +131,7 @@ async def update_variant_image( await deployment_manager.stop_and_delete_service(deployment) await db_manager.remove_deployment(deployment) - if os.environ["FEATURE_FLAG"] in ["ee", "oss"]: + if FEATURE_FLAG in ["ee", "oss"]: await deployment_manager.remove_image(app_variant_db.base.image) await db_manager.remove_image(app_variant_db.base.image) @@ -140,8 +142,8 @@ async def update_variant_image( docker_id=image.docker_id, user=app_variant_db.user, deletable=True, - organization=app_variant_db.organization, - workspace=app_variant_db.workspace, + organization=app_variant_db.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace=app_variant_db.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa ) # Update base with new image await db_manager.update_base(app_variant_db.base, image=db_image) @@ -157,7 +159,7 @@ async def update_variant_image( async def terminate_and_remove_app_variant( - app_variant_id: str = None, app_variant_db=None, **kwargs: dict + app_variant_id: str = None, app_variant_db=None ) -> None: """ Removes app variant from the database. If it's the last one using an image, performs additional operations: @@ -219,20 +221,20 @@ async def terminate_and_remove_app_variant( # If image deletable is True, remove docker image and image db if image.deletable: try: - if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: + if FEATURE_FLAG in ["cloud", "ee"]: await deployment_manager.remove_repository(image.tags) else: await deployment_manager.remove_image(image) except RuntimeError as e: logger.error(f"Failed to remove image {image} {e}") - await db_manager.remove_image(image, **kwargs) + await db_manager.remove_image(image) logger.debug("remove base") - await db_manager.remove_app_variant_from_db(app_variant_db, **kwargs) + await db_manager.remove_app_variant_from_db(app_variant_db) logger.debug("Remove image object from db") if deployment: await db_manager.remove_deployment(deployment) - await db_manager.remove_base_from_db(app_variant_db.base, **kwargs) + await db_manager.remove_base_from_db(app_variant_db.base) logger.debug("remove_app_variant_from_db") # Only delete the docker image for users that are running the oss version @@ -244,13 +246,13 @@ async def terminate_and_remove_app_variant( else: # remove variant + config logger.debug("remove_app_variant_from_db") - await db_manager.remove_app_variant_from_db(app_variant_db, **kwargs) + await db_manager.remove_app_variant_from_db(app_variant_db) logger.debug("list_app_variants") - app_variants = await db_manager.list_app_variants(app_id=app_id, **kwargs) + app_variants = await db_manager.list_app_variants(app_id) logger.debug(f"{app_variants}") if len(app_variants) == 0: # this was the last variant for an app logger.debug("remove_app_related_resources") - await remove_app_related_resources(app_id=app_id, **kwargs) + await remove_app_related_resources(app_id) except Exception as e: logger.error( f"An error occurred while deleting app variant {app_variant_db.app.app_name}/{app_variant_db.variant_name}: {str(e)}" @@ -258,7 +260,7 @@ async def terminate_and_remove_app_variant( raise e from None -async def remove_app_related_resources(app_id: str, **kwargs: dict): +async def remove_app_related_resources(app_id: str): """Removes environments and testsets associated with an app after its deletion. When an app or its last variant is deleted, this function ensures that @@ -269,17 +271,15 @@ async def remove_app_related_resources(app_id: str, **kwargs: dict): """ try: # Delete associated environments - environments: List[AppEnvironmentDB] = await db_manager.list_environments( - app_id, **kwargs - ) + environments: List[AppEnvironmentDB] = await db_manager.list_environments(app_id) for environment_db in environments: - await db_manager.remove_environment(environment_db, **kwargs) + await db_manager.remove_environment(environment_db) logger.info(f"Successfully deleted environment {environment_db.name}.") # Delete associated testsets - await db_manager.remove_app_testsets(app_id, **kwargs) + await db_manager.remove_app_testsets(app_id) logger.info(f"Successfully deleted test sets associated with app {app_id}.") - await db_manager.remove_app_by_id(app_id, **kwargs) + await db_manager.remove_app_by_id(app_id) logger.info(f"Successfully remove app object {app_id}.") except Exception as e: logger.error( @@ -288,7 +288,7 @@ async def remove_app_related_resources(app_id: str, **kwargs: dict): raise e from None -async def remove_app(app_id: str, **kwargs: dict): +async def remove_app(app_id: str): """Removes all app variants from db, if it is the last one using an image, then deletes the image from the db, shutdowns the container, deletes it and remove the image from the registry @@ -303,18 +303,16 @@ async def remove_app(app_id: str, **kwargs: dict): logger.error(error_msg) raise ValueError(error_msg) try: - app_variants = await db_manager.list_app_variants(app_id=app_id, **kwargs) + app_variants = await db_manager.list_app_variants(app_id) for app_variant_db in app_variants: - await terminate_and_remove_app_variant( - app_variant_db=app_variant_db, **kwargs - ) + await terminate_and_remove_app_variant(app_variant_db=app_variant_db) logger.info( f"Successfully deleted app variant {app_variant_db.app.app_name}/{app_variant_db.variant_name}." ) if len(app_variants) == 0: # Failsafe in case something went wrong before logger.debug("remove_app_related_resources") - await remove_app_related_resources(app_id=app_id, **kwargs) + await remove_app_related_resources(app_id) except Exception as e: logger.error( @@ -324,7 +322,7 @@ async def remove_app(app_id: str, **kwargs: dict): async def update_variant_parameters( - app_variant_id: str, parameters: Dict[str, Any], **kwargs: dict + app_variant_id: str, parameters: Dict[str, Any] ): """Updates the parameters for app variant in the database. @@ -339,9 +337,7 @@ async def update_variant_parameters( logger.error(error_msg) raise ValueError(error_msg) try: - await db_manager.update_variant_parameters( - app_variant_db=app_variant_db, parameters=parameters, **kwargs - ) + await db_manager.update_variant_parameters(app_variant_db=app_variant_db, parameters=parameters) except Exception as e: logger.error( f"Error updating app variant {app_variant_db.app.app_name}/{app_variant_db.variant_name}" @@ -353,11 +349,11 @@ async def add_variant_based_on_image( app: AppDB, variant_name: str, docker_id_or_template_uri: str, + user_uid: str, tags: str = None, base_name: str = None, config_name: str = "default", is_template_image: bool = False, - **user_org_data: dict, ) -> AppVariantDB: """ Adds a new variant to the app based on the specified Docker image. @@ -370,7 +366,7 @@ async def add_variant_based_on_image( base_name (str, optional): The name of the base to use for the new variant. Defaults to None. config_name (str, optional): The name of the configuration to use for the new variant. Defaults to "default". is_template_image (bool, optional): Whether or not the image used is for a template (in this case we won't delete it in the future). - **user_org_data (dict): Additional user and organization data. + user_uid (str): The UID of the user. Returns: AppVariantDB: The newly created app variant. @@ -388,9 +384,9 @@ async def add_variant_based_on_image( or variant_name in [None, ""] or docker_id_or_template_uri in [None, ""] ): - raise ValueError("App variant or image is None") + raise ValueError("App variant, variant name or docker_id/template_uri is None") - if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"]: + if FEATURE_FLAG not in ["cloud", "ee"]: if tags in [None, ""]: raise ValueError("OSS: Tags is None") @@ -401,7 +397,7 @@ async def add_variant_based_on_image( # Check if app variant already exists logger.debug("Step 2: Checking if app variant already exists") variants = await db_manager.list_app_variants_for_app_id( - app_id=str(app.id), **user_org_data + app_id=str(app.id) ) already_exists = any(av for av in variants if av.variant_name == variant_name) if already_exists: @@ -410,18 +406,18 @@ async def add_variant_based_on_image( # Retrieve user and image objects logger.debug("Step 3: Retrieving user and image objects") - user_instance = await db_manager.get_user(user_uid=user_org_data["uid"]) + user_instance = await db_manager.get_user(user_uid) if parsed_url.scheme and parsed_url.netloc: db_image = await db_manager.get_orga_image_instance_by_uri( - organization_id=str(app.organization.id), - workspace_id=str(app.workspace.id), template_uri=docker_id_or_template_uri, + organization_id=str(app.organization.id) if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace_id=str(app.workspace.id) if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa ) else: db_image = await db_manager.get_orga_image_instance_by_docker_id( - organization_id=str(app.organization.id), - workspace_id=str(app.workspace.id), docker_id=docker_id_or_template_uri, + organization_id=str(app.organization.id) if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace_id=str(app.workspace.id) if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa ) # Create new image if not exists @@ -433,8 +429,8 @@ async def add_variant_based_on_image( template_uri=docker_id_or_template_uri, deletable=not (is_template_image), user=user_instance, - organization=app.organization, - workspace=app.workspace, + organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa ) else: docker_id = docker_id_or_template_uri @@ -444,8 +440,8 @@ async def add_variant_based_on_image( tags=tags, deletable=not (is_template_image), user=user_instance, - organization=app.organization, - workspace=app.workspace, + organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa ) # Create config @@ -462,8 +458,8 @@ async def add_variant_based_on_image( ] # TODO: Change this in SDK2 to directly use base_name db_base = await db_manager.create_new_variant_base( app=app, - organization=app.organization, - workspace=app.workspace, + organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa user=user_instance, base_name=base_name, # the first variant always has default base image=db_image, @@ -476,8 +472,8 @@ async def add_variant_based_on_image( variant_name=variant_name, image=db_image, user=user_instance, - organization=app.organization, - workspace=app.workspace, + organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa parameters={}, base_name=base_name, config_name=config_name, diff --git a/agenta-backend/agenta_backend/services/container_manager.py b/agenta-backend/agenta_backend/services/container_manager.py index 5ee1418703..bee8047979 100644 --- a/agenta-backend/agenta_backend/services/container_manager.py +++ b/agenta-backend/agenta_backend/services/container_manager.py @@ -23,6 +23,7 @@ client = docker.from_env() +FEATURE_FLAG = os.environ["FEATURE_FLAG"] logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -30,7 +31,7 @@ async def build_image(app_db: AppDB, base_name: str, tar_file: UploadFile) -> Image: app_name = app_db.app_name - organization_id = str(app_db.organization.id) + user_id = app_db.user.id image_name = f"agentaai/{app_name.lower()}_{base_name.lower()}:latest" # Get event loop @@ -53,10 +54,10 @@ async def build_image(app_db: AppDB, base_name: str, tar_file: UploadFile) -> Im *( app_name, base_name, - organization_id, tar_path, image_name, temp_dir, + user_id, ), ) image_result = await asyncio.wrap_future(future) @@ -66,10 +67,10 @@ async def build_image(app_db: AppDB, base_name: str, tar_file: UploadFile) -> Im def build_image_job( app_name: str, base_name: str, - organization_id: str, tar_path: Path, image_name: str, temp_dir: Path, + user_id: str, ) -> Image: """Business logic for building a docker image from a tar file @@ -80,13 +81,13 @@ def build_image_job( base_name -- The `base_name` parameter is a string that represents the variant of the \ application. It could be a specific version, configuration, or any other distinguishing \ factor for the application - organization_id -- The id of the organization the app belongs to tar_path -- The `tar_path` parameter is the path to the tar file that contains the source code \ or files needed to build the Docker image image_name -- The `image_name` parameter is a string that represents the name of the Docker \ image that will be built. It is used as the tag for the image temp_dir -- The `temp_dir` parameter is a `Path` object that represents the temporary directory where the contents of the tar file will be extracted + user_id -- The id of the user that owns the app Raises: HTTPException: _description_ @@ -107,19 +108,21 @@ def build_image_job( image, build_log = client.images.build( path=str(temp_dir), tag=image_name, - buildargs={"ROOT_PATH": f"/{organization_id}/{app_name}/{base_name}"}, + buildargs={"ROOT_PATH": f"/{user_id}/{app_name}/{base_name}"}, rm=True, dockerfile=dockerfile, pull=True, ) for line in build_log: logger.info(line) - return Image( + pydantic_image = Image( type="image", docker_id=image.id, tags=image.tags[0], - organization_id=organization_id, ) + + return pydantic_image + except docker.errors.BuildError as ex: log = "Error building Docker image:\n" log += str(ex) + "\n" diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index aeea4a4c7e..c97e2cd66e 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -7,14 +7,8 @@ from agenta_backend.models.api.api_models import ( App, - AppVariant, - ImageExtended, Template, ) -from agenta_backend.models.api.workspace_models import ( - Workspace, - CreateWorkspace, -) from agenta_backend.models.converters import ( app_db_to_pydantic, image_db_to_pydantic, @@ -22,38 +16,21 @@ ) from agenta_backend.services.json_importer_helper import get_json from agenta_backend.models.db_models import ( - HumanEvaluationDB, - HumanEvaluationScenarioDB, - Result, AggregatedResult, - AppDB, - AppVariantDB, EvaluationScenarioInputDB, EvaluationScenarioOutputDB, EvaluationScenarioResult, - EvaluatorConfigDB, - VariantBaseDB, ConfigDB, ConfigVersionDB, - AppEnvironmentDB, - EvaluationDB, - EvaluationScenarioDB, - ImageDB, OrganizationDB, DeploymentDB, TemplateDB, - TestSetDB, - UserDB, - Permission, WorkspaceDB, - WorkspaceRole, - WorkspaceMemberDB, - WorkspacePermissionDB, ) from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum from agenta_backend.utils.common import ( check_user_org_access, - check_user_workspace_access, + check_user_org_workspace_access, ) from fastapi import HTTPException @@ -62,7 +39,63 @@ from beanie.operators import In from beanie import PydanticObjectId as ObjectId +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.services import db_manager_ee + + from agenta_backend.commons.models.api.api_models import ( + AppVariant_ as AppVariant, + ImageExtended_ as ImageExtended, + ) + + from agenta_backend.commons.models.db_models import ( + AppDB_ as AppDB, + UserDB_ as UserDB, + ImageDB_ as ImageDB, + TestSetDB_ as TestSetDB, + AppVariantDB_ as AppVariantDB, + EvaluationDB_ as EvaluationDB, + VariantBaseDB_ as VariantBaseDB, + AppEnvironmentDB_ as AppEnvironmentDB, + EvaluatorConfigDB_ as EvaluatorConfigDB, + HumanEvaluationDB_ as HumanEvaluationDB, + EvaluationScenarioDB_ as EvaluationScenarioDB, + HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, + ) +else: + from agenta_backend.models.api.api_models import ( + AppVariant, + ImageExtended, + ) + from agenta_backend.models.db_models import ( + AppDB, + UserDB, + ImageDB, + TestSetDB, + AppVariantDB, + EvaluationDB, + VariantBaseDB, + AppEnvironmentDB, + EvaluatorConfigDB, + HumanEvaluationDB, + EvaluationScenarioDB, + HumanEvaluationScenarioDB, + ) + +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.services import db_manager_ee + from agenta_backend.commons.services.selectors import ( + get_user_and_org_id, + ) # noqa pylint: disable-all + from agenta_backend.commons.utils.permissions import ( + check_action_access, + check_rbac_permission + ) + from agenta_backend.commons.models.db_models import ( + Permission, + WorkspaceRole + ) # Define logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -73,11 +106,11 @@ async def add_testset_to_app_variant( app_id: str, - org_id: str, - workspace_id: str, template_name: str, app_name: str, - **kwargs: dict, + user_uid: str, + org_id: str = None, + workspace_id: str = None, ): """Add testset to app variant. Args: @@ -90,9 +123,7 @@ async def add_testset_to_app_variant( try: app_db = await get_app_instance_by_id(app_id) - org_db = await get_organization_object(org_id) - workspace_db = await get_workspace(workspace_id) - user_db = await get_user(user_uid=kwargs["uid"]) + user_db = await get_user(user_uid) json_path = os.path.join( PARENT_DIRECTORY, @@ -113,9 +144,20 @@ async def add_testset_to_app_variant( **testset, app=app_db, user=user_db, - organization=org_db, - workspace=workspace_db, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace_id is also provided, and vice versa + assert ( + org_id is not None and workspace_id is not None + ), "organization and workspace must be provided together" + + organization_db = await db_manager_ee.get_organization(org_id) + workspace_db = await db_manager_ee.get_workspace(workspace_id) + + testset_db.organization = organization_db + testset_db.workspace = workspace_db + await testset_db.create() except Exception as e: @@ -136,10 +178,14 @@ async def get_image(app_variant: AppVariant, **kwargs: dict) -> ImageExtended: query_expression = ( AppVariantDB.app.id == app_variant.app_id, AppVariantDB.variant_name == app_variant.variant_name, - AppVariantDB.organization.id == app_variant.organization, - AppVariantDB.workspace.id == app_variant.workspace, ) - + + if FEATURE_FLAG in ["cloud", "ee"]: + query_expression += ( + AppVariantDB.organization.id == app_variant.organization, + AppVariantDB.workspace.id == app_variant.workspace, + ) + db_app_variant = await AppVariantDB.find_one(query_expression) if db_app_variant: image_db = await ImageDB.find_one(ImageDB.id == db_app_variant.image.id) @@ -162,7 +208,7 @@ async def get_image_by_id(image_id: str) -> ImageDB: return image -async def fetch_app_by_id(app_id: str, **kwargs: dict) -> AppDB: +async def fetch_app_by_id(app_id: str) -> AppDB: """Fetches an app by its ID. Args: @@ -173,33 +219,6 @@ async def fetch_app_by_id(app_id: str, **kwargs: dict) -> AppDB: return app -async def fetch_app_by_name( - app_name: str, - organization_id: Optional[str] = None, - workspace_id: Optional[str] = None, - **user_org_data: dict, -) -> Optional[AppDB]: - """Fetches an app by its name. - - Args: - app_name (str): The name of the app to fetch. - - Returns: - AppDB: the instance of the app - """ - - if not organization_id and not workspace_id: - user = await get_user(user_uid=user_org_data["uid"]) - app = await AppDB.find_one(AppDB.app_name == app_name, AppDB.user.id == user.id) - else: - app = await AppDB.find_one( - AppDB.app_name == app_name, - AppDB.organization.id == ObjectId(organization_id), - AppDB.workspace.id == ObjectId(workspace_id), - ) - return app - - async def fetch_app_variant_by_id( app_variant_id: str, ) -> Optional[AppVariantDB]: @@ -261,11 +280,11 @@ async def fetch_app_variant_by_name_and_appid( async def create_new_variant_base( app: AppDB, - organization: OrganizationDB, - workspace: WorkspaceDB, user: UserDB, base_name: str, image: ImageDB, + organization: OrganizationDB = None, + workspace: WorkspaceDB = None, ) -> VariantBaseDB: """Create a new base. Args: @@ -281,12 +300,20 @@ async def create_new_variant_base( logger.debug(f"Creating new base: {base_name} with image: {image} for app: {app}") base = VariantBaseDB( app=app, - organization=organization, - workspace=workspace, user=user, base_name=base_name, image=image, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace_id is also provided, and vice versa + assert ( + organization is not None and workspace is not None + ), "organization and workspace must be provided together" + + base.organization = organization + base.workspace = workspace + await base.create() return base @@ -318,8 +345,6 @@ async def create_new_config( async def create_new_app_variant( app: AppDB, - organization: OrganizationDB, - workspace: WorkspaceDB, user: UserDB, variant_name: str, image: ImageDB, @@ -328,6 +353,8 @@ async def create_new_app_variant( base_name: str, config_name: str, parameters: Dict, + organization: OrganizationDB = None, + workspace: WorkspaceDB = None, ) -> AppVariantDB: """Create a new variant. Args: @@ -340,8 +367,6 @@ async def create_new_app_variant( """ variant = AppVariantDB( app=app, - organization=organization, - workspace=workspace, user=user, variant_name=variant_name, image=image, @@ -351,6 +376,16 @@ async def create_new_app_variant( config_name=config_name, parameters=parameters, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace_id is also provided, and vice versa + assert ( + organization is not None and workspace is not None + ), "organization and workspace must be provided together" + + variant.organization = organization + variant.workspace = workspace + await variant.create() return variant @@ -359,8 +394,8 @@ async def create_image( image_type: str, user: UserDB, deletable: bool, - organization: OrganizationDB, - workspace: WorkspaceDB, + organization: OrganizationDB = None, + workspace: WorkspaceDB = None, template_uri: str = None, docker_id: str = None, tags: str = None, @@ -392,38 +427,40 @@ async def create_image( elif image_type == "zip" and template_uri is None: raise Exception("template_uri must be provided for type zip") + image = ImageDB( + deletable=deletable, + user=user, + ) + if image_type == "zip": - image = ImageDB( - type="zip", - template_uri=template_uri, - deletable=deletable, - user=user, - organization=organization, - workspace=workspace, - ) + image.type = "zip" + image.template_uri = template_uri elif image_type == "image": - image = ImageDB( - type="image", - docker_id=docker_id, - tags=tags, - deletable=deletable, - user=user, - organization=organization, - workspace=workspace, - ) + image.type = "image" + image.docker_id = docker_id + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace_id is also provided, and vice versa + assert ( + organization is not None and workspace is not None + ), "organization and workspace must be provided together" + + image.organization = organization + image.workspace = workspace + await image.create() return image async def create_deployment( app: AppVariantDB, - organization: OrganizationDB, - workspace: WorkspaceDB, user: UserDB, container_name: str, container_id: str, uri: str, status: str, + organization: OrganizationDB = None, + workspace: WorkspaceDB = None, ) -> DeploymentDB: """Create a new deployment. Args: @@ -440,29 +477,35 @@ async def create_deployment( """ deployment = DeploymentDB( app=app, - organization=organization, - workspace=workspace, user=user, container_name=container_name, container_id=container_id, uri=uri, status=status, ) + + if FEATURE_FLAG in ["lcoud", "ee"]: + deployment.organization = organization + deployment.workspace = workspace + await deployment.create() return deployment async def create_app_and_envs( - app_name: str, organization_id: str, workspace_id: str, **user_org_data + app_name: str, + user_uid: str, + organization_id: str = None, + workspace_id: str = None, ) -> AppDB: """ Create a new app with the given name and organization ID. Args: app_name (str): The name of the app to create. + user_uid (str): The UID of the user that the app belongs to. organization_id (str): The ID of the organization that the app belongs to. workspace_id (str): The ID of the workspace that the app belongs to. - **user_org_data: Additional keyword arguments. Returns: AppDB: The created app. @@ -471,23 +514,32 @@ async def create_app_and_envs( ValueError: If an app with the same name already exists. """ - user_instance = await get_user(user_uid=user_org_data["uid"]) - app = await fetch_app_by_name( - app_name, organization_id, workspace_id, **user_org_data + user_instance = await get_user(user_uid) + app = await fetch_app_by_name_and_parameters( + app_name, + user_uid, + organization_id, + workspace_id, ) if app is not None: raise ValueError("App with the same name already exists") - organization_db = await get_organization_object(organization_id) - workspace_db = await get_workspace(workspace_id) - app = AppDB( - app_name=app_name, - organization=organization_db, - workspace=workspace_db, - user=user_instance, - ) + app = AppDB(app_name=app_name, user=user_instance) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization_id is provided, workspace_id is also provided, and vice versa + assert ( + organization_id is not None and workspace_id is not None + ), "org_id and workspace_id must be provided together" + + organization_db = await db_manager_ee.get_organization(organization_id) + workspace_db = await db_manager_ee.get_workspace(workspace_id) + + app.organization = organization_db + app.workspace = workspace_db + await app.create() - await initialize_environments(app, **user_org_data) + await initialize_environments(app) return app @@ -510,22 +562,6 @@ async def get_deployment_by_objectid( return deployment -async def get_organization_object(organization_id: str) -> OrganizationDB: - """ - Fetches an organization by its ID. - - Args: - organization_id (str): The ID of the organization to fetch. - - Returns: - OrganizationDB: The fetched organization. - """ - organization = await OrganizationDB.find_one( - OrganizationDB.id == ObjectId(organization_id) - ) - return organization - - async def get_organizations_by_list_ids(organization_ids: List) -> List: """ Retrieve organizations from the database by their IDs. @@ -544,7 +580,7 @@ async def get_organizations_by_list_ids(organization_ids: List) -> List: async def list_app_variants_for_app_id( - app_id: str, **kwargs: dict + app_id: str, ) -> List[AppVariantDB]: """ Lists all the app variants from the db @@ -561,7 +597,7 @@ async def list_app_variants_for_app_id( async def list_bases_for_app_id( - app_id: str, base_name: Optional[str] = None, **kwargs: dict + app_id: str, base_name: Optional[str] = None ) -> List[VariantBaseDB]: assert app_id is not None, "app_id cannot be None" query_expressions = VariantBaseDB.app.id == ObjectId(app_id) @@ -572,7 +608,7 @@ async def list_bases_for_app_id( async def list_variants_for_base( - base: VariantBaseDB, **kwargs: dict + base: VariantBaseDB ) -> List[AppVariantDB]: """ Lists all the app variants from the db for a base @@ -591,62 +627,6 @@ async def list_variants_for_base( return app_variants_db -async def get_workspace(workspace_id: str): - """ - Retrieve a workspace. - - Args: - workspace_id (str): The workspace id. - - Returns: - Workspace: The retrieved workspace. - """ - logger.debug(f"workspace_id: {workspace_id}") - workspace = await WorkspaceDB.find_one( - WorkspaceDB.id == ObjectId(workspace_id), fetch_links=True - ) - return workspace - - -async def create_workspace( - payload: CreateWorkspace, organization: OrganizationDB, user: UserDB -) -> WorkspaceDB: - """Create a new workspace. - - Args: - payload (Workspace): The workspace payload. - organization (OrganizationDB): The organization that the workspace belongs to. - user (UserDB): The user that the workspace belongs to. - - Returns: - Workspace: The created workspace. - """ - - # create default workspace - workspace = WorkspaceDB( - name=payload.name, - type=payload.type if payload.type else "", - description=payload.description if payload.description else "", - organization=organization, - ) - - # Assign the creator as the owner with all permissions - workspace.members = [ - WorkspaceMemberDB( - user_id=user.id, - roles=[ - WorkspacePermissionDB( - role_name=WorkspaceRole.OWNER, permissions=list(Permission) - ) - ], - ) - ] - await workspace.create() - logger.info(f"Created workspace {workspace} for organization {organization.id}") - - return workspace - - async def get_user(user_uid: str) -> UserDB: """Get the user object from the database. @@ -668,22 +648,8 @@ async def get_user(user_uid: str) -> UserDB: org_db = OrganizationDB(type="default", owner=str(user.id)) org = await org_db.create() - # create default workspace for user - workspace_payload = CreateWorkspace( - name=org_db.name, - type=org_db.type, - description="My Default Workspace", - organization_id=str(org_db.id), - ) - workspace = await create_workspace(workspace_payload, org, user) - - # update organization with default workspace id - org_db.workspaces = [workspace.id] - await org_db.update({"$set": org_db.dict(exclude_unset=True)}) - - # update user with organization and workspace + # update user with organization user_db.organizations.append(org.id) - user_db.workspaces.append(workspace.id) await user_db.update({"$set": user_db.dict(exclude_unset=True)}) return user @@ -756,7 +722,7 @@ async def get_users_by_ids(user_ids: List) -> List: async def get_orga_image_instance_by_docker_id( - organization_id: str, workspace_id: str, docker_id: str + docker_id: str, organization_id: str = None, workspace_id: str = None ) -> ImageDB: """Get the image object from the database with the provided id. @@ -767,17 +733,28 @@ async def get_orga_image_instance_by_docker_id( Returns: ImageDB: instance of image object """ - - image = await ImageDB.find_one( - ImageDB.docker_id == docker_id, - ImageDB.organization.id == ObjectId(organization_id), - ImageDB.workspace.id == ObjectId(workspace_id), + + query_expression = ( + ImageDB.docker_id == docker_id, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace_id is also provided, and vice versa + assert ( + organization_id is not None and workspace_id is not None + ), "organization and workspace must be provided together" + + query_expression += ( + ImageDB.organization.id == ObjectId(organization_id), + ImageDB.workspace.id == ObjectId(workspace_id), + ) + + image = await ImageDB.find_one(query_expression) return image async def get_orga_image_instance_by_uri( - organization_id: str, workspace_id: str, template_uri: str + template_uri: str, organization_id: str = None, workspace_id: str = None ) -> ImageDB: """Get the image object from the database with the provided id. @@ -792,12 +769,23 @@ async def get_orga_image_instance_by_uri( if not parsed_url.scheme and not parsed_url.netloc: raise ValueError(f"Invalid URL: {template_uri}") - - image = await ImageDB.find_one( - ImageDB.template_uri == template_uri, - ImageDB.organization.id == ObjectId(organization_id), - ImageDB.workspace.id == ObjectId(workspace_id), + + query_expression = ( + ImageDB.template_uri == template_uri, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace_id is also provided, and vice versa + assert ( + organization_id is not None and workspace_id is not None + ), "organization and workspace must be provided together" + + query_expression += ( + ImageDB.organization.id == ObjectId(organization_id), + ImageDB.workspace.id == ObjectId(workspace_id), + ) + + image = await ImageDB.find_one(query_expression) return image @@ -819,7 +807,7 @@ async def add_variant_from_base_and_config( base_db: VariantBaseDB, new_config_name: str, parameters: Dict[str, Any], - **user_org_data: dict, + user_uid: str, ): """ Add a new variant to the database based on an existing base and a new configuration. @@ -846,7 +834,7 @@ async def add_variant_from_base_and_config( ) if already_exists: raise ValueError("App variant with the same name already exists") - user_db = await get_user(user_uid=user_org_data["uid"]) + user_db = await get_user(user_uid) config_db = ConfigDB( config_name=new_config_name, parameters=parameters, @@ -878,10 +866,10 @@ async def add_variant_from_base_and_config( async def list_apps( + user_uid: str, app_name: str = None, org_id: str = None, workspace_id: str = None, - **user_org_data: dict, ) -> List[App]: """ Lists all the unique app names and their IDs from the database @@ -893,49 +881,55 @@ async def list_apps( List[App] """ - user = await get_user(user_uid=user_org_data["uid"]) + user = await get_user(user_uid) assert user is not None, "User is None" - # assert that if org_id is provided, workspace_id is also provided, and vice versa - # assert ( # TODO: Enable this check when workspace_id is provided - # org_id is not None and workspace_id is not None - # ), "org_id and workspace_id must be provided together" - if app_name is not None: - app_db = await fetch_app_by_name( - app_name, org_id, workspace_id, **user_org_data + app_db = await fetch_app_by_name_and_parameters( + app_name=app_name, user_uid=user_uid, organization_id=org_id, workspace_id=workspace_id ) return [app_db_to_pydantic(app_db)] - # elif org_id is not None and workspace_id is not None: # TODO: Enable this check when workspace_id is provided - # action_access = await check_user_workspace_access( - # user_org_data, workspace_id, org_id + + + # elif (org_id is not None) or (workspace_id is not None): # TODO: Remember to enable this when workspace_id is provided after the RBAC is implemented + # if not FEATURE_FLAG in ["cloud", "ee"]: + # return JSONResponse( + # {"error": "organization and/or workspace is only available in Cloud and EE"}, + # status_code=400, + # ) + + # # assert that if org_id is provided, workspace_id is also provided, and vice versa + # assert ( + # org_id is not None and workspace_id is not None + # ), "org_id and workspace_id must be provided together" + + # user_org_data = await get_user_and_org_id(user_uid) + # has_permission = await check_rbac_permission( + # user_org_data=user_org_data, + # workspace_id=ObjectId(workspace_id), + # organization_id=ObjectId(org_id), + # permission=Permission.CREATE_APPLICATION, # ) - # if action_access: - # apps: List[AppDB] = await AppDB.find( - # AppDB.organization.id == ObjectId(org_id), - # AppDB.workspace.id == ObjectId(workspace_id), - # ).to_list() - # return [app_db_to_pydantic(app) for app in apps] - elif org_id is not None: - organization_access = await check_user_org_access(user_org_data, org_id) - if organization_access: - apps: List[AppDB] = await AppDB.find( - AppDB.organization.id == ObjectId(org_id) - ).to_list() - return [app_db_to_pydantic(app) for app in apps] - - else: - return JSONResponse( - {"error": "You do not have permission to access this organization"}, - status_code=403, - ) + # logger.debug(f"User has Permission to list apps: {has_permission}") + # if not has_permission: + # error_msg = f"You do not have access to perform this action. Please contact your organization admin." + # return JSONResponse( + # {"detail": error_msg}, + # status_code=400, + # ) + + # apps: List[AppDB] = await AppDB.find( + # AppDB.organization.id == ObjectId(org_id), + # AppDB.workspace.id == ObjectId(workspace_id), + # ).to_list() + # return [app_db_to_pydantic(app) for app in apps] else: apps = await AppDB.find(AppDB.user.id == user.id).to_list() return [app_db_to_pydantic(app) for app in apps] -async def list_app_variants(app_id: str = None, **kwargs: dict) -> List[AppVariantDB]: +async def list_app_variants(app_id: str) -> List[AppVariantDB]: """ Lists all the app variants from the db Args: @@ -983,7 +977,7 @@ async def remove_deployment(deployment_db: DeploymentDB, **kwargs: dict): await deployment_db.delete() -async def remove_app_variant_from_db(app_variant_db: AppVariantDB, **kwargs: dict): +async def remove_app_variant_from_db(app_variant_db: AppVariantDB): """Remove an app variant from the db the logic for removing the image is in app_manager.py @@ -995,10 +989,7 @@ async def remove_app_variant_from_db(app_variant_db: AppVariantDB, **kwargs: dic # Remove the variant from the associated environments logger.debug("list_environments_by_variant") - environments = await list_environments_by_variant( - app_variant_db, - **kwargs, - ) + environments = await list_environments_by_variant(app_variant_db) for environment in environments: environment.deployed_app_variant = None await environment.delete() @@ -1009,7 +1000,7 @@ async def remove_app_variant_from_db(app_variant_db: AppVariantDB, **kwargs: dic await app_variant_db.delete() -async def deploy_to_environment(environment_name: str, variant_id: str, **kwargs: dict): +async def deploy_to_environment(environment_name: str, variant_id: str): """ Deploys an app variant to a specified environment. @@ -1047,7 +1038,7 @@ async def deploy_to_environment(environment_name: str, variant_id: str, **kwargs await environment_db.save() -async def list_environments(app_id: str, **kwargs: dict) -> List[AppEnvironmentDB]: +async def list_environments(app_id: str) -> List[AppEnvironmentDB]: """ List all environments for a given app ID. @@ -1071,7 +1062,7 @@ async def list_environments(app_id: str, **kwargs: dict) -> List[AppEnvironmentD async def initialize_environments( - app_db: AppDB, **kwargs: dict + app_db: AppDB ) -> List[AppEnvironmentDB]: """ Initializes the environments for the app with the given database. @@ -1085,13 +1076,13 @@ async def initialize_environments( """ environments = [] for env_name in ["development", "staging", "production"]: - env = await create_environment(name=env_name, app_db=app_db, **kwargs) + env = await create_environment(name=env_name, app_db=app_db) environments.append(env) return environments async def create_environment( - name: str, app_db: AppDB, **kwargs: dict + name: str, app_db: AppDB ) -> AppEnvironmentDB: """ Creates a new environment in the database. @@ -1107,16 +1098,19 @@ async def create_environment( environment_db = AppEnvironmentDB( app=app_db, name=name, - user=app_db.user, - organization=app_db.organization, - workspace=app_db.workspace, + user=app_db.user ) + + if FEATURE_FLAG in ["cloud", "ee"]: + environment_db.organization = app_db.organization + environment_db.workspace = app_db.workspace + await environment_db.create() return environment_db async def list_environments_by_variant( - app_variant: AppVariantDB, **kwargs: dict + app_variant: AppVariantDB ) -> List[AppEnvironmentDB]: """ Returns a list of environments for a given app variant. @@ -1135,7 +1129,7 @@ async def list_environments_by_variant( return environments_db -async def remove_image(image: ImageDB, **kwargs: dict): +async def remove_image(image: ImageDB): """ Removes an image from the database. @@ -1154,7 +1148,7 @@ async def remove_image(image: ImageDB, **kwargs: dict): await image.delete() -async def remove_environment(environment_db: AppEnvironmentDB, **kwargs: dict): +async def remove_environment(environment_db: AppEnvironmentDB): """ Removes an environment from the database. @@ -1172,7 +1166,7 @@ async def remove_environment(environment_db: AppEnvironmentDB, **kwargs: dict): await environment_db.delete() -async def remove_app_testsets(app_id: str, **kwargs): +async def remove_app_testsets(app_id: str): """Returns a list of testsets owned by an app. Args: @@ -1203,7 +1197,7 @@ async def remove_app_testsets(app_id: str, **kwargs): return 0 -async def remove_base_from_db(base: VariantBaseDB, **kwargs): +async def remove_base_from_db(base: VariantBaseDB): """ Remove a base from the database. @@ -1222,7 +1216,7 @@ async def remove_base_from_db(base: VariantBaseDB, **kwargs): await base.delete() -async def remove_app_by_id(app_id: str, **kwargs): +async def remove_app_by_id(app_id: str): """ Removes an app instance from the database by its ID. @@ -1242,7 +1236,7 @@ async def remove_app_by_id(app_id: str, **kwargs): async def update_variant_parameters( - app_variant_db: AppVariantDB, parameters: Dict[str, Any], **kwargs: dict + app_variant_db: AppVariantDB, parameters: Dict[str, Any] ) -> None: """ Update the parameters of an app variant in the database. @@ -1549,21 +1543,6 @@ async def get_templates() -> List[Template]: return templates_db_to_pydantic(templates) -async def count_apps(**user_org_data: dict) -> int: - """ - Counts all the unique app names from the database - """ - - # Get user object - user = await get_user(user_uid=user_org_data["uid"]) - if user is None: - return 0 - - query_expressions = AppVariantDB.user.id == user.id - no_of_apps = await AppVariantDB.find(query_expressions).count() - return no_of_apps - - async def update_base( base: VariantBaseDB, **kwargs: dict, @@ -1599,112 +1578,11 @@ async def update_app_variant( return app_variant -async def fetch_base_and_check_access( - base_id: str, user_org_data: dict, check_owner=False -): - """ - Fetches a base from the database and checks if the user has access to it. - - Args: - base_id (str): The ID of the base to fetch. - user_org_data (dict): The user's organization data. - check_owner (bool, optional): Whether to check if the user is the owner of the base. Defaults to False. - - Raises: - Exception: If no base_id is provided. - HTTPException: If the base is not found or the user does not have access to it. - - Returns: - VariantBaseDB: The fetched base. - """ - if base_id is None: - raise Exception("No base_id provided") - base = await VariantBaseDB.find_one( - VariantBaseDB.id == ObjectId(base_id), fetch_links=True - ) - if base is None: - logger.error("Base not found") - raise HTTPException(status_code=404, detail="Base not found") - organization_id = base.organization.id - access = await check_user_org_access( - user_org_data, str(organization_id), check_owner - ) - if not access: - error_msg = f"You do not have access to this base: {base_id}" - raise HTTPException(status_code=403, detail=error_msg) - return base - - -async def fetch_app_and_check_access( - app_id: str, user_org_data: dict, check_owner=False -): - """ - Fetches an app from the database and checks if the user has access to it. - - Args: - app_id (str): The ID of the app to fetch. - user_org_data (dict): The user's organization data. - check_owner (bool, optional): Whether to check if the user is the owner of the app. Defaults to False. - - Returns: - dict: The fetched app. - - Raises: - HTTPException: If the app is not found or the user does not have access to it. - """ - app = await AppDB.find_one(AppDB.id == ObjectId(app_id), fetch_links=True) - if app is None: - logger.error("App not found") - raise HTTPException - - # Check user's access to the organization linked to the app. - organization_id = app.organization.id - access = await check_user_org_access( - user_org_data, str(organization_id), check_owner - ) - if not access: - error_msg = f"You do not have access to this app: {app_id}" - raise HTTPException(status_code=403, detail=error_msg) - return app - - -async def fetch_app_variant_and_check_access( - app_variant_id: str, user_org_data: dict, check_owner=False -): - """ - Fetches an app variant from the database and checks if the user has access to it. - - Args: - app_variant_id (str): The ID of the app variant to fetch. - user_org_data (dict): The user's organization data. - check_owner (bool, optional): Whether to check if the user is the owner of the app variant. Defaults to False. - - Returns: - AppVariantDB: The fetched app variant. - - Raises: - HTTPException: If the app variant is not found or the user does not have access to it. - """ - app_variant = await AppVariantDB.find_one( - AppVariantDB.id == ObjectId(app_variant_id), fetch_links=True - ) - if app_variant is None: - logger.error("App variant not found") - raise HTTPException - - # Check user's access to the organization linked to the app. - organization_id = app_variant.organization.id - access = await check_user_org_access( - user_org_data, str(organization_id), check_owner - ) - if not access: - error_msg = f"You do not have access to this app variant: {app_variant_id}" - raise HTTPException(status_code=403, detail=error_msg) - return app_variant - - -async def fetch_app_by_name_and_organization_and_workspace( - app_name: str, organization_id: str, workspace_id: str, **user_org_data: dict +async def fetch_app_by_name_and_parameters( + app_name: str, + user_uid: str, + organization_id: str = None, + workspace_id: str = None, ): """Fetch an app by it's name, organization id and workspace id. @@ -1717,26 +1595,39 @@ async def fetch_app_by_name_and_organization_and_workspace( AppDB: the instance of the app """ - app_db = await AppDB.find_one( - { - "app_name": app_name, - "organization": ObjectId(organization_id), - "workspace": ObjectId(workspace_id), - }, - fetch_links=True, + query_expression = ( + AppDB.app_name == app_name, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace_id is also provided, and vice versa + assert ( + organization_id is not None and workspace_id is not None + ), "organization_id and workspace_id must be provided together" + + query_expression += ( + AppDB.organization.id == ObjectId(organization_id), + AppDB.workspace.id == ObjectId(workspace_id), + ) + else: + query_expression += ( + AppDB.user.id == ObjectId(user_uid), + ) + + app_db = await AppDB.find_one(query_expression, fetch_links=True) + return app_db async def create_new_evaluation( app: AppDB, - organization: OrganizationDB, - workspace: WorkspaceDB, user: UserDB, testset: TestSetDB, status: str, variant: AppVariantDB, evaluators_configs: List[str], + organization: OrganizationDB = None, + workspace: WorkspaceDB = None, ) -> EvaluationDB: """Create a new evaluation scenario. Returns: @@ -1744,8 +1635,6 @@ async def create_new_evaluation( """ evaluation = EvaluationDB( app=app, - organization=organization, - workspace=workspace, user=user, testset=testset, status=status, @@ -1755,14 +1644,22 @@ async def create_new_evaluation( created_at=datetime.now().isoformat(), updated_at=datetime.now().isoformat(), ) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace is also provided, and vice versa + assert ( + organization is not None and workspace is not None + ), "organization and workspace must be provided together" + + evaluation.organization = organization + evaluation.workspace = workspace + await evaluation.create() return evaluation async def create_new_evaluation_scenario( user: UserDB, - organization: OrganizationDB, - workspace: WorkspaceDB, evaluation: EvaluationDB, variant_id: str, inputs: List[EvaluationScenarioInputDB], @@ -1772,6 +1669,8 @@ async def create_new_evaluation_scenario( note: Optional[str], evaluators_configs: List[EvaluatorConfigDB], results: List[EvaluationScenarioResult], + organization: OrganizationDB = None, + workspace: WorkspaceDB = None, ) -> EvaluationScenarioDB: """Create a new evaluation scenario. Returns: @@ -1779,8 +1678,6 @@ async def create_new_evaluation_scenario( """ evaluation_scenario = EvaluationScenarioDB( user=user, - organization=organization, - workspace=workspace, evaluation=evaluation, variant_id=ObjectId(variant_id), inputs=inputs, @@ -1793,6 +1690,16 @@ async def create_new_evaluation_scenario( created_at=datetime.utcnow(), updated_at=datetime.utcnow(), ) + + if FEATURE_FLAG in ["cloud", "ee"]: + # assert that if organization is provided, workspace is also provided, and vice versa + assert ( + organization is not None and workspace is not None + ), "organization and workspace must be provided together" + + evaluation_scenario.organization = organization + evaluation_scenario.workspace = workspace + await evaluation_scenario.create() return evaluation_scenario @@ -1898,10 +1805,10 @@ async def fetch_evaluator_config_by_appId( async def create_evaluator_config( app: AppDB, user: UserDB, - organization: OrganizationDB, - workspace: WorkspaceDB, name: str, evaluator_key: str, + organization: OrganizationDB = None, + workspace: WorkspaceDB = None, settings_values: Optional[Dict[str, Any]] = None, ) -> EvaluatorConfigDB: """Create a new evaluator configuration in the database.""" @@ -1909,12 +1816,20 @@ async def create_evaluator_config( new_evaluator_config = EvaluatorConfigDB( app=app, user=user, - organization=organization, - workspace=workspace, name=name, evaluator_key=evaluator_key, settings_values=settings_values, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + + # assert that if organization is provided, workspace is also provided, and vice versa + assert ( + organization is not None and workspace is not None + ), "organization and workspace must be provided together" + + new_evaluator_config.organization = organization + new_evaluator_config.workspace = workspace try: await new_evaluator_config.create() diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index 489ccb7ab9..dee580a31f 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -11,6 +11,8 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) +FEATURE_FLAG = os.environ["FEATURE_FLAG"] + async def start_service( app_variant_db: AppVariantDB, env_vars: Dict[str, str] @@ -51,13 +53,13 @@ async def start_service( deployment = await db_manager.create_deployment( app=app_variant_db.app, - organization=app_variant_db.organization, - workspace=app_variant_db.workspace, user=app_variant_db.user, container_name=container_name, container_id=container_id, uri=uri, status="running", + organization=app_variant_db.organization if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace=app_variant_db.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, ) return deployment diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 963be715da..b78c04d8e4 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -1,8 +1,13 @@ +import os import logging + from datetime import datetime +from fastapi import HTTPException from typing import Dict, List, Any +from beanie import PydanticObjectId as ObjectId -from fastapi import HTTPException +from agenta_backend.models import converters +from agenta_backend.services import db_manager from agenta_backend.models.api.evaluation_model import ( Evaluation, @@ -17,25 +22,35 @@ EvaluationStatusEnum, NewHumanEvaluation, ) -from agenta_backend.models import converters -from agenta_backend.services import db_manager -from agenta_backend.services.db_manager import get_user -from agenta_backend.utils.common import check_access_to_app + +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.utils.permissions import check_access_to_app + from agenta_backend.commons.models.db_models import ( + AppDB_ as AppDB, + UserDB_ as UserDB, + AppVariantDB_ as AppVariantDB, + EvaluationDB_ as EvaluationDB, + HumanEvaluationDB_ as HumanEvaluationDB, + EvaluationScenarioDB_ as EvaluationScenarioDB, + HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, + ) +else: + from agenta_backend.models.db_models import ( + AppDB, + UserDB, + AppVariantDB, + EvaluationDB, + HumanEvaluationDB, + EvaluationScenarioDB, + HumanEvaluationScenarioDB, + ) + from agenta_backend.models.db_models import ( - AppVariantDB, - EvaluationDB, - EvaluationScenarioDB, - HumanEvaluationDB, - HumanEvaluationScenarioDB, HumanEvaluationScenarioInput, HumanEvaluationScenarioOutput, - UserDB, - AppDB, ) -from beanie import PydanticObjectId as ObjectId - - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -46,31 +61,6 @@ class UpdateEvaluationScenarioError(Exception): pass -async def _fetch_evaluation_and_check_access( - evaluation_id: str, **user_org_data: dict -) -> EvaluationDB: - # Fetch the evaluation by ID - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id=evaluation_id) - - # Check if the evaluation exists - if evaluation is None: - raise HTTPException( - status_code=404, - detail=f"Evaluation with id {evaluation_id} not found", - ) - - # Check for access rights - access = await check_access_to_app( - user_org_data=user_org_data, app_id=evaluation.app.id - ) - if not access: - raise HTTPException( - status_code=403, - detail=f"You do not have access to this app: {str(evaluation.app.id)}", - ) - return evaluation - - async def _fetch_human_evaluation_and_check_access( evaluation_id: str, **user_org_data: dict ) -> HumanEvaluationDB: @@ -214,9 +204,7 @@ async def create_evaluation_scenario( Raises: HTTPException: If evaluation not found or access denied. """ - evaluation = await _fetch_evaluation_and_check_access( - evaluation_id=evaluation_id, **user_org_data - ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) scenario_inputs = [ EvaluationScenarioInput( @@ -272,14 +260,13 @@ async def update_human_evaluation_service( async def fetch_evaluation_scenarios_for_evaluation( - evaluation_id: str, **user_org_data: dict + evaluation_id: str ) -> List[EvaluationScenario]: """ Fetch evaluation scenarios for a given evaluation ID. Args: evaluation_id (str): The ID of the evaluation. - user_org_data (dict): User and organization data. Raises: HTTPException: If the evaluation is not found or access is denied. @@ -287,10 +274,7 @@ async def fetch_evaluation_scenarios_for_evaluation( Returns: List[EvaluationScenario]: A list of evaluation scenarios. """ - evaluation = await _fetch_evaluation_and_check_access( - evaluation_id=evaluation_id, - **user_org_data, - ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) scenarios = await EvaluationScenarioDB.find( EvaluationScenarioDB.evaluation.id == ObjectId(evaluation.id), fetch_links=True ).to_list() @@ -465,25 +449,17 @@ def _extend_with_correct_answer(evaluation_type: EvaluationType, row: dict): async def fetch_list_evaluations( app_id: str, - **user_org_data: dict, ) -> List[Evaluation]: """ Fetches a list of evaluations based on the provided filtering criteria. Args: app_id (Optional[str]): An optional app ID to filter the evaluations. - user_org_data (dict): User and organization data. Returns: List[Evaluation]: A list of evaluations. """ - access = await check_access_to_app(user_org_data=user_org_data, app_id=app_id) - if not access: - raise HTTPException( - status_code=403, - detail=f"You do not have access to this app: {app_id}", - ) - + evaluations_db = await EvaluationDB.find( EvaluationDB.app.id == ObjectId(app_id), fetch_links=True ).to_list() @@ -493,20 +469,17 @@ async def fetch_list_evaluations( ] -async def fetch_evaluation(evaluation_id: str, **user_org_data: dict) -> Evaluation: +async def fetch_evaluation(evaluation_id: str) -> Evaluation: """ Fetches a single evaluation based on its ID. Args: evaluation_id (str): The ID of the evaluation. - user_org_data (dict): User and organization data. Returns: Evaluation: The fetched evaluation. """ - evaluation = await _fetch_evaluation_and_check_access( - evaluation_id=evaluation_id, **user_org_data - ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) return await converters.evaluation_db_to_pydantic(evaluation) @@ -579,21 +552,18 @@ async def delete_human_evaluations( await evaluation.delete() -async def delete_evaluations(evaluation_ids: List[str], **user_org_data: dict) -> None: +async def delete_evaluations(evaluation_ids: List[str]) -> None: """ Delete evaluations by their IDs. Args: evaluation_ids (List[str]): A list of evaluation IDs. - user_org_data (dict): User and organization data. Raises: HTTPException: If evaluation not found or access denied. """ for evaluation_id in evaluation_ids: - evaluation = await _fetch_evaluation_and_check_access( - evaluation_id=evaluation_id, **user_org_data - ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) await evaluation.delete() @@ -610,7 +580,7 @@ async def create_new_human_evaluation( Returns: HumanEvaluationDB """ - user = await get_user(user_uid=user_org_data["uid"]) + user = await db_manager.get_user(user_uid=user_org_data["uid"]) # Initialize evaluation type settings evaluation_type_settings = {} @@ -684,19 +654,19 @@ async def create_new_evaluation( evaluation_db = await db_manager.create_new_evaluation( app=app, - organization=app.organization, - workspace=app.workspace, user=app.user, testset=testset, status=EvaluationStatusEnum.EVALUATION_STARTED, variant=variant_id, evaluators_configs=evaluator_config_ids, + organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, ) return await converters.evaluation_db_to_pydantic(evaluation_db) async def retrieve_evaluation_results( - evaluation_id: str, **user_org_data: dict + evaluation_id: str ) -> List[dict]: """Retrieve the aggregated results for a given evaluation. @@ -709,20 +679,11 @@ async def retrieve_evaluation_results( # Check for access rights evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) - access = await check_access_to_app( - user_org_data=user_org_data, app_id=str(evaluation.app.id) - ) - if not access: - raise HTTPException( - status_code=403, - detail=f"You do not have access to this app: {str(evaluation.app.id)}", - ) return await converters.aggregated_result_to_pydantic(evaluation.aggregated_results) async def compare_evaluations_scenarios( evaluations_ids: List[str], - **user_org_data: dict, ): evaluation = await db_manager.fetch_evaluation_by_id(evaluations_ids[0]) testset = evaluation.testset @@ -733,9 +694,7 @@ async def compare_evaluations_scenarios( all_scenarios = [] for evaluation_id in evaluations_ids: - eval_scenarios = await fetch_evaluation_scenarios_for_evaluation( - evaluation_id, **user_org_data - ) + eval_scenarios = await fetch_evaluation_scenarios_for_evaluation(evaluation_id) all_scenarios.append(eval_scenarios) grouped_scenarios_by_inputs = find_scenarios_by_input( diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index b15131f913..a602eb6337 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -6,8 +6,15 @@ from agenta_backend.services import db_manager +FEATURE_FLAG = os.environ["FEATURE_FLAG"] -from agenta_backend.models.db_models import AppDB, EvaluatorConfigDB +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.db_models import ( + AppDB_ as AppDB, + EvaluatorConfigDB_ as EvaluatorConfigDB + ) +else: + from agenta_backend.models.db_models import AppDB, EvaluatorConfigDB from agenta_backend.models.api.evaluation_model import Evaluator, EvaluatorConfig from agenta_backend.models.converters import evaluator_config_db_to_pydantic @@ -85,8 +92,8 @@ async def create_evaluator_config( app = await db_manager.fetch_app_by_id(app_id) evaluator_config = await db_manager.create_evaluator_config( app=app, - organization=app.organization, - workspace=app.workspace, + organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, + workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, user=app.user, name=name, evaluator_key=evaluator_key, @@ -152,8 +159,8 @@ async def create_ready_to_use_evaluators(app: AppDB): for evaluator in direct_use_evaluators: await db_manager.create_evaluator_config( app=app, - organization=app.organization, - workspace=app.workspace, + organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, + workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, user=app.user, name=evaluator["name"], evaluator_key=evaluator["key"], diff --git a/agenta-backend/agenta_backend/services/selectors.py b/agenta-backend/agenta_backend/services/selectors.py index 0d42e36642..143f53606f 100644 --- a/agenta-backend/agenta_backend/services/selectors.py +++ b/agenta-backend/agenta_backend/services/selectors.py @@ -7,68 +7,52 @@ ) -async def get_user_and_org_id(user_uid_id) -> Dict[str, List]: - """Retrieves the user ID and organization ID based on the logged-in session. +# async def get_user_and_org_id(user_uid_id) -> Dict[str, List]: +# """Retrieves the user ID and organization ID based on the logged-in session. - Arguments: - session (SessionContainer): Used to store and manage the user's session data +# Arguments: +# session (SessionContainer): Used to store and manage the user's session data - Returns: - A dictionary containing the user_id and a list of the user's organization_ids. - """ - user, org_ids = await get_user_objectid(user_uid_id) - return {"uid": str(user.uid), "id": str(user.id), "organization_ids": org_ids} +# Returns: +# A dictionary containing the user_id and a list of the user's organization_ids. +# """ +# user, org_ids = await get_user_objectid(user_uid_id) +# return {"uid": str(user.uid), "id": str(user.id), "organization_ids": org_ids} -async def get_user_objectid(user_uid: str) -> Tuple[str, List]: - """Retrieves the user object ID and organization IDs from the database - based on the user ID. +# async def get_user_objectid(user_uid: str) -> Tuple[str, List]: +# """Retrieves the user object ID and organization IDs from the database +# based on the user ID. - Arguments: - user_id (str): The unique identifier of a user +# Arguments: +# user_id (str): The unique identifier of a user - Returns: - a tuple containing the string representation of the user's ObjectId and the List - of the user's organization_ids. - """ +# Returns: +# a tuple containing the string representation of the user's ObjectId and the List +# of the user's organization_ids. +# """ - user = await UserDB.find_one(UserDB.uid == user_uid) - if user is not None: - organization_ids: List = ( - [org for org in user.organizations] if user.organizations else [] - ) - return user, organization_ids - return None, [] +# user = await UserDB.find_one(UserDB.uid == user_uid) +# if user is not None: +# organization_ids: List = ( +# [org for org in user.organizations] if user.organizations else [] +# ) +# return user, organization_ids +# return None, [] -async def get_user_own_org(user_uid: str) -> OrganizationDB: - """Get's the default users' organization from the database. +# async def get_user_own_org(user_uid: str) -> OrganizationDB: +# """Get's the default users' organization from the database. - Arguments: - user_uid (str): The uid of the user +# Arguments: +# user_uid (str): The uid of the user - Returns: - Organization: Instance of OrganizationDB - """ +# Returns: +# Organization: Instance of OrganizationDB +# """ - user = await UserDB.find_one(UserDB.uid == user_uid) - org: OrganizationDB = await OrganizationDB.find_one( - OrganizationDB.owner == str(user.id), OrganizationDB.type == "default" - ) - return org - - -async def get_org_default_workspace(organization: OrganizationDB) -> WorkspaceDB: - """Get's the default workspace for an organization from the database. - - Arguments: - organization (OrganizationDB): The organization - - Returns: - WorkspaceDB: Instance of WorkspaceDB - """ - - workspace: WorkspaceDB = await WorkspaceDB.find_one( - WorkspaceDB.organization.id == organization.id, WorkspaceDB.type == "default" - ) - return workspace +# user = await UserDB.find_one(UserDB.uid == user_uid) +# org: OrganizationDB = await OrganizationDB.find_one( +# OrganizationDB.owner == str(user.id), OrganizationDB.type == "default" +# ) +# return org diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index abef76ea22..011ba02457 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -1,6 +1,6 @@ +import os import asyncio import logging -import os import traceback from collections import defaultdict from typing import Any, Dict, List @@ -30,6 +30,8 @@ ) from celery import shared_task, states +FEATURE_FLAG = os.environ["FEATURE_FLAG"] + # Set logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -160,7 +162,6 @@ def evaluate( loop.run_until_complete( create_new_evaluation_scenario( user=app.user, - organization=app.organization, evaluation=new_evaluation_db, variant_id=variant_id, evaluators_configs=new_evaluation_db.evaluators_configs, @@ -172,6 +173,8 @@ def evaluate( EvaluationScenarioOutputDB(type="text", value=app_output.output) ], results=evaluators_results, + organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, ) ) diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index 39027e3334..bc021239c0 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -169,130 +169,3 @@ async def check_access_to_base( return False organization_id = base.organization.id return await check_user_org_access(user_org_data, str(organization_id), check_owner) - - -async def check_user_workspace_access( - user_org_data: Dict[str, Union[str, list]], - workspace_id: str, - org_id: str = None, -) -> bool: - """ - Check if a user has access to a specific workspace. - - Args: - user_org_data (Dict[str, Union[str, list]]): User-specific information. - workspace_id (str): The ID of the workspace. - - Returns: - bool: True if the user has access, False otherwise. - - Raises: - Exception: If neither or both `app` and `app_id` are provided. - - """ - workspace = await WorkspaceDB.find_one( - WorkspaceDB.id == ObjectId(workspace_id), fetch_links=True - ) - if workspace is None: - raise Exception("Workspace not found") - - if org_id is not None: - organization_id = ObjectId(org_id) - - # validate organization exists - organization = await get_organization(str(organization_id)) - if organization is None: - raise Exception("Organization not found") - - # check that workspace belongs to the organization - if workspace.organization.id != organization_id: - raise Exception("Workspace does not belong to the provided organization") - else: - organization_id = workspace.organization.id - - # check that user belongs to the organization - if not await check_user_org_access(user_org_data, str(organization_id)): - logger.error("User does not belong to the organization") - return False - - # check that user belongs to the workspace - user_id = user_org_data["id"] - if ObjectId(user_id) not in workspace.get_all_members(): - logger.error("User does not belong to the workspace") - return False - - # check that workspace is in the user's workspaces - user = await UserDB.find_one(UserDB.id == ObjectId(user_id)) - if ObjectId(workspace_id) not in user.workspaces: - logger.error("Workspace not in user's workspaces") - return False - - return True - - -async def check_rbac_permission( - user_org_data: Dict[str, Union[str, list]], - workspace_id: ObjectId, - organization_id: ObjectId, - permission: Permission = None, - role: str = None, -) -> bool: - """ - Check if a user belongs to a workspace and has a certain permission. - - Args: - user_org_data (Dict[str, Union[str, list]]): User-specific information. - user_id (ObjectId): The user's ID. - organization_id (ObjectId): The ID of the organization to which the workspace belongs. - workspace_id (ObjectId): The ID of the workspace to check. - permission (Permission): The permission to check. - role (str): The role to check. - - Returns: - bool: True if the user belongs to the workspace and has the specified permission, False otherwise. - """ - if permission is None and role is None: - raise Exception("Either permission or role must be provided") - elif permission is not None and role is not None: - raise Exception("Only one of permission or role must be provided") - - # Retrieve the workspace object using the provided workspace_id - workspace = await WorkspaceDB.find_one( - WorkspaceDB.id == workspace_id, fetch_links=True - ) - if workspace is None: - raise Exception("Workspace not found") - - provided_organization = await get_organization(str(organization_id)) - if provided_organization is None: - raise Exception("Organization not found") - - # confirm that the workspace belongs to the provided organization - if workspace.organization.id != organization_id: - raise Exception("Workspace does not belong to the provided organization") - - # get workspace organization and check if user belongs to it - workspace_organization_id = workspace.organization.id - if not await check_user_org_access(user_org_data, str(workspace_organization_id)): - return False - - user_id = ObjectId(user_org_data["id"]) - # Check if the user belongs to the workspace - if user_id not in workspace.get_all_members(): - return False - - # Check if user is the owner of the workspace ( they have all permissions ) - if workspace.is_owner(user_id): - return True - - # Check if the user has the specified permission or role - if role is not None: - # validate role exists - if role not in list(WorkspaceRole): - raise Exception("Invalid role specified") - return workspace.has_role(user_id, role) - else: - # validate permission exists - if permission not in list(Permission): - raise Exception("Invalid permission specified") - return workspace.has_permission(user_id, permission) From 145ab6895aab50ffd4f17c11d4e74ca8c6afc123 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 19 Jan 2024 09:35:10 +0100 Subject: [PATCH 23/99] Feat: complete refactoring of moving cloud features and adding rbac --- agenta-backend/agenta_backend/main.py | 2 - .../routers/human_evaluation_router.py | 265 +++++++++++++----- .../routers/organization_router.py | 81 ------ .../agenta_backend/services/db_manager.py | 112 ++------ .../services/evaluation_service.py | 96 ++----- .../services/evaluator_manager.py | 5 +- .../services/llm_apps_service.py | 6 +- .../services/results_service.py | 20 +- .../agenta_backend/services/selectors.py | 58 ---- .../services/templates_manager.py | 12 +- agenta-backend/agenta_backend/utils/common.py | 124 +------- 11 files changed, 270 insertions(+), 511 deletions(-) delete mode 100644 agenta-backend/agenta_backend/routers/organization_router.py delete mode 100644 agenta-backend/agenta_backend/services/selectors.py diff --git a/agenta-backend/agenta_backend/main.py b/agenta-backend/agenta_backend/main.py index 594703181a..56549a7f74 100644 --- a/agenta-backend/agenta_backend/main.py +++ b/agenta-backend/agenta_backend/main.py @@ -12,7 +12,6 @@ human_evaluation_router, evaluators_router, observability_router, - organization_router, testset_router, user_profile, variants_router, @@ -91,7 +90,6 @@ async def lifespan(application: FastAPI, cache=True): app.include_router(container_router.router, prefix="/containers") app.include_router(environment_router.router, prefix="/environments") app.include_router(observability_router.router, prefix="/observability") -app.include_router(organization_router.router, prefix="/organizations") app.include_router(bases_router.router, prefix="/bases") app.include_router(configs_router.router, prefix="/configs") diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 9dbdb2240d..f931f10254 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -6,6 +6,11 @@ from fastapi.encoders import jsonable_encoder from fastapi import HTTPException, APIRouter, Body, Request, status, Response +from agenta_backend.models import converters +from agenta_backend.services import db_manager +from agenta_backend.services import results_service +from agenta_backend.services import evaluation_service + from agenta_backend.models.api.evaluation_model import ( DeleteEvaluation, EvaluationScenarioScoreUpdate, @@ -18,12 +23,6 @@ SimpleEvaluationOutput, ) -from agenta_backend.services import evaluation_service -from agenta_backend.utils.common import check_access_to_app -from agenta_backend.services import db_manager -from agenta_backend.models import converters -from agenta_backend.services import results_service - from agenta_backend.services.evaluation_service import ( UpdateEvaluationScenarioError, get_evaluation_scenario_score_service, @@ -32,13 +31,19 @@ update_human_evaluation_service, ) - -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.services.selectors import ( # noqa pylint: disable-all - get_user_and_org_id, + get_user_org_and_workspace_id, + ) + from agenta_backend.commons.utils.permissions import ( # noqa pylint: disable-all + check_action_access, + check_rbac_permission + ) + from agenta_backend.commons.models.db_models import ( # noqa pylint: disable-all + Permission, + WorkspaceRole ) -else: - from agenta_backend.services.selectors import get_user_and_org_id router = APIRouter() @@ -57,25 +62,27 @@ async def create_evaluation( _description_ """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - access_app = await check_access_to_app( - user_org_data=user_org_data, - app_id=payload.app_id, - check_owner=False, - ) - if not access_app: - error_msg = f"You do not have access to this app: {payload.app_id}" - return JSONResponse( - {"detail": error_msg}, - status_code=400, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=payload.app_id, + object_type="app", + permission=Permission.CREATE_EVALUATION ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: raise HTTPException(status_code=404, detail="App not found") new_human_evaluation_db = await evaluation_service.create_new_human_evaluation( - payload, **user_org_data + payload, request.state.user_id ) return converters.human_evaluation_db_to_simple_evaluation_output( new_human_evaluation_db @@ -100,10 +107,24 @@ async def fetch_list_human_evaluations( Returns: List[HumanEvaluation]: A list of evaluations. """ - user_org_data = await get_user_and_org_id(request.state.user_id) - return await evaluation_service.fetch_list_human_evaluations( - app_id=app_id, **user_org_data - ) + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=app_id, + object_type="app", + permissin=Permission.VIEW_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + + return await evaluation_service.fetch_list_human_evaluations(app_id) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e @router.get("/{evaluation_id}/", response_model=HumanEvaluation) @@ -119,10 +140,24 @@ async def fetch_human_evaluation( Returns: HumanEvaluation: The fetched evaluation. """ - user_org_data = await get_user_and_org_id(request.state.user_id) - return await evaluation_service.fetch_human_evaluation( - evaluation_id, **user_org_data - ) + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluation_id, + object_type="human_evaluation", + permissin=Permission.VIEW_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + + return await evaluation_service.fetch_human_evaluation(evaluation_id) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e @router.get( @@ -146,14 +181,28 @@ async def fetch_evaluation_scenarios( List[EvaluationScenario]: A list of evaluation scenarios. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - eval_scenarios = ( - await evaluation_service.fetch_human_evaluation_scenarios_for_evaluation( - evaluation_id, **user_org_data + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluation_id, + object_type="human_evaluation_scenario_by_evaluation_id", + permissin=Permission.VIEW_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + + eval_scenarios = ( + await evaluation_service.fetch_human_evaluation_scenarios_for_evaluation(evaluation_id) ) - ) - return eval_scenarios + return eval_scenarios + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e @router.put("/{evaluation_id}/", operation_id="update_human_evaluation") @@ -171,10 +220,22 @@ async def update_human_evaluation( None: A 204 No Content status code, indicating that the update was successful. """ try: - # Get user and organization id - user_org_data: dict = await get_user_and_org_id(request.state.user_id) + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluation_id, + object_type="human_evaluation", + permission=Permission.EDIT_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + await update_human_evaluation_service( - evaluation_id, update_data, **user_org_data + evaluation_id, update_data ) return Response(status_code=status.HTTP_204_NO_CONTENT) @@ -202,14 +263,26 @@ async def update_evaluation_scenario_router( Returns: None: 204 No Content status code upon successful update. - """ - user_org_data = await get_user_and_org_id(request.state.user_id) + """ try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluation_scenario_id, + object_type="human_evaluation_scenario", + permission=Permission.EDIT_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + await update_human_evaluation_scenario( evaluation_scenario_id, evaluation_scenario, evaluation_type, - **user_org_data, ) return Response(status_code=status.HTTP_204_NO_CONTENT) except UpdateEvaluationScenarioError as e: @@ -231,10 +304,24 @@ async def get_evaluation_scenario_score_router( Returns: Dictionary containing the scenario ID and its score. """ - user_org_data = await get_user_and_org_id(request.state.user_id) - return await get_evaluation_scenario_score_service( - evaluation_scenario_id, **user_org_data - ) + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluation_scenario_id, + object_type="human_evaluation_scenario", + permission=Permission.VIEW_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + + return await get_evaluation_scenario_score_service(evaluation_scenario_id) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e @router.put("/evaluation_scenario/{evaluation_scenario_id}/score/") @@ -251,10 +338,23 @@ async def update_evaluation_scenario_score_router( Returns: None: 204 No Content status code upon successful update. """ - user_org_data = await get_user_and_org_id(request.state.user_id) try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluation_scenario_id, + object_type="human_evaluation_scenario", + permission=Permission.VIEW_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + await update_evaluation_scenario_score_service( - evaluation_scenario_id, payload.score, **user_org_data + evaluation_scenario_id, payload.score ) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as e: @@ -275,20 +375,33 @@ async def fetch_results( _description_ """ - # Get user and organization id - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - evaluation = await evaluation_service._fetch_human_evaluation_and_check_access( - evaluation_id, **user_org_data - ) - if evaluation.evaluation_type == EvaluationType.human_a_b_testing: - results = await results_service.fetch_results_for_evaluation(evaluation) - return {"votes_data": results} - - elif evaluation.evaluation_type == EvaluationType.single_model_test: - results = await results_service.fetch_results_for_single_model_test( - evaluation_id - ) - return {"results_data": results} + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluation_id, + object_type="evaluation", + permission=Permission.VIEW_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + + evaluation = await evaluation_service._fetch_human_evaluation(evaluation_id) + if evaluation.evaluation_type == EvaluationType.human_a_b_testing: + results = await results_service.fetch_results_for_evaluation(evaluation) + return {"votes_data": results} + + elif evaluation.evaluation_type == EvaluationType.single_model_test: + results = await results_service.fetch_results_for_single_model_test( + evaluation_id + ) + return {"results_data": results} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e @router.delete("/", response_model=List[str]) @@ -306,9 +419,23 @@ async def delete_evaluations( A list of the deleted comparison tables' IDs. """ - # Get user and organization id - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - await evaluation_service.delete_human_evaluations( - delete_evaluations.evaluations_ids, **user_org_data - ) - return Response(status_code=status.HTTP_204_NO_CONTENT) + try: + if FEATURE_FLAG in ["cloud", "ee"]: + for evaluation_id in delete_evaluations.evaluations_ids: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluation_id, + object_type="evaluation", + permission=Permission.DELETE_EVALUATION + ) + if not has_permission: + error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + + await evaluation_service.delete_human_evaluations(delete_evaluations.evaluations_ids) + return Response(status_code=status.HTTP_204_NO_CONTENT) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/agenta-backend/agenta_backend/routers/organization_router.py b/agenta-backend/agenta_backend/routers/organization_router.py deleted file mode 100644 index c8f4edf129..0000000000 --- a/agenta-backend/agenta_backend/routers/organization_router.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Routes for image-related operations (push, remove). -Does not deal with the instanciation of the images -""" - -import os -import logging -from fastapi import HTTPException, Request -from agenta_backend.utils.common import APIRouter -from agenta_backend.services.selectors import get_user_own_org -from agenta_backend.models.api.organization_models import ( - OrganizationOutput, - Organization, -) -from agenta_backend.services import db_manager - -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id - - -router = APIRouter() -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -@router.get("/", response_model=list[Organization], operation_id="list_organizations") -async def list_organizations( - request: Request, -): - """ - Returns a list of organizations associated with the user's session. - - Args: - stoken_session (SessionContainer): The user's session token. - - Returns: - list[Organization]: A list of organizations associated with the user's session. - - Raises: - HTTPException: If there is an error retrieving the organizations from the database. - """ - - try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - organizations_db = await db_manager.get_organizations_by_list_ids( - user_org_data["organization_ids"] - ) - response = [ - Organization( - id=str(org.id), - name=str(org.name), - description=str(org.description), - owner=str(org.owner), - ).dict(exclude_unset=True) - for org in organizations_db - ] - - return response - - except Exception as e: - raise HTTPException( - status_code=500, - detail=str(e), - ) - - -@router.get("/own/", response_model=OrganizationOutput, operation_id="get_own_org") -async def get_user_organization( - request: Request, -): - try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - org_db = await get_user_own_org(user_org_data["uid"]) - if org_db is None: - raise HTTPException(404, detail="User does not have an organization") - return OrganizationOutput(id=str(org_db.id), name=org_db.name) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index c97e2cd66e..29122b4257 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -36,8 +36,10 @@ from fastapi import HTTPException from fastapi.responses import JSONResponse -from beanie.operators import In -from beanie import PydanticObjectId as ObjectId +from beanie import ( + In, + PydanticObjectId as ObjectId +) FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: @@ -562,23 +564,6 @@ async def get_deployment_by_objectid( return deployment -async def get_organizations_by_list_ids(organization_ids: List) -> List: - """ - Retrieve organizations from the database by their IDs. - - Args: - organization_ids (List): A list of organization IDs to retrieve. - - Returns: - List: A list of dictionaries representing the retrieved organizations. - """ - - organizations_db = await OrganizationDB.find( - In(OrganizationDB.id, organization_ids) - ).to_list() - return organizations_db - - async def list_app_variants_for_app_id( app_id: str, ) -> List[AppVariantDB]: @@ -1418,6 +1403,23 @@ async def fetch_human_evaluation_scenario_by_id( return evaluation_scenario +async def fetch_human_evaluation_scenario_by_evaluation_id( + evaluation_id: str, +) -> Optional[HumanEvaluationScenarioDB]: + """Fetches and evaluation scenario by its ID. + Args: + evaluation_id (str): The ID of the evaluation object to use in fetching the human evaluation. + Returns: + EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. + """ + evaluation = await fetch_human_evaluation_by_id(evaluation_id) + human_eval_scenario = await HumanEvaluationScenarioDB.find_one( + HumanEvaluationScenarioDB.evaluation.id == ObjectId(evaluation.id), + fetch_links=True, + ) + return human_eval_scenario + + async def find_previous_variant_from_base_id( base_id: str, ) -> Optional[AppVariantDB]: @@ -1898,75 +1900,3 @@ async def update_evaluation( setattr(evaluation, key, value) await evaluation.save() return evaluation - - -async def get_object_workspace_org_id(object_id: str, type: str) -> dict: - """ - Get the organization and workspace id of the object. - - Args: - object_id (str): The ID of the object. - type (str): The type of the object. - - Returns: - dict: The organization and workspace id of the object. - """ - try: - if type == "app": - app = await fetch_app_by_id(object_id) - organization_id = app.organization.id - workspace_id = app.workspace.id - - elif type == "app_variant": - app_variant = await fetch_app_variant_by_id(object_id) - organization_id = app_variant.organization.id - workspace_id = app_variant.workspace.id - - elif type == "base": - base = await fetch_base_by_id(object_id) - organization_id = base.organization.id - workspace_id = base.workspace.id - - elif type == "deployment": - deployment = await get_deployment_by_objectid(object_id) - organization_id = deployment.organization.id - workspace_id = deployment.workspace.id - - elif type == "testset": - testset = await fetch_testset_by_id(object_id) - organization_id = testset.organization.id - workspace_id = testset.workspace.id - - elif type == "evaluation": - evaluation = await fetch_evaluation_by_id(object_id) - organization_id = evaluation.organization.id - workspace_id = evaluation.workspace.id - - elif type == "evaluation_scenario": - evaluation_scenario = await fetch_evaluation_scenario_by_id(object_id) - organization_id = evaluation_scenario.organization.id - workspace_id = evaluation_scenario.workspace.id - - elif type == "evaluator_config": - evaluator_config = await fetch_evaluator_config(object_id) - organization_id = evaluator_config.organization.id - workspace_id = evaluator_config.workspace.id - - elif type == "human_evaluation": - human_evaluation = await fetch_human_evaluation_by_id(object_id) - organization_id = human_evaluation.human_evaluation.id - workspace_id = human_evaluation.human_evaluation.id - - elif type == "human_evaluation_scenario": - human_evaluation_scenario = await fetch_human_evaluation_scenario_by_id( - object_id - ) - organization_id = human_evaluation_scenario.organization.id - workspace_id = human_evaluation_scenario.workspace.id - - else: - raise ValueError(f"Unknown object type: {type}") - - return {"organization_id": organization_id, "workspace_id": workspace_id} - except Exception as e: - raise e diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index b78c04d8e4..10631e9475 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -61,14 +61,14 @@ class UpdateEvaluationScenarioError(Exception): pass -async def _fetch_human_evaluation_and_check_access( - evaluation_id: str, **user_org_data: dict +async def _fetch_human_evaluation( + evaluation_id: str ) -> HumanEvaluationDB: # Fetch the evaluation by ID evaluation = await db_manager.fetch_human_evaluation_by_id( evaluation_id=evaluation_id ) - + # Check if the evaluation exists if evaluation is None: raise HTTPException( @@ -76,20 +76,11 @@ async def _fetch_human_evaluation_and_check_access( detail=f"Evaluation with id {evaluation_id} not found", ) - # Check for access rights - access = await check_access_to_app( - user_org_data=user_org_data, app_id=evaluation.app.id - ) - if not access: - raise HTTPException( - status_code=403, - detail=f"You do not have access to this app: {str(evaluation.app.id)}", - ) return evaluation -async def _fetch_human_evaluation_scenario_and_check_access( - evaluation_scenario_id: str, **user_org_data: dict +async def _fetch_human_evaluation_scenario( + evaluation_scenario_id: str ) -> HumanEvaluationDB: # Fetch the evaluation by ID evaluation_scenario = await db_manager.fetch_human_evaluation_scenario_by_id( @@ -109,15 +100,6 @@ async def _fetch_human_evaluation_scenario_and_check_access( detail=f"Evaluation scenario for evaluation scenario with id {evaluation_scenario_id} not found", ) - # Check for access rights - access = await check_access_to_app( - user_org_data=user_org_data, app_id=evaluation.app.id - ) - if not access: - raise HTTPException( - status_code=403, - detail=f"You do not have access to this app: {str(evaluation.app.id)}", - ) return evaluation_scenario @@ -191,7 +173,7 @@ async def prepare_csvdata_and_create_evaluation_scenario( async def create_evaluation_scenario( - evaluation_id: str, payload: EvaluationScenario, **user_org_data: dict + evaluation_id: str, payload: EvaluationScenario ) -> None: """ Create a new evaluation scenario. @@ -199,7 +181,6 @@ async def create_evaluation_scenario( Args: evaluation_id (str): The ID of the evaluation. payload (EvaluationScenario): Evaluation scenario data. - user_org_data (dict): User and organization data. Raises: HTTPException: If evaluation not found or access denied. @@ -232,7 +213,7 @@ async def create_evaluation_scenario( async def update_human_evaluation_service( - evaluation_id: str, update_payload: HumanEvaluationUpdate, **user_org_data: dict + evaluation_id: str, update_payload: HumanEvaluationUpdate ) -> None: """ Update an existing evaluation based on the provided payload. @@ -245,10 +226,7 @@ async def update_human_evaluation_service( HTTPException: If the evaluation is not found or access is denied. """ # Fetch the evaluation by ID - evaluation = await _fetch_human_evaluation_and_check_access( - evaluation_id=evaluation_id, - **user_org_data, - ) + evaluation = await _fetch_human_evaluation(evaluation_id=evaluation_id) # Prepare updates updates = {} @@ -286,14 +264,13 @@ async def fetch_evaluation_scenarios_for_evaluation( async def fetch_human_evaluation_scenarios_for_evaluation( - evaluation_id: str, **user_org_data: dict + evaluation_id: str, ) -> List[HumanEvaluationScenario]: """ Fetch evaluation scenarios for a given evaluation ID. Args: evaluation_id (str): The ID of the evaluation. - user_org_data (dict): User and organization data. Raises: HTTPException: If the evaluation is not found or access is denied. @@ -301,9 +278,8 @@ async def fetch_human_evaluation_scenarios_for_evaluation( Returns: List[EvaluationScenario]: A list of evaluation scenarios. """ - evaluation = await _fetch_human_evaluation_and_check_access( + evaluation = await _fetch_human_evaluation( evaluation_id=evaluation_id, - **user_org_data, ) scenarios = await HumanEvaluationScenarioDB.find( HumanEvaluationScenarioDB.evaluation.id == ObjectId(evaluation.id), @@ -319,8 +295,7 @@ async def fetch_human_evaluation_scenarios_for_evaluation( async def update_human_evaluation_scenario( evaluation_scenario_id: str, evaluation_scenario_data: EvaluationScenarioUpdate, - evaluation_type: EvaluationType, - **user_org_data, + evaluation_type: EvaluationType ) -> None: """ Updates an evaluation scenario. @@ -329,15 +304,11 @@ async def update_human_evaluation_scenario( evaluation_scenario_id (str): The ID of the evaluation scenario. evaluation_scenario_data (EvaluationScenarioUpdate): New data for the scenario. evaluation_type (EvaluationType): Type of the evaluation. - user_org_data (dict): User and organization data. Raises: HTTPException: If evaluation scenario not found or access denied. """ - eval_scenario = await _fetch_human_evaluation_scenario_and_check_access( - evaluation_scenario_id=evaluation_scenario_id, - **user_org_data, - ) + eval_scenario = await _fetch_human_evaluation_scenario(evaluation_scenario_id) updated_data = evaluation_scenario_data.dict() updated_data["updated_at"] = datetime.utcnow() @@ -386,7 +357,7 @@ async def update_human_evaluation_scenario( async def update_evaluation_scenario_score_service( - evaluation_scenario_id: str, score: float, **user_org_data: dict + evaluation_scenario_id: str, score: float ) -> None: """ Updates the score of an evaluation scenario. @@ -394,14 +365,11 @@ async def update_evaluation_scenario_score_service( Args: evaluation_scenario_id (str): The ID of the evaluation scenario. score (float): The new score to set. - user_org_data (dict): User and organization data. Raises: HTTPException: If evaluation scenario not found or access denied. """ - eval_scenario = await _fetch_human_evaluation_scenario_and_check_access( - evaluation_scenario_id, **user_org_data - ) + eval_scenario = await _fetch_human_evaluation_scenario(evaluation_scenario_id) eval_scenario.score = score # Save the updated evaluation scenario @@ -409,21 +377,18 @@ async def update_evaluation_scenario_score_service( async def get_evaluation_scenario_score_service( - evaluation_scenario_id: str, **user_org_data: dict + evaluation_scenario_id: str ) -> Dict[str, str]: """ Retrieve the score of a given evaluation scenario. Args: evaluation_scenario_id: The ID of the evaluation scenario. - user_org_data: Additional user and organization data. Returns: Dictionary with 'scenario_id' and 'score' keys. """ - evaluation_scenario = await _fetch_human_evaluation_scenario_and_check_access( - evaluation_scenario_id, **user_org_data - ) + evaluation_scenario = await _fetch_human_evaluation_scenario(evaluation_scenario_id) return { "scenario_id": str(evaluation_scenario.id), "score": evaluation_scenario.score, @@ -485,25 +450,16 @@ async def fetch_evaluation(evaluation_id: str) -> Evaluation: async def fetch_list_human_evaluations( app_id: str, - **user_org_data: dict, ) -> List[HumanEvaluation]: """ Fetches a list of evaluations based on the provided filtering criteria. Args: app_id (Optional[str]): An optional app ID to filter the evaluations. - user_org_data (dict): User and organization data. Returns: List[Evaluation]: A list of evaluations. """ - access = await check_access_to_app(user_org_data=user_org_data, app_id=app_id) - if not access: - raise HTTPException( - status_code=403, - detail=f"You do not have access to this app: {app_id}", - ) - evaluations_db = await HumanEvaluationDB.find( HumanEvaluationDB.app.id == ObjectId(app_id), fetch_links=True ).to_list() @@ -514,40 +470,38 @@ async def fetch_list_human_evaluations( async def fetch_human_evaluation( - evaluation_id: str, **user_org_data: dict + evaluation_id: str ) -> HumanEvaluation: """ Fetches a single evaluation based on its ID. Args: evaluation_id (str): The ID of the evaluation. - user_org_data (dict): User and organization data. Returns: Evaluation: The fetched evaluation. """ - evaluation = await _fetch_human_evaluation_and_check_access( - evaluation_id=evaluation_id, **user_org_data + evaluation = await _fetch_human_evaluation( + evaluation_id=evaluation_id ) return await converters.human_evaluation_db_to_pydantic(evaluation) async def delete_human_evaluations( - evaluation_ids: List[str], **user_org_data: dict + evaluation_ids: List[str] ) -> None: """ Delete evaluations by their IDs. Args: evaluation_ids (List[str]): A list of evaluation IDs. - user_org_data (dict): User and organization data. Raises: HTTPException: If evaluation not found or access denied. """ for evaluation_id in evaluation_ids: - evaluation = await _fetch_human_evaluation_and_check_access( - evaluation_id=evaluation_id, **user_org_data + evaluation = await _fetch_human_evaluation( + evaluation_id=evaluation_id ) await evaluation.delete() @@ -568,19 +522,19 @@ async def delete_evaluations(evaluation_ids: List[str]) -> None: async def create_new_human_evaluation( - payload: NewHumanEvaluation, **user_org_data: dict + payload: NewHumanEvaluation, user_uid: str ) -> HumanEvaluationDB: """ Create a new evaluation based on the provided payload and additional arguments. Args: payload (NewEvaluation): The evaluation payload. - **user_org_data (dict): Additional keyword arguments, e.g., user id. + user_uid (str): The user_uid of the user Returns: HumanEvaluationDB """ - user = await db_manager.get_user(user_uid=user_org_data["uid"]) + user = await db_manager.get_user(user_uid) # Initialize evaluation type settings evaluation_type_settings = {} diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index a602eb6337..382b5eaa38 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -1,5 +1,5 @@ -import json import os +import json from typing import Any, Dict, Optional, List, Tuple from fastapi.responses import JSONResponse @@ -7,7 +7,6 @@ from agenta_backend.services import db_manager FEATURE_FLAG = os.environ["FEATURE_FLAG"] - if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.db_models import ( AppDB_ as AppDB, @@ -15,8 +14,8 @@ ) else: from agenta_backend.models.db_models import AppDB, EvaluatorConfigDB -from agenta_backend.models.api.evaluation_model import Evaluator, EvaluatorConfig from agenta_backend.models.converters import evaluator_config_db_to_pydantic +from agenta_backend.models.api.evaluation_model import Evaluator, EvaluatorConfig def get_evaluators() -> Optional[List[Evaluator]]: diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index 3caa069dbe..e6d5a2ba2f 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -1,10 +1,10 @@ -import asyncio import json +import httpx import logging -from typing import Any, Dict, List +import asyncio import traceback +from typing import Any, Dict, List -import httpx from agenta_backend.models.api.evaluation_model import AppOutput diff --git a/agenta-backend/agenta_backend/services/results_service.py b/agenta-backend/agenta_backend/services/results_service.py index d33a1c419f..7ef943b07f 100644 --- a/agenta-backend/agenta_backend/services/results_service.py +++ b/agenta-backend/agenta_backend/services/results_service.py @@ -1,8 +1,18 @@ -from agenta_backend.models.db_models import ( - HumanEvaluationDB, - EvaluationScenarioDB, - HumanEvaluationScenarioDB, -) +import os + +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.models.db_models import ( + HumanEvaluationDB_ as HumanEvaluationDB, + EvaluationScenarioDB_ as EvaluationScenarioDB, + HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, + ) +else: + from agenta_backend.models.db_models import ( + HumanEvaluationDB, + EvaluationScenarioDB, + HumanEvaluationScenarioDB, + ) from agenta_backend.services import db_manager from agenta_backend.models.api.evaluation_model import EvaluationType diff --git a/agenta-backend/agenta_backend/services/selectors.py b/agenta-backend/agenta_backend/services/selectors.py deleted file mode 100644 index 143f53606f..0000000000 --- a/agenta-backend/agenta_backend/services/selectors.py +++ /dev/null @@ -1,58 +0,0 @@ -from typing import Tuple, Dict, List - -from agenta_backend.models.db_models import ( - UserDB, - OrganizationDB, - WorkspaceDB, -) - - -# async def get_user_and_org_id(user_uid_id) -> Dict[str, List]: -# """Retrieves the user ID and organization ID based on the logged-in session. - -# Arguments: -# session (SessionContainer): Used to store and manage the user's session data - -# Returns: -# A dictionary containing the user_id and a list of the user's organization_ids. -# """ -# user, org_ids = await get_user_objectid(user_uid_id) -# return {"uid": str(user.uid), "id": str(user.id), "organization_ids": org_ids} - - -# async def get_user_objectid(user_uid: str) -> Tuple[str, List]: -# """Retrieves the user object ID and organization IDs from the database -# based on the user ID. - -# Arguments: -# user_id (str): The unique identifier of a user - -# Returns: -# a tuple containing the string representation of the user's ObjectId and the List -# of the user's organization_ids. -# """ - -# user = await UserDB.find_one(UserDB.uid == user_uid) -# if user is not None: -# organization_ids: List = ( -# [org for org in user.organizations] if user.organizations else [] -# ) -# return user, organization_ids -# return None, [] - - -# async def get_user_own_org(user_uid: str) -> OrganizationDB: -# """Get's the default users' organization from the database. - -# Arguments: -# user_uid (str): The uid of the user - -# Returns: -# Organization: Instance of OrganizationDB -# """ - -# user = await UserDB.find_one(UserDB.uid == user_uid) -# org: OrganizationDB = await OrganizationDB.find_one( -# OrganizationDB.owner == str(user.id), OrganizationDB.type == "default" -# ) -# return org diff --git a/agenta-backend/agenta_backend/services/templates_manager.py b/agenta-backend/agenta_backend/services/templates_manager.py index 28b793ce00..c77da181e6 100644 --- a/agenta-backend/agenta_backend/services/templates_manager.py +++ b/agenta-backend/agenta_backend/services/templates_manager.py @@ -1,13 +1,15 @@ +import os import json +import httpx import backoff + from typing import Any, Dict, List -import httpx -import os +from asyncio.exceptions import CancelledError +from httpx import ConnectError, TimeoutException + from agenta_backend.config import settings -from agenta_backend.services import db_manager from agenta_backend.utils import redis_utils -from httpx import ConnectError, TimeoutException -from asyncio.exceptions import CancelledError +from agenta_backend.services import db_manager if os.environ["FEATURE_FLAG"] in ["oss", "cloud"]: from agenta_backend.services import container_manager diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index bc021239c0..f3a6eb3502 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -1,23 +1,9 @@ import logging -from typing import Dict, List, Union, Optional, Any, Callable +from typing import Any, Callable from fastapi.types import DecoratedCallable from fastapi import APIRouter as FastAPIRouter -from agenta_backend.models.db_models import ( - UserDB, - AppVariantDB, - OrganizationDB, - AppDB, - VariantBaseDB, - WorkspaceDB, - Permission, - WorkspaceRole, -) - -from beanie import PydanticObjectId as ObjectId - - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -61,111 +47,3 @@ def decorator(func: DecoratedCallable) -> DecoratedCallable: return add_path(func) return decorator - - -async def get_organization(org_id: str) -> OrganizationDB: - org = await OrganizationDB.find_one(OrganizationDB.id == ObjectId(org_id)) - if org is not None: - return org - else: - return None - - -async def get_app_instance( - app_id: str, variant_name: str = None, show_deleted: bool = False -) -> AppVariantDB: - queries = (AppVariantDB.is_deleted == show_deleted, AppVariantDB.app == app_id) - if variant_name is not None: - queries += AppVariantDB.variant_name == variant_name - - app_instance = await AppVariantDB.find_one(*queries) - return app_instance - - -async def check_user_org_access( - kwargs: dict, organization_id: str, check_owner=False -) -> bool: - if check_owner: # Check that the user is the owner of the organization - user = await UserDB.find_one(UserDB.uid == kwargs["uid"]) - organization = await get_organization(organization_id) - if not organization: - logger.error("Organization not found") - raise Exception("Organization not found") - return organization.owner == str(user.id) - else: - user_organizations: List = kwargs["organization_ids"] - object_organization_id = ObjectId(organization_id) - logger.debug( - f"object_organization_id: {object_organization_id}, user_organizations: {user_organizations}" - ) - user_exists_in_organizations = object_organization_id in user_organizations - return user_exists_in_organizations - - -async def check_access_to_app( - user_org_data: Dict[str, Union[str, list]], - app: Optional[AppDB] = None, - app_id: Optional[str] = None, - check_owner: bool = False, -) -> bool: - """ - Check if a user has access to a specific application. - - Args: - user_org_data (Dict[str, Union[str, list]]): User-specific information. - app (Optional[AppDB]): An instance of the AppDB model representing the application. - app_id (Optional[str]): The ID of the application. - check_owner (bool): Whether to check if the user is the owner of the application. - - Returns: - bool: True if the user has access, False otherwise. - - Raises: - Exception: If neither or both `app` and `app_id` are provided. - - """ - if (app is None) == (app_id is None): - raise Exception("Provide either app or app_id, not both or neither") - - # Fetch the app if only app_id is provided. - if app is None: - app = await AppDB.find_one(AppDB.id == ObjectId(app_id), fetch_links=True) - if app is None: - logger.error("App not found") - return False - - # Check user's access to the organization linked to the app. - organization_id = app.organization.id - return await check_user_org_access(user_org_data, str(organization_id), check_owner) - - -async def check_access_to_variant( - user_org_data: Dict[str, Union[str, list]], - variant_id: str, - check_owner: bool = False, -) -> bool: - if variant_id is None: - raise Exception("No variant_id provided") - variant = await AppVariantDB.find_one( - AppVariantDB.id == ObjectId(variant_id), fetch_links=True - ) - if variant is None: - logger.error("Variant not found") - return False - organization_id = variant.organization.id - return await check_user_org_access(user_org_data, str(organization_id), check_owner) - - -async def check_access_to_base( - user_org_data: Dict[str, Union[str, list]], - base_id: str, - check_owner: bool = False, -) -> bool: - if base_id is None: - raise Exception("No base_id provided") - base = await VariantBaseDB.find_one(VariantBaseDB.id == base_id, fetch_links=True) - if base is None: - logger.error("Base not found") - return False - organization_id = base.organization.id - return await check_user_org_access(user_org_data, str(organization_id), check_owner) From fc1d7127a665df53f98ecdb54033e4920275dae7 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 19 Jan 2024 11:55:35 +0100 Subject: [PATCH 24/99] Improvements and fix bugs with refactor --- .../agenta_backend/models/converters.py | 2 +- .../agenta_backend/routers/configs_router.py | 4 +- .../routers/environment_router.py | 1 + .../routers/evaluation_router.py | 8 + .../routers/observability_router.py | 47 +--- .../agenta_backend/routers/testset_router.py | 229 +++++++++--------- .../agenta_backend/routers/user_profile.py | 10 +- .../agenta_backend/routers/variants_router.py | 195 ++++++++------- .../agenta_backend/services/app_manager.py | 9 +- .../agenta_backend/services/db_manager.py | 74 +++--- .../services/event_db_manager.py | 38 +-- agenta-backend/agenta_backend/utils/common.py | 2 + 12 files changed, 290 insertions(+), 329 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 5c2faad350..74c40e6207 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -2,7 +2,7 @@ """ import json from typing import List -from agenta_backend.services import db_manager +from agenta_backend.utils.common import db_manager from agenta_backend.models.api.user_models import User from agenta_backend.models.db_models import ( AppVariantDB, diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index cc1395b27f..1715857b4e 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -37,7 +37,7 @@ async def save_config( user_id=request.state.user_id, object_id=payload.base_id, object_type="base", - permission=Permission.MODIFY_CONFIGURATIONS, + permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." @@ -98,7 +98,7 @@ async def get_config( user_id=request.state.user_id, object_id=base_id, object_type="base", - permission=Permission.MODIFY_CONFIGURATIONS, + permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 653d948375..08d486c66f 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -41,6 +41,7 @@ async def deploy_to_environment( object_type="app_variant", permission=Permission.DEPLOY_APPLICATION, ) + logger.debug(f"User has permission deploy to environment: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 23dee0c531..62c8e8447f 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -52,6 +52,7 @@ async def create_evaluation( object_type="app", permission=Permission.CREATE_EVALUATION, ) + logger.debug(f"User has permission to create evaluation: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -119,6 +120,7 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) + logger.debug(f"User has permission to fetch evaluation status: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -153,6 +155,7 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) + logger.debug(f"User has permission to get evaluation results: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -196,6 +199,7 @@ async def fetch_evaluation_scenarios( object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) + logger.debug(f"User has permission to get evaluation scenarios: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -232,6 +236,7 @@ async def fetch_list_evaluations( object_type="app", permission=Permission.VIEW_EVALUATION, ) + logger.debug(f"User has permission to get list of evaluations: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -268,6 +273,7 @@ async def fetch_evaluation( object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) + logger.debug(f"User has permission to get single evaluation: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -305,6 +311,7 @@ async def delete_evaluations( object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) + logger.debug(f"User has permission to delete evaluation: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -367,6 +374,7 @@ async def fetch_evaluation_scenarios( object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) + logger.debug(f"User has permission to get evaluation scenarios: {has_permission}") if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) diff --git a/agenta-backend/agenta_backend/routers/observability_router.py b/agenta-backend/agenta_backend/routers/observability_router.py index f385a99c33..aa82dcd2a4 100644 --- a/agenta-backend/agenta_backend/routers/observability_router.py +++ b/agenta-backend/agenta_backend/routers/observability_router.py @@ -27,13 +27,6 @@ UpdateTrace, ) -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id - router = APIRouter() @@ -43,9 +36,7 @@ async def create_trace( payload: CreateTrace, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - trace = await create_app_trace(payload, **kwargs) + trace = await create_app_trace(payload, request.state.user_id) return trace @@ -59,9 +50,7 @@ async def get_traces( variant_id: str, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - traces = await get_variant_traces(app_id, variant_id, **kwargs) + traces = await get_variant_traces(app_id, variant_id, request.state.user_id) return traces @@ -72,9 +61,7 @@ async def get_single_trace( trace_id: str, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - trace = await get_trace_single(trace_id, **kwargs) + trace = await get_trace_single(trace_id, request.state.user_id) return trace @@ -83,9 +70,7 @@ async def create_span( payload: CreateSpan, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - spans_id = await create_trace_span(payload, **kwargs) + spans_id = await create_trace_span(payload, request.state.user_id) return spans_id @@ -96,9 +81,7 @@ async def get_spans_of_trace( trace_id: str, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - spans = await get_trace_spans(trace_id, **kwargs) + spans = await get_trace_spans(trace_id, request.state.user_id) return spans @@ -110,9 +93,7 @@ async def update_trace_status( payload: UpdateTrace, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - trace = await trace_status_update(trace_id, payload, **kwargs) + trace = await trace_status_update(trace_id, payload, request.state.user_id) return trace @@ -124,9 +105,7 @@ async def create_feedback( payload: CreateFeedback, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - feedback = await add_feedback_to_trace(trace_id, payload, **kwargs) + feedback = await add_feedback_to_trace(trace_id, payload, request.state.user_id) return feedback @@ -136,9 +115,7 @@ async def create_feedback( operation_id="get_feedbacks", ) async def get_feedbacks(trace_id: str, request: Request): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - feedbacks = await get_trace_feedbacks(trace_id, **kwargs) + feedbacks = await get_trace_feedbacks(trace_id, request.state.user_id) return feedbacks @@ -152,9 +129,7 @@ async def get_feedback( feedback_id: str, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - feedback = await get_feedback_detail(trace_id, feedback_id, **kwargs) + feedback = await get_feedback_detail(trace_id, feedback_id, request.state.user_id) return feedback @@ -169,7 +144,5 @@ async def update_feedback( payload: UpdateFeedback, request: Request, ): - # Get user and org id - kwargs: dict = await get_user_and_org_id(request.state.user_id) - feedback = await update_trace_feedback(trace_id, feedback_id, payload, **kwargs) + feedback = await update_trace_feedback(trace_id, feedback_id, payload, request.state.user_id) return feedback diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 964b2489fb..a19c57c245 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -12,12 +12,9 @@ from fastapi.responses import JSONResponse from agenta_backend.services import db_manager from agenta_backend.utils.common import APIRouter -from agenta_backend.models.db_models import TestSetDB -from agenta_backend.models.db_models import Permission from agenta_backend.services.db_manager import get_user from fastapi import HTTPException, UploadFile, File, Form, Request from agenta_backend.models.converters import testset_db_to_pydantic -from agenta_backend.utils.common import APIRouter, check_rbac_permission from agenta_backend.models.api.testset_model import ( @@ -27,6 +24,18 @@ TestSetOutputResponse, ) +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all + from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all + from agenta_backend.commons.models.db_models import ( + TestSetDB_ as TestSetDB + ) # noqa pylint: disable-all +else: + from agenta_backend.models.db_models import ( + TestSetDB + ) + router = APIRouter() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -34,14 +43,6 @@ upload_folder = "./path/to/upload/folder" -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id - - @router.post( "/upload/", response_model=TestSetSimpleResponse, operation_id="upload_file" ) @@ -64,21 +65,21 @@ async def upload_file( dict: The result of the upload process. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.CREATE_TESTSET, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + permission=Permission.CREATE_TESTSET, ) + logger.debug(f"User has Permission to upload Testset: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) app = await db_manager.fetch_app_by_id(app_id=app_id) # Create a document @@ -114,7 +115,7 @@ async def upload_file( for row in csv_reader: document["csvdata"].append(row) - user = await get_user(user_uid=user_org_data["uid"]) + user = await get_user(request.state.user_id) try: testset_instance = TestSetDB(**document, user=user) except ValidationError as e: @@ -148,21 +149,21 @@ async def import_testset( Returns: dict: The result of the import process. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.CREATE_TESTSET, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + permission=Permission.CREATE_TESTSET, ) + logger.debug(f"User has Permission to import Testset: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) app = await db_manager.fetch_app_by_id(app_id=app_id) @@ -189,7 +190,7 @@ async def import_testset( for row in json_response: document["csvdata"].append(row) - user = await get_user(user_uid=user_org_data["uid"]) + user = await get_user(request.state.user_id) testset_instance = TestSetDB(**document, user=user) result = await testset_instance.create() @@ -235,23 +236,23 @@ async def create_testset( str: The id of the test set created. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.CREATE_TESTSET, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + permission=Permission.CREATE_TESTSET, ) + logger.debug(f"User has Permission to create Testset: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) - user = await get_user(user_uid=user_org_data["uid"]) + user = await get_user(request.state.user_id) app = await db_manager.fetch_app_by_id(app_id=app_id) testset = { "created_at": datetime.now().isoformat(), @@ -294,22 +295,21 @@ async def update_testset( Returns: str: The id of the test set updated. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(testset_id, "testset") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.EDIT_TESTSET, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=testset_id, + object_type="testset", + permission=Permission.EDIT_TESTSET, ) - + logger.debug(f"User has Permission to update Testset: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) testset_update = { "name": csvdata.name, @@ -350,21 +350,21 @@ async def get_testsets( Raises: - `HTTPException` with status code 404 if no testsets are found. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.VIEW_TESTSET, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=app_id, + object_type="app", + permission=Permission.VIEW_TESTSET, ) + logger.debug(f"User has Permission to view Testsets: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) app = await db_manager.fetch_app_by_id(app_id=app_id) @@ -396,21 +396,21 @@ async def get_single_testset( Returns: The requested testset if found, else an HTTPException. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(testset_id, "testset") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.VIEW_TESTSET, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=testset_id, + object_type="testset", + permission=Permission.VIEW_TESTSET, ) + logger.debug(f"User has Permission to view Testset: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: @@ -433,21 +433,22 @@ async def delete_testsets( Returns: A list of the deleted testsets' IDs. """ - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(app_id, "app") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.DELETE_TESTSET, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, - ) + if FEATURE_FLAG in ["cloud", "ee"]: + for testset_id in delete_testsets.testset_ids: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=testset_id, + object_type="testset", + permission=Permission.VIEW_TESTSET, + ) + logger.debug(f"User has Permission to delete Testset: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) deleted_ids = [] for testset_id in delete_testsets.testset_ids: diff --git a/agenta-backend/agenta_backend/routers/user_profile.py b/agenta-backend/agenta_backend/routers/user_profile.py index c34774cddc..d0e3332472 100644 --- a/agenta-backend/agenta_backend/routers/user_profile.py +++ b/agenta-backend/agenta_backend/routers/user_profile.py @@ -6,21 +6,13 @@ router = APIRouter() -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id - @router.get("/", operation_id="user_profile") async def user_profile( request: Request, ): try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - user = await db_manager.get_user(user_uid=user_org_data["uid"]) + user = await db_manager.get_user(request.state.user_id) return User( id=str(user.id), uid=str(user.uid), diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 7c956b77ee..0cd204a3e1 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -4,34 +4,38 @@ from typing import Any, Optional, Union from docker.errors import DockerException from fastapi.responses import JSONResponse +from agenta_backend.models import converters from fastapi import HTTPException, Request, Body -from agenta_backend.models.db_models import Permission -from agenta_backend.utils.common import APIRouter, check_rbac_permission +from agenta_backend.utils.common import APIRouter from agenta_backend.services import ( app_manager, db_manager, ) -from agenta_backend.models import converters + +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all + from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all + from agenta_backend.models.api.api_models import ( + Image_ as Image, + AppVariantOutput_ as AppVariantOutput, + ) +else: + from agenta_backend.models.api.api_models import ( + Image, + AppVariantOutput, + ) from agenta_backend.models.api.api_models import ( - Image, URI, DockerEnvVars, - AddVariantFromBasePayload, - AppVariantOutput, - UpdateVariantParameterPayload, VariantAction, VariantActionEnum, + AddVariantFromBasePayload, + UpdateVariantParameterPayload, ) -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all -else: - from agenta_backend.services.selectors import get_user_and_org_id - router = APIRouter() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -59,22 +63,22 @@ async def add_variant_from_base_and_config( logger.debug("Initiating process to add a variant based on a previous one.") logger.debug(f"Received payload: {payload}") - # Check user has permission to add variant - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(payload.base_id, "base") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.CREATE_APPLICATION, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + # Check user has permission to add variant + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=payload.base_id, + object_type="base", + permission=Permission.CREATE_APPLICATION, ) + logger.debug(f"User has Permission to create variant from base and config: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) base_db = await db_manager.fetch_base_by_id(payload.base_id) @@ -83,7 +87,7 @@ async def add_variant_from_base_and_config( base_db=base_db, new_config_name=payload.new_config_name, parameters=payload.parameters, - **user_org_data, + user_uid=request.state.user_id, ) logger.debug(f"Successfully added new variant: {db_app_variant}") app_variant_db = await db_manager.get_app_variant_instance_by_id( @@ -114,25 +118,23 @@ async def remove_variant( HTTPException: If there is a problem removing the app variant """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(variant_id, "app_variant") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.DELETE_APPLICATION_VARIANT, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=variant_id, + object_type="app_variant", + permission=Permission.DELETE_APPLICATION_VARIANT, ) - - await app_manager.terminate_and_remove_app_variant( - app_variant_id=variant_id, **user_org_data - ) + logger.debug(f"User has Permission to delete app variant: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + await app_manager.terminate_and_remove_app_variant(app_variant_id=variant_id) except DockerException as e: detail = f"Docker error while trying to remove the app variant: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -162,26 +164,25 @@ async def update_variant_parameters( JSONResponse: A JSON response containing the updated app variant parameters. """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(variant_id, "app_variant") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.MODIFY_CONFIGURATIONS, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=variant_id, + object_type="app_variant", + permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) + logger.debug(f"User has Permission to update variant parameters: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) await app_manager.update_variant_parameters( app_variant_id=variant_id, parameters=payload.parameters, - **user_org_data, ) except ValueError as e: detail = f"Error while trying to update the app variant: {str(e)}" @@ -211,27 +212,27 @@ async def update_variant_image( JSONResponse: A JSON response indicating whether the update was successful or not. """ try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - workspace_org_data = await db_manager.get_object_workspace_org_id(variant_id, "app_variant") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.CREATE_APPLICATION, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=variant_id, + object_type="app_variant", + permission=Permission.CREATE_APPLICATION, ) + logger.debug(f"User has Permission to update variant image: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) db_app_variant = await db_manager.fetch_app_variant_by_id( app_variant_id=variant_id ) - await app_manager.update_variant_image(db_app_variant, image, **user_org_data) + await app_manager.update_variant_image(db_app_variant, image) except ValueError as e: detail = f"Error while trying to update the app variant: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -266,8 +267,24 @@ async def start_variant( HTTPException: If the app container cannot be started. """ + # Check user has permission to start variant + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_id=request.state.user_id, + object_id=variant_id, + object_type="app_variant", + permission=Permission.CREATE_APPLICATION, + ) + logger.debug(f"User has Permission to start variant: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + logger.debug("Starting variant %s", variant_id) - user_org_data: dict = await get_user_and_org_id(request.state.user_id) # Inject env vars to docker container if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: @@ -281,26 +298,8 @@ async def start_variant( } else: envvars = {} if env_vars is None else env_vars.env_vars - - # Check user has permission to start variant - workspace_org_data = await db_manager.get_object_workspace_org_id(variant_id, "app_variant") - has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=workspace_org_data["workspace_id"], - organization_id=workspace_org_data["organization_id"], - role=Permission.CREATE_APPLICATION, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, - ) app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id=variant_id) if action.action == VariantActionEnum.START: - url: URI = await app_manager.start_variant( - app_variant_db, envvars, **user_org_data - ) + url: URI = await app_manager.start_variant(app_variant_db, envvars) return url diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 4522ef72d2..133a5124b9 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -21,7 +21,7 @@ evaluator_manager, ) -FEATURE_FLAG = s.environ["FEATURE_FLAG"] +FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud"]: from agenta_backend.cloud.services import ( @@ -45,8 +45,7 @@ async def start_variant( db_app_variant: AppVariantDB, - env_vars: DockerEnvVars = None, - **kwargs: dict, + env_vars: DockerEnvVars = None ) -> URI: """ Starts a Docker container for a given app variant. @@ -112,7 +111,7 @@ async def start_variant( async def update_variant_image( - app_variant_db: AppVariantDB, image: Image, **kwargs: dict + app_variant_db: AppVariantDB, image: Image ): """Updates the image for app variant in the database. @@ -155,7 +154,7 @@ async def update_variant_image( app_variant_db = await db_manager.update_app_variant(app_variant_db, image=db_image) # Start variant - await start_variant(app_variant_db, **kwargs) + await start_variant(app_variant_db) async def terminate_and_remove_app_variant( diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 29122b4257..8ca3f53bdf 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -1,44 +1,31 @@ import os import logging + from pathlib import Path from datetime import datetime from urllib.parse import urlparse from typing import Any, Dict, List, Optional +from fastapi import HTTPException +from fastapi.responses import JSONResponse + +from agenta_backend.services.json_importer_helper import get_json +from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum + from agenta_backend.models.api.api_models import ( App, Template, ) -from agenta_backend.models.converters import ( - app_db_to_pydantic, - image_db_to_pydantic, - templates_db_to_pydantic, -) -from agenta_backend.services.json_importer_helper import get_json + from agenta_backend.models.db_models import ( + Result, AggregatedResult, EvaluationScenarioInputDB, EvaluationScenarioOutputDB, EvaluationScenarioResult, ConfigDB, ConfigVersionDB, - OrganizationDB, - DeploymentDB, TemplateDB, - WorkspaceDB, -) -from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum -from agenta_backend.utils.common import ( - check_user_org_access, - check_user_org_workspace_access, -) - -from fastapi import HTTPException -from fastapi.responses import JSONResponse - -from beanie import ( - In, - PydanticObjectId as ObjectId ) FEATURE_FLAG = os.environ["FEATURE_FLAG"] @@ -57,6 +44,7 @@ TestSetDB_ as TestSetDB, AppVariantDB_ as AppVariantDB, EvaluationDB_ as EvaluationDB, + DeploymentDB_ as DeploymentDB, VariantBaseDB_ as VariantBaseDB, AppEnvironmentDB_ as AppEnvironmentDB, EvaluatorConfigDB_ as EvaluatorConfigDB, @@ -77,6 +65,7 @@ TestSetDB, AppVariantDB, EvaluationDB, + DeploymentDB, VariantBaseDB, AppEnvironmentDB, EvaluatorConfigDB, @@ -98,6 +87,10 @@ Permission, WorkspaceRole ) + +from beanie.operators import In +from beanie import PydanticObjectId as ObjectId + # Define logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -285,8 +278,8 @@ async def create_new_variant_base( user: UserDB, base_name: str, image: ImageDB, - organization: OrganizationDB = None, - workspace: WorkspaceDB = None, + organization = None, + workspace = None, ) -> VariantBaseDB: """Create a new base. Args: @@ -355,8 +348,8 @@ async def create_new_app_variant( base_name: str, config_name: str, parameters: Dict, - organization: OrganizationDB = None, - workspace: WorkspaceDB = None, + organization = None, + workspace = None, ) -> AppVariantDB: """Create a new variant. Args: @@ -396,8 +389,8 @@ async def create_image( image_type: str, user: UserDB, deletable: bool, - organization: OrganizationDB = None, - workspace: WorkspaceDB = None, + organization = None, + workspace = None, template_uri: str = None, docker_id: str = None, tags: str = None, @@ -461,8 +454,8 @@ async def create_deployment( container_id: str, uri: str, status: str, - organization: OrganizationDB = None, - workspace: WorkspaceDB = None, + organization = None, + workspace = None, ) -> DeploymentDB: """Create a new deployment. Args: @@ -625,18 +618,11 @@ async def get_user(user_uid: str) -> UserDB: user = await UserDB.find_one(UserDB.uid == user_uid) if user is None: if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"]: + # create user user_db = UserDB(uid="0") user = await user_db.create() - # create default organization for user - org_db = OrganizationDB(type="default", owner=str(user.id)) - org = await org_db.create() - - # update user with organization - user_db.organizations.append(org.id) - await user_db.update({"$set": user_db.dict(exclude_unset=True)}) - return user raise Exception("Please login or signup") return user @@ -1628,8 +1614,8 @@ async def create_new_evaluation( status: str, variant: AppVariantDB, evaluators_configs: List[str], - organization: OrganizationDB = None, - workspace: WorkspaceDB = None, + organization = None, + workspace = None, ) -> EvaluationDB: """Create a new evaluation scenario. Returns: @@ -1671,8 +1657,8 @@ async def create_new_evaluation_scenario( note: Optional[str], evaluators_configs: List[EvaluatorConfigDB], results: List[EvaluationScenarioResult], - organization: OrganizationDB = None, - workspace: WorkspaceDB = None, + organization = None, + workspace = None, ) -> EvaluationScenarioDB: """Create a new evaluation scenario. Returns: @@ -1809,8 +1795,8 @@ async def create_evaluator_config( user: UserDB, name: str, evaluator_key: str, - organization: OrganizationDB = None, - workspace: WorkspaceDB = None, + organization = None, + workspace = None, settings_values: Optional[Dict[str, Any]] = None, ) -> EvaluatorConfigDB: """Create a new evaluator configuration in the database.""" diff --git a/agenta-backend/agenta_backend/services/event_db_manager.py b/agenta-backend/agenta_backend/services/event_db_manager.py index 7a5d35ffdb..bc2a5b8703 100644 --- a/agenta-backend/agenta_backend/services/event_db_manager.py +++ b/agenta-backend/agenta_backend/services/event_db_manager.py @@ -34,7 +34,7 @@ async def get_variant_traces( - app_id: str, variant_id: str, **kwargs: dict + app_id: str, variant_id: str, user_uid: str ) -> List[Trace]: """Get the traces for a given app variant. @@ -46,7 +46,7 @@ async def get_variant_traces( List[Trace]: the list of traces for the given app variant """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) traces = await TraceDB.find( TraceDB.user.id == user.id, TraceDB.app_id == app_id, @@ -56,7 +56,7 @@ async def get_variant_traces( return [trace_db_to_pydantic(trace) for trace in traces] -async def create_app_trace(payload: CreateTrace, **kwargs: dict) -> str: +async def create_app_trace(payload: CreateTrace, user_uid: str) -> str: """Create a new trace. Args: @@ -66,7 +66,7 @@ async def create_app_trace(payload: CreateTrace, **kwargs: dict) -> str: Trace: the created trace """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) # Ensure spans exists in the db for span in payload.spans: @@ -79,7 +79,7 @@ async def create_app_trace(payload: CreateTrace, **kwargs: dict) -> str: return str(trace_db.id) -async def get_trace_single(trace_id: str, **kwargs: dict) -> Trace: +async def get_trace_single(trace_id: str, user_uid: str) -> Trace: """Get a single trace. Args: @@ -89,7 +89,7 @@ async def get_trace_single(trace_id: str, **kwargs: dict) -> Trace: Trace: the trace """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) # Get trace trace = await TraceDB.find_one( @@ -99,7 +99,7 @@ async def get_trace_single(trace_id: str, **kwargs: dict) -> Trace: async def trace_status_update( - trace_id: str, payload: UpdateTrace, **kwargs: dict + trace_id: str, payload: UpdateTrace, user_uid: str ) -> bool: """Update status of trace. @@ -111,7 +111,7 @@ async def trace_status_update( bool: True if successful """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) # Get trace trace = await TraceDB.find_one( @@ -124,7 +124,7 @@ async def trace_status_update( return True -async def create_trace_span(payload: CreateSpan, **kwargs: dict) -> str: +async def create_trace_span(payload: CreateSpan) -> str: """Create a new span for a given trace. Args: @@ -139,7 +139,7 @@ async def create_trace_span(payload: CreateSpan, **kwargs: dict) -> str: return str(span_db.id) -async def get_trace_spans(trace_id: str, **kwargs: dict) -> List[Span]: +async def get_trace_spans(trace_id: str, user_uid: str) -> List[Span]: """Get the spans for a given trace. Args: @@ -149,7 +149,7 @@ async def get_trace_spans(trace_id: str, **kwargs: dict) -> List[Span]: List[Span]: the list of spans for the given trace """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) # Get trace trace = await TraceDB.find_one( @@ -162,7 +162,7 @@ async def get_trace_spans(trace_id: str, **kwargs: dict) -> List[Span]: async def add_feedback_to_trace( - trace_id: str, payload: CreateFeedback, **kwargs: dict + trace_id: str, payload: CreateFeedback, user_uid: str ) -> str: """Add a feedback to a trace. @@ -174,7 +174,7 @@ async def add_feedback_to_trace( str: the feedback id """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) feedback = FeedbackDB( user_id=str(user.id), feedback=payload.feedback, @@ -193,7 +193,7 @@ async def add_feedback_to_trace( return feedback.uid -async def get_trace_feedbacks(trace_id: str, **kwargs: dict) -> List[Feedback]: +async def get_trace_feedbacks(trace_id: str, user_uid: str) -> List[Feedback]: """Get the feedbacks for a given trace. Args: @@ -203,7 +203,7 @@ async def get_trace_feedbacks(trace_id: str, **kwargs: dict) -> List[Feedback]: List[Feedback]: the list of feedbacks for the given trace """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) # Get feedbacks in trace trace = await TraceDB.find_one( @@ -214,7 +214,7 @@ async def get_trace_feedbacks(trace_id: str, **kwargs: dict) -> List[Feedback]: async def get_feedback_detail( - trace_id: str, feedback_id: str, **kwargs: dict + trace_id: str, feedback_id: str, user_uid: str ) -> Feedback: """Get a single feedback. @@ -226,7 +226,7 @@ async def get_feedback_detail( Feedback: the feedback """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) # Get trace trace = await TraceDB.find_one( @@ -243,7 +243,7 @@ async def get_feedback_detail( async def update_trace_feedback( - trace_id: str, feedback_id: str, payload: UpdateFeedback, **kwargs: dict + trace_id: str, feedback_id: str, payload: UpdateFeedback, user_uid: str ) -> Feedback: """Update a feedback. @@ -256,7 +256,7 @@ async def update_trace_feedback( Feedback: the feedback """ - user = await db_manager.get_user(user_uid=kwargs["uid"]) + user = await db_manager.get_user(user_uid) # Get trace trace = await TraceDB.find_one( diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index f3a6eb3502..ee1860e6bd 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -4,6 +4,8 @@ from fastapi.types import DecoratedCallable from fastapi import APIRouter as FastAPIRouter +from agenta_backend.services import db_manager # To fix circular import + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) From 6748ee079bbb01a746c78f6bc9ed84b0ec253414 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 19 Jan 2024 12:21:44 +0100 Subject: [PATCH 25/99] import converters --- agenta-backend/agenta_backend/services/db_manager.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 8ca3f53bdf..cab835f636 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -17,6 +17,12 @@ Template, ) +from agenta_backend.models.converters import ( + app_db_to_pydantic, + image_db_to_pydantic, + templates_db_to_pydantic, +) + from agenta_backend.models.db_models import ( Result, AggregatedResult, @@ -787,7 +793,7 @@ async def add_variant_from_base_and_config( base_db (VariantBaseDB): The existing base to use as a template for the new variant. new_config_name (str): The name of the new configuration to use for the new variant. parameters (Dict[str, Any]): The parameters to use for the new configuration. - **user_org_data (dict): Additional user and organization data. + user_uid (str): The UID of the user Returns: AppVariantDB: The newly created app variant. From d9579fa18381b9df87804e1910870401c7e83f7b Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 19 Jan 2024 12:21:54 +0100 Subject: [PATCH 26/99] import db_manager from source --- agenta-backend/agenta_backend/models/converters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 74c40e6207..5c2faad350 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -2,7 +2,7 @@ """ import json from typing import List -from agenta_backend.utils.common import db_manager +from agenta_backend.services import db_manager from agenta_backend.models.api.user_models import User from agenta_backend.models.db_models import ( AppVariantDB, From 407529465999f411b5165442a788499cdfb7ec60 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 19 Jan 2024 13:22:04 +0100 Subject: [PATCH 27/99] fix imports --- .../agenta_backend/models/converters.py | 101 +++++++++++------- .../routers/container_router.py | 4 +- .../routers/human_evaluation_router.py | 3 +- .../agenta_backend/tasks/evaluations.py | 25 +++-- 4 files changed, 87 insertions(+), 46 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 5c2faad350..6b20bec58e 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -1,57 +1,86 @@ """Converts db models to pydantic models """ +import os import json +import logging from typing import List from agenta_backend.services import db_manager from agenta_backend.models.api.user_models import User -from agenta_backend.models.db_models import ( - AppVariantDB, - EvaluationScenarioResult, - EvaluatorConfigDB, - HumanEvaluationDB, - HumanEvaluationScenarioDB, - ImageDB, - TemplateDB, - AppDB, - AppEnvironmentDB, - TestSetDB, - SpanDB, - TraceDB, - Feedback as FeedbackDB, - EvaluationDB, - EvaluationScenarioDB, - VariantBaseDB, - UserDB, - AggregatedResult, -) -from agenta_backend.models.api.api_models import ( - AppVariant, - ImageExtended, - Template, - TemplateImageInfo, - AppVariantOutput, - App, - EnvironmentOutput, - TestSetOutput, - BaseOutput, -) + from agenta_backend.models.api.observability_models import ( Span, Trace, Feedback as FeedbackOutput, ) from agenta_backend.models.api.evaluation_model import ( - HumanEvaluation, - HumanEvaluationScenario, - SimpleEvaluationOutput, - EvaluationScenario, Evaluation, + HumanEvaluation, EvaluatorConfig, + EvaluationScenario, + SimpleEvaluationOutput, EvaluationScenarioInput, + HumanEvaluationScenario, EvaluationScenarioOutput, ) -import logging +FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import ( + AppDB_ as AppDB, + UserDB_ as UserDB, + ImageDB_ as ImageDB, + TestSetDB_ as TestSetDB, + EvaluationDB_ as EvaluationDB, + AppVariantDB_ as AppVariantDB_, + VariantBaseDB_ as VariantBaseDB, + AppEnvironmentDB_ as AppEnvironmentDB, + EvaluatorConfigDB_ as EvaluatorConfigDB, + HumanEvaluationDB_ as HumanEvaluationDB, + EvaluationScenarioDB_ as EvaluationScenarioDB, + HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, + ) + from agenta_backend.commons.models.api.api_models import ( + AppVariant_ as AppVariant, + ImageExtended_ as ImageExtended, + AppVariantOutput_ as AppVariantOutput, + ) +else: + from agenta_backend.models.db_models import ( + AppDB, + UserDB, + ImageDB, + TestSetDB, + EvaluationDB, + AppVariantDB, + VariantBaseDB, + AppEnvironmentDB, + EvaluatorConfigDB, + HumanEvaluationDB, + EvaluationScenarioDB, + HumanEvaluationScenarioDB, + ) + from agenta_backend.models.api.api_models import ( + AppVariant, + ImageExtended, + AppVariantOutput, + ) + +from agenta_backend.models.db_models import ( + SpanDB, + TraceDB, + TemplateDB, + AggregatedResult, + Feedback as FeedbackDB, + EvaluationScenarioResult, +) +from agenta_backend.models.api.api_models import ( + App, + Template, + BaseOutput, + TestSetOutput, + TemplateImageInfo, + EnvironmentOutput, +) logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 7ce307be1b..c6eb06ae62 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -12,6 +12,9 @@ if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.models.db_models import WorkspaceRole, Permission from agenta_backend.commons.utils.permissions import check_action_access + from agenta_backend.models.api.api_models import Image_ as Image +else: + from agenta_backend.models.api.api_models import Image_ as Image if FEATURE_FLAG in ["cloud"]: @@ -23,7 +26,6 @@ from agenta_backend.models.api.api_models import ( URI, - Image, RestartAppContainer, Template, ) diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index f931f10254..111b535908 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -4,7 +4,8 @@ from fastapi.responses import JSONResponse from fastapi.encoders import jsonable_encoder -from fastapi import HTTPException, APIRouter, Body, Request, status, Response +from agenta_backend.utils.common import APIRouter +from fastapi import HTTPException, Body, Request, status, Response from agenta_backend.models import converters from agenta_backend.services import db_manager diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index 9ed8131e05..a0a421403e 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -7,14 +7,6 @@ from agenta_backend.models.api.evaluation_model import AppOutput, NewEvaluation from agenta_backend.models.db_engine import DBEngine -from agenta_backend.models.db_models import ( - AggregatedResult, - AppDB, - EvaluationScenarioInputDB, - EvaluationScenarioOutputDB, - EvaluationScenarioResult, - Result, -) from agenta_backend.services import evaluators_service, llm_apps_service from agenta_backend.services.db_manager import ( create_new_evaluation_scenario, @@ -31,6 +23,23 @@ from celery import shared_task, states FEATURE_FLAG = os.environ["FEATURE_FLAG"] +if FEATURE_FLAG in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import ( + AppDB_ as AppDB, + EvaluationScenarioOutputDB_ as EvaluationScenarioOutputDB, + ) +else: + from agenta_backend.models.db_models import ( + AppDB, + EvaluationScenarioOutputDB, + ) + + from agenta_backend.models.db_models import ( + AggregatedResult, + EvaluationScenarioInputDB, + EvaluationScenarioResult, + Result, + ) # Set logger logger = logging.getLogger(__name__) From 62b41fc9b57072327e3df0f4b076e552d6b59e6f Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 19 Jan 2024 13:42:37 +0100 Subject: [PATCH 28/99] fix circular imports --- .../routers/container_router.py | 2 +- .../agenta_backend/services/db_manager.py | 30 ++++--------------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index c6eb06ae62..72d8fd71da 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -14,7 +14,7 @@ from agenta_backend.commons.utils.permissions import check_action_access from agenta_backend.models.api.api_models import Image_ as Image else: - from agenta_backend.models.api.api_models import Image_ as Image + from agenta_backend.models.api.api_models import Image if FEATURE_FLAG in ["cloud"]: diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 6d63c76efe..2ede925894 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -17,11 +17,7 @@ Template, ) -from agenta_backend.models.converters import ( - app_db_to_pydantic, - image_db_to_pydantic, - templates_db_to_pydantic, -) +from agenta_backend.models import converters from agenta_backend.models.db_models import ( Result, @@ -79,20 +75,6 @@ EvaluationScenarioDB, HumanEvaluationScenarioDB, ) - -if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.services import db_manager_ee - from agenta_backend.commons.services.selectors import ( - get_user_and_org_id, - ) # noqa pylint: disable-all - from agenta_backend.commons.utils.permissions import ( - check_action_access, - check_rbac_permission - ) - from agenta_backend.commons.models.db_models import ( - Permission, - WorkspaceRole - ) from beanie.operators import In from beanie import PydanticObjectId as ObjectId @@ -190,7 +172,7 @@ async def get_image(app_variant: AppVariant, **kwargs: dict) -> ImageExtended: db_app_variant = await AppVariantDB.find_one(query_expression) if db_app_variant: image_db = await ImageDB.find_one(ImageDB.id == db_app_variant.image.id) - return image_db_to_pydantic(image_db) + return converters.image_db_to_pydantic(image_db) else: raise Exception("App variant not found") @@ -865,7 +847,7 @@ async def list_apps( app_db = await fetch_app_by_name_and_parameters( app_name=app_name, user_uid=user_uid, organization_id=org_id, workspace_id=workspace_id ) - return [app_db_to_pydantic(app_db)] + return [converters.app_db_to_pydantic(app_db)] # elif (org_id is not None) or (workspace_id is not None): # TODO: Remember to enable this when workspace_id is provided after the RBAC is implemented @@ -899,11 +881,11 @@ async def list_apps( # AppDB.organization.id == ObjectId(org_id), # AppDB.workspace.id == ObjectId(workspace_id), # ).to_list() - # return [app_db_to_pydantic(app) for app in apps] + # return [converters.app_db_to_pydantic(app) for app in apps] else: apps = await AppDB.find(AppDB.user.id == user.id).to_list() - return [app_db_to_pydantic(app) for app in apps] + return [converters.app_db_to_pydantic(app) for app in apps] async def list_app_variants(app_id: str) -> List[AppVariantDB]: @@ -1534,7 +1516,7 @@ async def remove_old_template_from_db(tag_ids: list) -> None: async def get_templates() -> List[Template]: templates = await TemplateDB.find().to_list() - return templates_db_to_pydantic(templates) + return converters.templates_db_to_pydantic(templates) async def update_base( From d1151375c185796d7ebfc12acf0eef1a5d5ac62d Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Fri, 19 Jan 2024 22:03:30 +0100 Subject: [PATCH 29/99] update docstring --- agenta-backend/agenta_backend/services/db_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 2ede925894..59e0097164 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -101,7 +101,7 @@ async def add_testset_to_app_variant( org_id (str): The id of the organization template_name (str): The name of the app template image app_name (str): The name of the app - **kwargs (dict): Additional keyword arguments + user_uid (str): The uid of the user """ try: From 3215d40169d965a4a334a959381f8a02c297dfc4 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Mon, 22 Jan 2024 16:30:38 +0100 Subject: [PATCH 30/99] fix bugs with imports and naming --- .../agenta_backend/models/converters.py | 2 +- .../agenta_backend/models/db_engine.py | 50 +++++----- .../agenta_backend/routers/app_router.py | 55 ++++++----- .../agenta_backend/routers/bases_router.py | 8 +- .../agenta_backend/routers/configs_router.py | 6 +- .../routers/container_router.py | 8 +- .../routers/environment_router.py | 2 +- .../routers/evaluation_router.py | 16 ++-- .../routers/evaluators_router.py | 2 +- .../routers/human_evaluation_router.py | 6 +- .../agenta_backend/routers/testset_router.py | 14 +-- .../agenta_backend/routers/variants_router.py | 12 +-- .../agenta_backend/services/db_manager.py | 91 ++++++------------- .../services/evaluation_service.py | 1 - .../services/evaluator_manager.py | 2 +- .../services/results_service.py | 2 +- .../agenta_backend/services/user_service.py | 7 +- .../agenta_backend/tasks/evaluations.py | 26 ++---- 18 files changed, 138 insertions(+), 172 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 6b20bec58e..6a8565306a 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -31,7 +31,7 @@ ImageDB_ as ImageDB, TestSetDB_ as TestSetDB, EvaluationDB_ as EvaluationDB, - AppVariantDB_ as AppVariantDB_, + AppVariantDB_ as AppVariantDB, VariantBaseDB_ as VariantBaseDB, AppEnvironmentDB_ as AppEnvironmentDB, EvaluatorConfigDB_ as EvaluatorConfigDB, diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index dbf4c3b3af..9050de120b 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -9,65 +9,65 @@ if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: from agenta_backend.commons.models.db_models import ( APIKeyDB, - OrganizationDB_ as OrganizationDB, WorkspaceDB, - AppEnvironmentDB_ as AppEnvironmentDB, + OrganizationDB, + AppDB_ as AppDB, UserDB_ as UserDB, ImageDB_ as ImageDB, - AppDB_ as AppDB, + TestSetDB_ as TestSetDB, + AppVariantDB_ as AppVariantDB, + EvaluationDB_ as EvaluationDB, DeploymentDB_ as DeploymentDB, VariantBaseDB_ as VariantBaseDB, - AppVariantDB_ as AppVariantDB, - TestSetDB_ as TestSetDB, + AppEnvironmentDB_ as AppEnvironmentDB, EvaluatorConfigDB_ as EvaluatorConfigDB, HumanEvaluationDB_ as HumanEvaluationDB, - HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, - EvaluationDB_ as EvaluationDB, EvaluationScenarioDB_ as EvaluationScenarioDB, + HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, ) else: from agenta_backend.models.db_models import ( - AppEnvironmentDB, + AppDB, UserDB, ImageDB, - AppDB, + TestSetDB, + EvaluationDB, DeploymentDB, - VariantBaseDB, AppVariantDB, - TestSetDB, + VariantBaseDB, + AppEnvironmentDB, EvaluatorConfigDB, HumanEvaluationDB, - HumanEvaluationScenarioDB, - EvaluationDB, EvaluationScenarioDB, + HumanEvaluationScenarioDB, ) from agenta_backend.models.db_models import ( - TemplateDB, - ConfigDB, SpanDB, TraceDB, + ConfigDB, + TemplateDB, ) # Define Document Models document_models: List[Document] = [ - AppEnvironmentDB, + AppDB, UserDB, + SpanDB, + TraceDB, ImageDB, - AppDB, - DeploymentDB, - VariantBaseDB, ConfigDB, - AppVariantDB, - TemplateDB, TestSetDB, + TemplateDB, + AppVariantDB, + DeploymentDB, + EvaluationDB, + VariantBaseDB, + AppEnvironmentDB, EvaluatorConfigDB, HumanEvaluationDB, - HumanEvaluationScenarioDB, - EvaluationDB, EvaluationScenarioDB, - SpanDB, - TraceDB, + HumanEvaluationScenarioDB, ] if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 858ff666ff..4a28d8facc 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -24,7 +24,7 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.models.api.api_models import ( - ImageDB_ as Image, + Image_ as Image, CreateApp_ as CreateApp, AppVariantOutput_ as AppVariantOutput, CreateAppVariant_ as CreateAppVariant, @@ -91,10 +91,10 @@ async def list_app_variants( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", - role=WorkspaceRole.VIEWER, + permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to list app variants: {has_permission}") if not has_permission: @@ -141,10 +141,10 @@ async def get_variant_by_env( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", - role=WorkspaceRole.VIEWER, + permission=Permission.VIEW_APPLICATION, ) logger.debug(f"user has Permission to get variant by environment: {has_permission}") if not has_permission: @@ -275,9 +275,12 @@ async def list_apps( Raises: HTTPException: If there was an error retrieving the list of apps. """ - try: + try: apps = await db_manager.list_apps( - app_name, org_id, workspace_id, request.state.user_id + app_name=app_name, + user_uid=request.state.user_id, + org_id=org_id, + workspace_id=workspace_id ) return apps except Exception as e: @@ -324,10 +327,10 @@ async def add_variant_from_image( if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", - role=WorkspaceRole.VIEWER, + permission=Permission.CREATE_APPLICATION, ) logger.debug(f"User has Permission to create app from image: {has_permission}") if not has_permission: @@ -370,7 +373,7 @@ async def remove_app(app_id: str, request: Request): if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", permission=Permission.DELETE_APPLICATION, @@ -421,11 +424,11 @@ async def create_app_and_variant_from_template( # Get user and org id logger.debug("Step 1: Getting user and organization ID") - user_org_data: dict = await get_user_org_and_workspace_id(request.state.user_id) + user_org_workspace_data: dict = await get_user_org_and_workspace_id(request.state.user_id) logger.debug("Step 2: Setting organization ID") if payload.organization_id is None: - organization = await get_user_own_org(user_org_data["uid"]) + organization = await get_user_own_org(user_org_workspace_data["uid"]) organization_id = str(organization.id) else: organization_id = payload.organization_id @@ -440,9 +443,9 @@ async def create_app_and_variant_from_template( logger.debug("Step 4: Checking user has permission to create app") has_permission = await check_rbac_permission( - user_org_data=user_org_data, - workspace_id=ObjectId(workspace_id), - organization_id=ObjectId(organization_id), + user_org_workspace_data=user_org_workspace_data, + workspace=workspace, + organization=organization, permission=Permission.CREATE_APPLICATION, ) logger.debug(f"User has Permission to create app from template: {has_permission}") @@ -454,7 +457,7 @@ async def create_app_and_variant_from_template( status_code=403, ) - logger.debug(f"Step 5: Checking if app {payload.app_name} already exists" if FEATURE_FLAG in ["cloud", "ee"] else f"Step 1: Checking if app {payload.app_name} already exists") + print(f"Step 5: Checking if app {payload.app_name} already exists" if FEATURE_FLAG in ["cloud", "ee"] else f"Step 1: Checking if app {payload.app_name} already exists") app_name = payload.app_name.lower() app = await db_manager.fetch_app_by_name_and_parameters( app_name, @@ -467,8 +470,9 @@ async def create_app_and_variant_from_template( f"App with name {app_name} already exists", ) - logger.debug("Step 6: Creating new app and initializing environments" if FEATURE_FLAG in ["cloud", "ee"] else "Step 2: Creating new app and initializing environments") + print("Step 6: Creating new app and initializing environments" if FEATURE_FLAG in ["cloud", "ee"] else "Step 2: Creating new app and initializing environments") if app is None: + print("Here next step") app = await db_manager.create_app_and_envs( app_name, request.state.user_id, @@ -476,12 +480,12 @@ async def create_app_and_variant_from_template( workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, ) - logger.debug("Step 7: Retrieve template from db" if FEATURE_FLAG in ["cloud", "ee"] else "Step 3: Retrieve template from db") + print("Step 7: Retrieve template from db" if FEATURE_FLAG in ["cloud", "ee"] else "Step 3: Retrieve template from db") template_db = await db_manager.get_template(payload.template_id) repo_name = os.environ.get("AGENTA_TEMPLATE_REPO", "agentaai/templates_v2") image_name = f"{repo_name}:{template_db.name}" - logger.debug( + print( "Step 8: Creating image instance and adding variant based on image" if FEATURE_FLAG in ["cloud", "ee"] else "Step 4: Creating image instance and adding variant based on image" ) app_variant_db = await app_manager.add_variant_based_on_image( @@ -496,9 +500,10 @@ async def create_app_and_variant_from_template( base_name="app", config_name="default", is_template_image=True, + user_uid=request.state.user_id ) - logger.debug("Step 9: Creating testset for app variant" if FEATURE_FLAG in ["cloud", "ee"] else "Step 5: Creating testset for app variant") + print("Step 9: Creating testset for app variant" if FEATURE_FLAG in ["cloud", "ee"] else "Step 5: Creating testset for app variant") await db_manager.add_testset_to_app_variant( app_id=str(app.id), org_id=organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, @@ -508,10 +513,10 @@ async def create_app_and_variant_from_template( user_uid=request.state.user_id, ) - logger.debug("Step 10: We create ready-to use evaluators" if FEATURE_FLAG in ["cloud", "ee"] else "Step 6: We create ready-to use evaluators") + print("Step 10: We create ready-to use evaluators" if FEATURE_FLAG in ["cloud", "ee"] else "Step 6: We create ready-to use evaluators") await evaluator_manager.create_ready_to_use_evaluators(app=app) - logger.debug("Step 11: Starting variant and injecting environment variables" if FEATURE_FLAG in ["cloud", "ee"] else "Step 7: Starting variant and injecting environment variables") + print("Step 11: Starting variant and injecting environment variables" if FEATURE_FLAG in ["cloud", "ee"] else "Step 7: Starting variant and injecting environment variables") if FEATURE_FLAG in ["cloud", "ee"]: if not os.environ["OPENAI_API_KEY"]: raise Exception( @@ -526,7 +531,7 @@ async def create_app_and_variant_from_template( else: envvars = {} if payload.env_vars is None else payload.env_vars - await app_manager.start_variant(app_variant_db, envvars, **user_org_data) + await app_manager.start_variant(app_variant_db, envvars) logger.debug("End: Successfully created app and variant") return await converters.app_variant_db_to_output(app_variant_db) @@ -560,9 +565,9 @@ async def list_environments( if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( user_id=request.state.user_id, - object_id=app_id, + object_uid=app_id, object_type="app", - role=WorkspaceRole.VIEWER, + permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to list environments: {has_permission}") if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index 4a13c6caed..8f6cf24e4c 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -12,8 +12,8 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.models.db_models import WorkspaceRole - from agenta_backend.cmmons.utils.permissions import check_action_access + from agenta_backend.commons.models.db_models import Permission + from agenta_backend.commons.utils.permissions import check_action_access router = APIRouter() @@ -44,10 +44,10 @@ async def list_bases( try: if os.environ["FEATURE_FLAG"] in ["cloud", "ee"] and app_id is not None: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", - role=WorkspaceRole.VIEWER, + permission=Permission.VIEW_APPLICATION, ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index d662575343..f471835e54 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -18,7 +18,7 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.models.db_models import Permission - from agenta_backend.cmmons.utils.permissions import check_action_access + from agenta_backend.commons.utils.permissions import check_action_access router = APIRouter() @@ -34,7 +34,7 @@ async def save_config( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=payload.base_id, object_type="base", permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, @@ -95,7 +95,7 @@ async def get_config( # detemine whether the user has access to the base if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=base_id, object_type="base", permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 72d8fd71da..51db0a9178 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -12,7 +12,7 @@ if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.models.db_models import WorkspaceRole, Permission from agenta_backend.commons.utils.permissions import check_action_access - from agenta_backend.models.api.api_models import Image_ as Image + from agenta_backend.commons.models.api.api_models import Image_ as Image else: from agenta_backend.models.api.api_models import Image @@ -59,7 +59,7 @@ async def build_image( """ if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", permission=Permission.CREATE_APPLICATION, @@ -155,10 +155,10 @@ async def construct_app_container_url( if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=base_id if base_id else variant_id, object_type="base" if base_id else "app_variant", - role=WorkspaceRole.VIEWER, + permission=Permission.VIEW_APPLICATION, ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 08d486c66f..4feb553dae 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -36,7 +36,7 @@ async def deploy_to_environment( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=payload.variant_id, object_type="app_variant", permission=Permission.DEPLOY_APPLICATION, diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 62c8e8447f..671d491371 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -47,7 +47,7 @@ async def create_evaluation( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=payload.app_id, object_type="app", permission=Permission.CREATE_EVALUATION, @@ -115,7 +115,7 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", permission=Permission.VIEW_EVALUATION, @@ -150,7 +150,7 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", permission=Permission.VIEW_EVALUATION, @@ -194,7 +194,7 @@ async def fetch_evaluation_scenarios( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", permission=Permission.VIEW_EVALUATION, @@ -231,7 +231,7 @@ async def fetch_list_evaluations( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", permission=Permission.VIEW_EVALUATION, @@ -268,7 +268,7 @@ async def fetch_evaluation( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", permission=Permission.VIEW_EVALUATION, @@ -306,7 +306,7 @@ async def delete_evaluations( if FEATURE_FLAG in ["cloud", "ee"]: for evaluation_id in delete_evaluations.evaluations_ids: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", permission=Permission.VIEW_EVALUATION, @@ -369,7 +369,7 @@ async def fetch_evaluation_scenarios( if FEATURE_FLAG in ["cloud", "ee"]: for evaluation_id in evaluations_ids_list: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", permission=Permission.VIEW_EVALUATION, diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index ce0e372fc5..5b91fb5981 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -79,7 +79,7 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=evaluator_config_id, object_type="evaluator_config", permission=Permission.VIEW_EVALUATION, diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 111b535908..627fc01b8a 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -114,7 +114,7 @@ async def fetch_list_human_evaluations( user_uid=request.state.user_id, object_id=app_id, object_type="app", - permissin=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -147,7 +147,7 @@ async def fetch_human_evaluation( user_uid=request.state.user_id, object_id=evaluation_id, object_type="human_evaluation", - permissin=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -188,7 +188,7 @@ async def fetch_evaluation_scenarios( user_uid=request.state.user_id, object_id=evaluation_id, object_type="human_evaluation_scenario_by_evaluation_id", - permissin=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index a19c57c245..cfd8936ba8 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -67,7 +67,7 @@ async def upload_file( if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", permission=Permission.CREATE_TESTSET, @@ -151,7 +151,7 @@ async def import_testset( """ if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", permission=Permission.CREATE_TESTSET, @@ -238,7 +238,7 @@ async def create_testset( if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", permission=Permission.CREATE_TESTSET, @@ -297,7 +297,7 @@ async def update_testset( """ if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=testset_id, object_type="testset", permission=Permission.EDIT_TESTSET, @@ -352,7 +352,7 @@ async def get_testsets( """ if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=app_id, object_type="app", permission=Permission.VIEW_TESTSET, @@ -398,7 +398,7 @@ async def get_single_testset( """ if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=testset_id, object_type="testset", permission=Permission.VIEW_TESTSET, @@ -436,7 +436,7 @@ async def delete_testsets( if FEATURE_FLAG in ["cloud", "ee"]: for testset_id in delete_testsets.testset_ids: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=testset_id, object_type="testset", permission=Permission.VIEW_TESTSET, diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 0cd204a3e1..6e4f0e662d 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -17,7 +17,7 @@ if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all - from agenta_backend.models.api.api_models import ( + from agenta_backend.commons.models.api.api_models import ( Image_ as Image, AppVariantOutput_ as AppVariantOutput, ) @@ -66,7 +66,7 @@ async def add_variant_from_base_and_config( # Check user has permission to add variant if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=payload.base_id, object_type="base", permission=Permission.CREATE_APPLICATION, @@ -120,7 +120,7 @@ async def remove_variant( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=variant_id, object_type="app_variant", permission=Permission.DELETE_APPLICATION_VARIANT, @@ -166,7 +166,7 @@ async def update_variant_parameters( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=variant_id, object_type="app_variant", permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, @@ -214,7 +214,7 @@ async def update_variant_image( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=variant_id, object_type="app_variant", permission=Permission.CREATE_APPLICATION, @@ -270,7 +270,7 @@ async def start_variant( # Check user has permission to start variant if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, + user_uid=request.state.user_id, object_id=variant_id, object_type="app_variant", permission=Permission.CREATE_APPLICATION, diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 2ede925894..31bcc60222 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -21,13 +21,14 @@ from agenta_backend.models.db_models import ( Result, + ConfigDB, + TemplateDB, + DeploymentDB, + ConfigVersionDB, AggregatedResult, + EvaluationScenarioResult, EvaluationScenarioInputDB, EvaluationScenarioOutputDB, - EvaluationScenarioResult, - ConfigDB, - ConfigVersionDB, - TemplateDB, ) FEATURE_FLAG = os.environ["FEATURE_FLAG"] @@ -46,7 +47,6 @@ TestSetDB_ as TestSetDB, AppVariantDB_ as AppVariantDB, EvaluationDB_ as EvaluationDB, - DeploymentDB_ as DeploymentDB, VariantBaseDB_ as VariantBaseDB, AppEnvironmentDB_ as AppEnvironmentDB, EvaluatorConfigDB_ as EvaluatorConfigDB, @@ -67,7 +67,6 @@ TestSetDB, AppVariantDB, EvaluationDB, - DeploymentDB, VariantBaseDB, AppEnvironmentDB, EvaluatorConfigDB, @@ -147,36 +146,6 @@ async def add_testset_to_app_variant( print(f"An error occurred in adding the default testset: {e}") -async def get_image(app_variant: AppVariant, **kwargs: dict) -> ImageExtended: - """Returns the image associated with the app variant - - Arguments: - app_variant -- AppVariant to fetch the image for - - Returns: - Image -- The Image associated with the app variant - """ - - # Build the query expression for the two conditions - query_expression = ( - AppVariantDB.app.id == app_variant.app_id, - AppVariantDB.variant_name == app_variant.variant_name, - ) - - if FEATURE_FLAG in ["cloud", "ee"]: - query_expression += ( - AppVariantDB.organization.id == app_variant.organization, - AppVariantDB.workspace.id == app_variant.workspace, - ) - - db_app_variant = await AppVariantDB.find_one(query_expression) - if db_app_variant: - image_db = await ImageDB.find_one(ImageDB.id == db_app_variant.image.id) - return converters.image_db_to_pydantic(image_db) - else: - raise Exception("App variant not found") - - async def get_image_by_id(image_id: str) -> ImageDB: """Get the image object from the database with the provided id. @@ -693,9 +662,7 @@ async def get_orga_image_instance_by_docker_id( ImageDB: instance of image object """ - query_expression = ( - ImageDB.docker_id == docker_id, - ) + query_expression = {'docker_id': docker_id} if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa @@ -703,10 +670,10 @@ async def get_orga_image_instance_by_docker_id( organization_id is not None and workspace_id is not None ), "organization and workspace must be provided together" - query_expression += ( - ImageDB.organization.id == ObjectId(organization_id), - ImageDB.workspace.id == ObjectId(workspace_id), - ) + query_expression.update({ + 'organization.id': ObjectId(organization_id), + 'workspace.id': ObjectId(workspace_id), + }) image = await ImageDB.find_one(query_expression) return image @@ -729,9 +696,7 @@ async def get_orga_image_instance_by_uri( if not parsed_url.scheme and not parsed_url.netloc: raise ValueError(f"Invalid URL: {template_uri}") - query_expression = ( - ImageDB.template_uri == template_uri, - ) + query_expression = {'template_uri': template_uri} if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa @@ -739,10 +704,10 @@ async def get_orga_image_instance_by_uri( organization_id is not None and workspace_id is not None ), "organization and workspace must be provided together" - query_expression += ( - ImageDB.organization.id == ObjectId(organization_id), - ImageDB.workspace.id == ObjectId(workspace_id), - ) + query_expression.update({ + 'organization.id': ObjectId(organization_id), + 'workspace.id': ObjectId(workspace_id), + }) image = await ImageDB.find_one(query_expression) return image @@ -862,12 +827,12 @@ async def list_apps( # org_id is not None and workspace_id is not None # ), "org_id and workspace_id must be provided together" - # user_org_data = await get_user_and_org_id(user_uid) + # user_org_workspace_data = await get_user_org_and_workspace_id(user_uid) # has_permission = await check_rbac_permission( - # user_org_data=user_org_data, + # user_org_workspace_data=user_org_workspace_data, # workspace_id=ObjectId(workspace_id), # organization_id=ObjectId(org_id), - # permission=Permission.CREATE_APPLICATION, + # permission=Permission.VIEW_APPLICATION, # ) # logger.debug(f"User has Permission to list apps: {has_permission}") # if not has_permission: @@ -1560,7 +1525,7 @@ async def fetch_app_by_name_and_parameters( organization_id: str = None, workspace_id: str = None, ): - """Fetch an app by it's name, organization id and workspace id. + """Fetch an app by its name, organization id, and workspace id. Args: app_name (str): The name of the app @@ -1571,9 +1536,7 @@ async def fetch_app_by_name_and_parameters( AppDB: the instance of the app """ - query_expression = ( - AppDB.app_name == app_name, - ) + query_expression = {'app_name': app_name} if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa @@ -1581,14 +1544,14 @@ async def fetch_app_by_name_and_parameters( organization_id is not None and workspace_id is not None ), "organization_id and workspace_id must be provided together" - query_expression += ( - AppDB.organization.id == ObjectId(organization_id), - AppDB.workspace.id == ObjectId(workspace_id), - ) + query_expression.update({ + 'organization.id': ObjectId(organization_id), + 'workspace.id': ObjectId(workspace_id), + }) else: - query_expression += ( - AppDB.user.id == ObjectId(user_uid), - ) + query_expression.update({ + 'user.id': ObjectId(user_uid), + }) app_db = await AppDB.find_one(query_expression, fetch_links=True) diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 589d525798..7401d63105 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -25,7 +25,6 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.utils.permissions import check_access_to_app from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, UserDB_ as UserDB, diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 382b5eaa38..51126450fd 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -8,7 +8,7 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.db_models import ( + from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, EvaluatorConfigDB_ as EvaluatorConfigDB ) diff --git a/agenta-backend/agenta_backend/services/results_service.py b/agenta-backend/agenta_backend/services/results_service.py index 9b0637aa89..c6e3d1bca8 100644 --- a/agenta-backend/agenta_backend/services/results_service.py +++ b/agenta-backend/agenta_backend/services/results_service.py @@ -2,7 +2,7 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.models.db_models import ( + from agenta_backend.commons.models.db_models import ( HumanEvaluationDB_ as HumanEvaluationDB, EvaluationScenarioDB_ as EvaluationScenarioDB, HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, diff --git a/agenta-backend/agenta_backend/services/user_service.py b/agenta-backend/agenta_backend/services/user_service.py index b1761598fd..bf452b316c 100644 --- a/agenta-backend/agenta_backend/services/user_service.py +++ b/agenta-backend/agenta_backend/services/user_service.py @@ -1,4 +1,9 @@ -from agenta_backend.models.db_models import UserDB +import os + +if os.environ["FEATURE_FLAG"] in ["cloud"]: + from agenta_backend.commons.models.db_models import UserDB_ as UserDB +else: + from agenta_backend.models.db_models import UserDB from agenta_backend.models.api.user_models import User, UserUpdate diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index a0a421403e..e30f2502e1 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -22,24 +22,18 @@ ) from celery import shared_task, states -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.models.db_models import ( - AppDB_ as AppDB, - EvaluationScenarioOutputDB_ as EvaluationScenarioOutputDB, - ) +if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: + from agenta_backend.commons.models.db_models import AppDB_ as AppDB else: - from agenta_backend.models.db_models import ( - AppDB, - EvaluationScenarioOutputDB, - ) + from agenta_backend.models.db_models import AppDB - from agenta_backend.models.db_models import ( - AggregatedResult, - EvaluationScenarioInputDB, - EvaluationScenarioResult, - Result, - ) +from agenta_backend.models.db_models import ( + Result, + AggregatedResult, + EvaluationScenarioResult, + EvaluationScenarioInputDB, + EvaluationScenarioOutputDB, +) # Set logger logger = logging.getLogger(__name__) From 220399049bfe0b3c3d7d0085cca92d45ae3b22ed Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 23 Jan 2024 12:42:11 +0100 Subject: [PATCH 31/99] Change Output to Response in api Models --- .../agenta_backend/models/api/api_models.py | 2 +- .../agenta_backend/models/converters.py | 8 ++++---- .../agenta_backend/routers/app_router.py | 16 ++++++++-------- .../agenta_backend/routers/variants_router.py | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index 1d68dbf79e..6984e29ebb 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -64,7 +64,7 @@ class AppVariantFromImagePayload(BaseModel): variant_name: str -class AppVariantOutput(BaseModel): +class AppVariantResponse(BaseModel): app_id: str app_name: str variant_id: str diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 6a8565306a..54d004e7d5 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -42,7 +42,7 @@ from agenta_backend.commons.models.api.api_models import ( AppVariant_ as AppVariant, ImageExtended_ as ImageExtended, - AppVariantOutput_ as AppVariantOutput, + AppVariantResponse_ as AppVariantResponse, ) else: from agenta_backend.models.db_models import ( @@ -62,7 +62,7 @@ from agenta_backend.models.api.api_models import ( AppVariant, ImageExtended, - AppVariantOutput, + AppVariantResponse, ) from agenta_backend.models.db_models import ( @@ -234,7 +234,7 @@ def app_variant_db_to_pydantic( ) -async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantOutput: +async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantResponse: if app_variant_db.base.deployment: deployment = await db_manager.get_deployment_by_objectid( app_variant_db.base.deployment @@ -244,7 +244,7 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantOu deployment = None uri = None logger.info(f"uri: {uri} deployment: {app_variant_db.base.deployment} {deployment}") - return AppVariantOutput( + return AppVariantResponse( app_id=str(app_variant_db.app.id), app_name=str(app_variant_db.app.app_name), variant_name=app_variant_db.variant_name, diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 4a28d8facc..b89579336e 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -26,14 +26,14 @@ from agenta_backend.commons.models.api.api_models import ( Image_ as Image, CreateApp_ as CreateApp, - AppVariantOutput_ as AppVariantOutput, + AppVariantResponse_ as AppVariantResponse, CreateAppVariant_ as CreateAppVariant, ) else: from agenta_backend.models.api.api_models import ( Image, CreateApp, - AppVariantOutput, + AppVariantResponse, CreateAppVariant, ) if FEATURE_FLAG in ["cloud", "ee"]: @@ -71,7 +71,7 @@ @router.get( "/{app_id}/variants/", - response_model=List[AppVariantOutput], + response_model=List[AppVariantResponse], operation_id="list_app_variants", ) async def list_app_variants( @@ -86,7 +86,7 @@ async def list_app_variants( stoken_session (SessionContainer, optional): The session container to verify the user's session. Defaults to Depends(verify_session()). Returns: - List[AppVariantOutput]: A list of app variants for the given app ID. + List[AppVariantResponse]: A list of app variants for the given app ID. """ try: if FEATURE_FLAG in ["cloud", "ee"]: @@ -116,7 +116,7 @@ async def list_app_variants( @router.get( "/get_variant_by_env/", - response_model=AppVariantOutput, + response_model=AppVariantResponse, operation_id="get_variant_by_env", ) async def get_variant_by_env( @@ -136,7 +136,7 @@ async def get_variant_by_env( HTTPException: If the app variant is not found (status_code=500), or if a ValueError is raised (status_code=400), or if any other exception is raised (status_code=500). Returns: - AppVariantOutput: The retrieved app variant. + AppVariantResponse: The retrieved app variant. """ try: if FEATURE_FLAG in ["cloud", "ee"]: @@ -403,7 +403,7 @@ async def remove_app(app_id: str, request: Request): async def create_app_and_variant_from_template( payload: CreateAppVariant, request: Request, -) -> AppVariantOutput: +) -> AppVariantResponse: """ Create an app and variant from a template. @@ -415,7 +415,7 @@ async def create_app_and_variant_from_template( HTTPException: If the user has reached the app limit or if an app with the same name already exists. Returns: - AppVariantOutput: The output of the created app variant. + AppVariantResponse: The output of the created app variant. """ try: logger.debug("Start: Creating app and variant from template") diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 6e4f0e662d..bc585f2f93 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -19,12 +19,12 @@ from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all from agenta_backend.commons.models.api.api_models import ( Image_ as Image, - AppVariantOutput_ as AppVariantOutput, + AppVariantResponse_ as AppVariantResponse, ) else: from agenta_backend.models.api.api_models import ( Image, - AppVariantOutput, + AppVariantResponse, ) from agenta_backend.models.api.api_models import ( @@ -45,7 +45,7 @@ async def add_variant_from_base_and_config( payload: AddVariantFromBasePayload, request: Request, -) -> Union[AppVariantOutput, Any]: +) -> Union[AppVariantResponse, Any]: """Add a new variant based on an existing one. Same as POST /config @@ -57,7 +57,7 @@ async def add_variant_from_base_and_config( HTTPException: Raised if the variant could not be added or accessed. Returns: - Union[AppVariantOutput, Any]: New variant details or exception. + Union[AppVariantResponse, Any]: New variant details or exception. """ try: logger.debug("Initiating process to add a variant based on a previous one.") From d85ea2c64bbaa04a79bd08ad730907dc8cf2951c Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 23 Jan 2024 12:45:48 +0100 Subject: [PATCH 32/99] Ran format --- .../agenta_backend/models/db_engine.py | 2 +- .../agenta_backend/routers/app_router.py | 108 ++++++---- .../agenta_backend/routers/bases_router.py | 2 +- .../agenta_backend/routers/configs_router.py | 4 +- .../routers/container_router.py | 6 +- .../routers/environment_router.py | 2 +- .../routers/evaluation_router.py | 56 +++-- .../routers/evaluators_router.py | 6 +- .../routers/human_evaluation_router.py | 62 +++--- .../routers/observability_router.py | 4 +- .../agenta_backend/routers/testset_router.py | 36 ++-- .../agenta_backend/routers/variants_router.py | 32 ++- .../agenta_backend/services/app_manager.py | 71 ++++--- .../services/container_manager.py | 4 +- .../agenta_backend/services/db_manager.py | 192 +++++++++--------- .../services/deployment_manager.py | 4 +- .../services/evaluation_service.py | 36 ++-- .../services/evaluator_manager.py | 16 +- .../agenta_backend/tasks/evaluations.py | 10 +- 19 files changed, 370 insertions(+), 283 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index 9050de120b..778c630958 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -41,7 +41,7 @@ EvaluationScenarioDB, HumanEvaluationScenarioDB, ) - + from agenta_backend.models.db_models import ( SpanDB, TraceDB, diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index b89579336e..ec0d7cb4b3 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -44,13 +44,10 @@ get_org_default_workspace, ) # noqa pylint: disable-all from agenta_backend.commons.utils.permissions import ( - check_action_access, - check_rbac_permission - ) - from agenta_backend.commons.models.db_models import ( - Permission, - WorkspaceRole + check_action_access, + check_rbac_permission, ) + from agenta_backend.commons.models.db_models import Permission, WorkspaceRole if FEATURE_FLAG in ["cloud"]: @@ -146,7 +143,9 @@ async def get_variant_by_env( object_type="app", permission=Permission.VIEW_APPLICATION, ) - logger.debug(f"user has Permission to get variant by environment: {has_permission}") + logger.debug( + f"user has Permission to get variant by environment: {has_permission}" + ) if not has_permission: error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( @@ -193,22 +192,25 @@ async def create_app( """ try: if FEATURE_FLAG in ["cloud", "ee"]: - try: - user_org_workspace_data = await get_user_org_and_workspace_id(request.state.user_id) + user_org_workspace_data = await get_user_org_and_workspace_id( + request.state.user_id + ) if user_org_workspace_data is None: - raise HTTPException( - status_code=400, - detail="Failed to get user org and workspace data", - ) - + raise HTTPException( + status_code=400, + detail="Failed to get user org and workspace data", + ) + if payload.organization_id: organization_id = payload.organization_id organization = await db_manager_ee.get_organization(organization_id) else: - organization = await get_user_own_org(user_org_workspace_data["uid"]) + organization = await get_user_own_org( + user_org_workspace_data["uid"] + ) organization_id = str(organization.id) - + if not organization: raise HTTPException( status_code=400, @@ -233,7 +235,9 @@ async def create_app( organization=organization, permission=Permission.CREATE_APPLICATION, ) - logger.debug(f"User has Permission to Create Application: {has_permission}") + logger.debug( + f"User has Permission to Create Application: {has_permission}" + ) if not has_permission: error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( @@ -244,7 +248,7 @@ async def create_app( raise HTTPException(status_code=500, detail=str(e)) app_db = await db_manager.create_app_and_envs( - payload.app_name, + payload.app_name, request.state.user_id, organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, @@ -280,7 +284,7 @@ async def list_apps( app_name=app_name, user_uid=request.state.user_id, org_id=org_id, - workspace_id=workspace_id + workspace_id=workspace_id, ) return apps except Exception as e: @@ -324,7 +328,6 @@ async def add_variant_from_image( raise HTTPException(status_code=404, detail="Image not found") try: - if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( user_uid=request.state.user_id, @@ -332,14 +335,16 @@ async def add_variant_from_image( object_type="app", permission=Permission.CREATE_APPLICATION, ) - logger.debug(f"User has Permission to create app from image: {has_permission}") + logger.debug( + f"User has Permission to create app from image: {has_permission}" + ) if not has_permission: error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( {"detail": error_msg}, status_code=400, ) - + app = await db_manager.fetch_app_by_id(app_id) variant_db = await app_manager.add_variant_based_on_image( @@ -370,7 +375,6 @@ async def remove_app(app_id: str, request: Request): app -- App to remove """ try: - if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( user_uid=request.state.user_id, @@ -421,10 +425,11 @@ async def create_app_and_variant_from_template( logger.debug("Start: Creating app and variant from template") if FEATURE_FLAG in ["cloud", "ee"]: - # Get user and org id logger.debug("Step 1: Getting user and organization ID") - user_org_workspace_data: dict = await get_user_org_and_workspace_id(request.state.user_id) + user_org_workspace_data: dict = await get_user_org_and_workspace_id( + request.state.user_id + ) logger.debug("Step 2: Setting organization ID") if payload.organization_id is None: @@ -448,7 +453,9 @@ async def create_app_and_variant_from_template( organization=organization, permission=Permission.CREATE_APPLICATION, ) - logger.debug(f"User has Permission to create app from template: {has_permission}") + logger.debug( + f"User has Permission to create app from template: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -457,7 +464,11 @@ async def create_app_and_variant_from_template( status_code=403, ) - print(f"Step 5: Checking if app {payload.app_name} already exists" if FEATURE_FLAG in ["cloud", "ee"] else f"Step 1: Checking if app {payload.app_name} already exists") + print( + f"Step 5: Checking if app {payload.app_name} already exists" + if FEATURE_FLAG in ["cloud", "ee"] + else f"Step 1: Checking if app {payload.app_name} already exists" + ) app_name = payload.app_name.lower() app = await db_manager.fetch_app_by_name_and_parameters( app_name, @@ -470,7 +481,11 @@ async def create_app_and_variant_from_template( f"App with name {app_name} already exists", ) - print("Step 6: Creating new app and initializing environments" if FEATURE_FLAG in ["cloud", "ee"] else "Step 2: Creating new app and initializing environments") + print( + "Step 6: Creating new app and initializing environments" + if FEATURE_FLAG in ["cloud", "ee"] + else "Step 2: Creating new app and initializing environments" + ) if app is None: print("Here next step") app = await db_manager.create_app_and_envs( @@ -480,13 +495,19 @@ async def create_app_and_variant_from_template( workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, ) - print("Step 7: Retrieve template from db" if FEATURE_FLAG in ["cloud", "ee"] else "Step 3: Retrieve template from db") + print( + "Step 7: Retrieve template from db" + if FEATURE_FLAG in ["cloud", "ee"] + else "Step 3: Retrieve template from db" + ) template_db = await db_manager.get_template(payload.template_id) repo_name = os.environ.get("AGENTA_TEMPLATE_REPO", "agentaai/templates_v2") image_name = f"{repo_name}:{template_db.name}" print( - "Step 8: Creating image instance and adding variant based on image" if FEATURE_FLAG in ["cloud", "ee"] else "Step 4: Creating image instance and adding variant based on image" + "Step 8: Creating image instance and adding variant based on image" + if FEATURE_FLAG in ["cloud", "ee"] + else "Step 4: Creating image instance and adding variant based on image" ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, @@ -494,16 +515,18 @@ async def create_app_and_variant_from_template( docker_id_or_template_uri=template_db.template_uri if FEATURE_FLAG in ["cloud", "ee"] else template_db.digest, - tags=f"{image_name}" - if FEATURE_FLAG not in ["cloud", "ee"] - else None, + tags=f"{image_name}" if FEATURE_FLAG not in ["cloud", "ee"] else None, base_name="app", config_name="default", is_template_image=True, - user_uid=request.state.user_id + user_uid=request.state.user_id, ) - print("Step 9: Creating testset for app variant" if FEATURE_FLAG in ["cloud", "ee"] else "Step 5: Creating testset for app variant") + print( + "Step 9: Creating testset for app variant" + if FEATURE_FLAG in ["cloud", "ee"] + else "Step 5: Creating testset for app variant" + ) await db_manager.add_testset_to_app_variant( app_id=str(app.id), org_id=organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, @@ -513,10 +536,18 @@ async def create_app_and_variant_from_template( user_uid=request.state.user_id, ) - print("Step 10: We create ready-to use evaluators" if FEATURE_FLAG in ["cloud", "ee"] else "Step 6: We create ready-to use evaluators") + print( + "Step 10: We create ready-to use evaluators" + if FEATURE_FLAG in ["cloud", "ee"] + else "Step 6: We create ready-to use evaluators" + ) await evaluator_manager.create_ready_to_use_evaluators(app=app) - print("Step 11: Starting variant and injecting environment variables" if FEATURE_FLAG in ["cloud", "ee"] else "Step 7: Starting variant and injecting environment variables") + print( + "Step 11: Starting variant and injecting environment variables" + if FEATURE_FLAG in ["cloud", "ee"] + else "Step 7: Starting variant and injecting environment variables" + ) if FEATURE_FLAG in ["cloud", "ee"]: if not os.environ["OPENAI_API_KEY"]: raise Exception( @@ -580,8 +611,7 @@ async def list_environments( environments_db = await db_manager.list_environments(app_id=app_id) logger.debug(f"environments_db: {environments_db}") return [ - await converters.environment_db_to_output(env) - for env in environments_db + await converters.environment_db_to_output(env) for env in environments_db ] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index 8f6cf24e4c..2f9e629337 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -56,7 +56,7 @@ async def list_bases( {"detail": error_msg}, status_code=403, ) - + bases = await db_manager.list_bases_for_app_id(app_id, base_name) return [converters.base_db_to_pydantic(base) for base in bases] except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index f471835e54..289e6fd318 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -46,7 +46,7 @@ async def save_config( {"detail": error_msg}, status_code=403, ) - + base_db = await db_manager.fetch_base_by_id(payload.base_id) variants_db = await db_manager.list_variants_for_base(base_db) variant_to_overwrite = None @@ -107,7 +107,7 @@ async def get_config( {"detail": error_msg}, status_code=403, ) - + base_db = await db_manager.fetch_base_by_id(base_id) # in case environment_name is provided, find the variant deployed if environment_name: diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 51db0a9178..500af783d1 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -94,7 +94,7 @@ async def restart_docker_container( Args: payload (RestartAppContainer) -- the required data (app_name and variant_name) """ - logger.debug(f"Restarting container for variant {payload.variant_id}") + logger.debug(f"Restarting container for variant {payload.variant_id}") app_variant_db = await db_manager.fetch_app_variant_by_id(payload.variant_id) try: deployment = await db_manager.get_deployment_by_objectid( @@ -152,7 +152,7 @@ async def construct_app_container_url( """ # assert that one of base_id or variant_id is provided assert base_id or variant_id, "Please provide either base_id or variant_id" - + if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( user_uid=request.state.user_id, @@ -167,7 +167,7 @@ async def construct_app_container_url( {"detail": error_msg}, status_code=403, ) - + if base_id: base_db = await db_manager.fetch_base_by_id(base_id) # TODO: Add status check if base_db.status == "running" diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 4feb553dae..577071ab81 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -49,7 +49,7 @@ async def deploy_to_environment( {"detail": error_msg}, status_code=403, ) - + await db_manager.deploy_to_environment( environment_name=payload.environment_name, variant_id=payload.variant_id, diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 671d491371..2738ead0d7 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -60,7 +60,7 @@ async def create_evaluation( {"detail": error_msg}, status_code=403, ) - + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: raise HTTPException(status_code=404, detail="App not found") @@ -120,7 +120,9 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) - logger.debug(f"User has permission to fetch evaluation status: {has_permission}") + logger.debug( + f"User has permission to fetch evaluation status: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -128,7 +130,7 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): {"detail": error_msg}, status_code=403, ) - + evaluation = await evaluation_service.fetch_evaluation(evaluation_id) return {"status": evaluation.status} except Exception as exc: @@ -155,7 +157,9 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) - logger.debug(f"User has permission to get evaluation results: {has_permission}") + logger.debug( + f"User has permission to get evaluation results: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -163,7 +167,7 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): {"detail": error_msg}, status_code=403, ) - + results = await evaluation_service.retrieve_evaluation_results(evaluation_id) return {"results": results, "evaluation_id": evaluation_id} except Exception as exc: @@ -199,7 +203,9 @@ async def fetch_evaluation_scenarios( object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) - logger.debug(f"User has permission to get evaluation scenarios: {has_permission}") + logger.debug( + f"User has permission to get evaluation scenarios: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -207,10 +213,14 @@ async def fetch_evaluation_scenarios( {"detail": error_msg}, status_code=403, ) - - eval_scenarios = await evaluation_service.fetch_evaluation_scenarios_for_evaluation(evaluation_id) + + eval_scenarios = ( + await evaluation_service.fetch_evaluation_scenarios_for_evaluation( + evaluation_id + ) + ) return eval_scenarios - + except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -236,7 +246,9 @@ async def fetch_list_evaluations( object_type="app", permission=Permission.VIEW_EVALUATION, ) - logger.debug(f"User has permission to get list of evaluations: {has_permission}") + logger.debug( + f"User has permission to get list of evaluations: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -244,7 +256,7 @@ async def fetch_list_evaluations( {"detail": error_msg}, status_code=403, ) - + return await evaluation_service.fetch_list_evaluations(app_id) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -273,7 +285,9 @@ async def fetch_evaluation( object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) - logger.debug(f"User has permission to get single evaluation: {has_permission}") + logger.debug( + f"User has permission to get single evaluation: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -281,7 +295,7 @@ async def fetch_evaluation( {"detail": error_msg}, status_code=403, ) - + return await evaluation_service.fetch_evaluation(evaluation_id) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -311,7 +325,9 @@ async def delete_evaluations( object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) - logger.debug(f"User has permission to delete evaluation: {has_permission}") + logger.debug( + f"User has permission to delete evaluation: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -319,7 +335,7 @@ async def delete_evaluations( {"detail": error_msg}, status_code=403, ) - + await evaluation_service.delete_evaluations(delete_evaluations.evaluations_ids) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as exc: @@ -374,7 +390,9 @@ async def fetch_evaluation_scenarios( object_type="evaluation", permission=Permission.VIEW_EVALUATION, ) - logger.debug(f"User has permission to get evaluation scenarios: {has_permission}") + logger.debug( + f"User has permission to get evaluation scenarios: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -382,8 +400,10 @@ async def fetch_evaluation_scenarios( {"detail": error_msg}, status_code=403, ) - - eval_scenarios = await evaluation_service.compare_evaluations_scenarios(evaluations_ids_list) + + eval_scenarios = await evaluation_service.compare_evaluations_scenarios( + evaluations_ids_list + ) return eval_scenarios except Exception as exc: diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 5b91fb5981..b1bbcbbc08 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -38,7 +38,9 @@ async def get_evaluators_endpoint(): evaluators = evaluator_manager.get_evaluators() if evaluators is None: - raise HTTPException(status_code=500, detail="Error processing evaluators file") + raise HTTPException( + status_code=500, detail="Error processing evaluators file" + ) if not evaluators: raise HTTPException(status_code=404, detail="No evaluators found") @@ -91,7 +93,7 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): {"detail": error_msg}, status_code=403, ) - + evaluators_configs = await evaluator_manager.get_evaluator_config( evaluator_config_id ) diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 627fc01b8a..b2cb881411 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -38,12 +38,12 @@ get_user_org_and_workspace_id, ) from agenta_backend.commons.utils.permissions import ( # noqa pylint: disable-all - check_action_access, - check_rbac_permission + check_action_access, + check_rbac_permission, ) from agenta_backend.commons.models.db_models import ( # noqa pylint: disable-all - Permission, - WorkspaceRole + Permission, + WorkspaceRole, ) router = APIRouter() @@ -68,7 +68,7 @@ async def create_evaluation( user_uid=request.state.user_id, object_id=payload.app_id, object_type="app", - permission=Permission.CREATE_EVALUATION + permission=Permission.CREATE_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -76,7 +76,7 @@ async def create_evaluation( {"detail": error_msg}, status_code=400, ) - + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: @@ -114,7 +114,7 @@ async def fetch_list_human_evaluations( user_uid=request.state.user_id, object_id=app_id, object_type="app", - permission=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -122,7 +122,7 @@ async def fetch_list_human_evaluations( {"detail": error_msg}, status_code=400, ) - + return await evaluation_service.fetch_list_human_evaluations(app_id) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e @@ -147,7 +147,7 @@ async def fetch_human_evaluation( user_uid=request.state.user_id, object_id=evaluation_id, object_type="human_evaluation", - permission=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -155,7 +155,7 @@ async def fetch_human_evaluation( {"detail": error_msg}, status_code=400, ) - + return await evaluation_service.fetch_human_evaluation(evaluation_id) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e @@ -188,7 +188,7 @@ async def fetch_evaluation_scenarios( user_uid=request.state.user_id, object_id=evaluation_id, object_type="human_evaluation_scenario_by_evaluation_id", - permission=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -196,9 +196,11 @@ async def fetch_evaluation_scenarios( {"detail": error_msg}, status_code=400, ) - + eval_scenarios = ( - await evaluation_service.fetch_human_evaluation_scenarios_for_evaluation(evaluation_id) + await evaluation_service.fetch_human_evaluation_scenarios_for_evaluation( + evaluation_id + ) ) return eval_scenarios @@ -226,7 +228,7 @@ async def update_human_evaluation( user_uid=request.state.user_id, object_id=evaluation_id, object_type="human_evaluation", - permission=Permission.EDIT_EVALUATION + permission=Permission.EDIT_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -234,10 +236,8 @@ async def update_human_evaluation( {"detail": error_msg}, status_code=400, ) - - await update_human_evaluation_service( - evaluation_id, update_data - ) + + await update_human_evaluation_service(evaluation_id, update_data) return Response(status_code=status.HTTP_204_NO_CONTENT) except KeyError: @@ -264,14 +264,14 @@ async def update_evaluation_scenario_router( Returns: None: 204 No Content status code upon successful update. - """ + """ try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_scenario_id, object_type="human_evaluation_scenario", - permission=Permission.EDIT_EVALUATION + permission=Permission.EDIT_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -279,7 +279,7 @@ async def update_evaluation_scenario_router( {"detail": error_msg}, status_code=400, ) - + await update_human_evaluation_scenario( evaluation_scenario_id, evaluation_scenario, @@ -311,7 +311,7 @@ async def get_evaluation_scenario_score_router( user_uid=request.state.user_id, object_id=evaluation_scenario_id, object_type="human_evaluation_scenario", - permission=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -319,7 +319,7 @@ async def get_evaluation_scenario_score_router( {"detail": error_msg}, status_code=400, ) - + return await get_evaluation_scenario_score_service(evaluation_scenario_id) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e @@ -345,7 +345,7 @@ async def update_evaluation_scenario_score_router( user_uid=request.state.user_id, object_id=evaluation_scenario_id, object_type="human_evaluation_scenario", - permission=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -353,7 +353,7 @@ async def update_evaluation_scenario_score_router( {"detail": error_msg}, status_code=400, ) - + await update_evaluation_scenario_score_service( evaluation_scenario_id, payload.score ) @@ -382,7 +382,7 @@ async def fetch_results( user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", - permission=Permission.VIEW_EVALUATION + permission=Permission.VIEW_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -390,7 +390,7 @@ async def fetch_results( {"detail": error_msg}, status_code=400, ) - + evaluation = await evaluation_service._fetch_human_evaluation(evaluation_id) if evaluation.evaluation_type == EvaluationType.human_a_b_testing: results = await results_service.fetch_results_for_evaluation(evaluation) @@ -427,7 +427,7 @@ async def delete_evaluations( user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", - permission=Permission.DELETE_EVALUATION + permission=Permission.DELETE_EVALUATION, ) if not has_permission: error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." @@ -435,8 +435,10 @@ async def delete_evaluations( {"detail": error_msg}, status_code=400, ) - - await evaluation_service.delete_human_evaluations(delete_evaluations.evaluations_ids) + + await evaluation_service.delete_human_evaluations( + delete_evaluations.evaluations_ids + ) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/agenta-backend/agenta_backend/routers/observability_router.py b/agenta-backend/agenta_backend/routers/observability_router.py index aa82dcd2a4..7e66f9edca 100644 --- a/agenta-backend/agenta_backend/routers/observability_router.py +++ b/agenta-backend/agenta_backend/routers/observability_router.py @@ -144,5 +144,7 @@ async def update_feedback( payload: UpdateFeedback, request: Request, ): - feedback = await update_trace_feedback(trace_id, feedback_id, payload, request.state.user_id) + feedback = await update_trace_feedback( + trace_id, feedback_id, payload, request.state.user_id + ) return feedback diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index cfd8936ba8..e34bda070e 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -26,15 +26,17 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all - from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all + from agenta_backend.commons.utils.permissions import ( + check_action_access, + ) # noqa pylint: disable-all from agenta_backend.commons.models.db_models import ( - TestSetDB_ as TestSetDB - ) # noqa pylint: disable-all + Permission, + ) # noqa pylint: disable-all + from agenta_backend.commons.models.db_models import ( + TestSetDB_ as TestSetDB, + ) # noqa pylint: disable-all else: - from agenta_backend.models.db_models import ( - TestSetDB - ) + from agenta_backend.models.db_models import TestSetDB router = APIRouter() logger = logging.getLogger(__name__) @@ -80,7 +82,7 @@ async def upload_file( {"detail": error_msg}, status_code=403, ) - + app = await db_manager.fetch_app_by_id(app_id=app_id) # Create a document document = { @@ -164,7 +166,7 @@ async def import_testset( {"detail": error_msg}, status_code=403, ) - + app = await db_manager.fetch_app_by_id(app_id=app_id) try: @@ -251,7 +253,7 @@ async def create_testset( {"detail": error_msg}, status_code=403, ) - + user = await get_user(request.state.user_id) app = await db_manager.fetch_app_by_id(app_id=app_id) testset = { @@ -310,17 +312,17 @@ async def update_testset( {"detail": error_msg}, status_code=403, ) - + testset_update = { "name": csvdata.name, "csvdata": csvdata.csvdata, "updated_at": datetime.now().isoformat(), } - + test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: raise HTTPException(status_code=404, detail="testset not found") - + try: await test_set.update({"$set": testset_update}) if isinstance(test_set.id, ObjectId): @@ -365,7 +367,7 @@ async def get_testsets( {"detail": error_msg}, status_code=403, ) - + app = await db_manager.fetch_app_by_id(app_id=app_id) if app is None: @@ -411,11 +413,11 @@ async def get_single_testset( {"detail": error_msg}, status_code=403, ) - + test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: raise HTTPException(status_code=404, detail="testset not found") - + return testset_db_to_pydantic(test_set) @@ -455,7 +457,7 @@ async def delete_testsets( test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: raise HTTPException(status_code=404, detail="testset not found") - + await test_set.delete() deleted_ids.append(testset_id) diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index bc585f2f93..e5f2980c26 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -15,8 +15,12 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all - from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all + from agenta_backend.commons.utils.permissions import ( + check_action_access, + ) # noqa pylint: disable-all + from agenta_backend.commons.models.db_models import ( + Permission, + ) # noqa pylint: disable-all from agenta_backend.commons.models.api.api_models import ( Image_ as Image, AppVariantResponse_ as AppVariantResponse, @@ -62,8 +66,8 @@ async def add_variant_from_base_and_config( try: logger.debug("Initiating process to add a variant based on a previous one.") logger.debug(f"Received payload: {payload}") - - # Check user has permission to add variant + + # Check user has permission to add variant if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( user_uid=request.state.user_id, @@ -71,7 +75,9 @@ async def add_variant_from_base_and_config( object_type="base", permission=Permission.CREATE_APPLICATION, ) - logger.debug(f"User has Permission to create variant from base and config: {has_permission}") + logger.debug( + f"User has Permission to create variant from base and config: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -79,7 +85,7 @@ async def add_variant_from_base_and_config( {"detail": error_msg}, status_code=403, ) - + base_db = await db_manager.fetch_base_by_id(payload.base_id) # Find the previous variant in the database @@ -171,7 +177,9 @@ async def update_variant_parameters( object_type="app_variant", permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) - logger.debug(f"User has Permission to update variant parameters: {has_permission}") + logger.debug( + f"User has Permission to update variant parameters: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -179,7 +187,7 @@ async def update_variant_parameters( {"detail": error_msg}, status_code=403, ) - + await app_manager.update_variant_parameters( app_variant_id=variant_id, parameters=payload.parameters, @@ -219,7 +227,9 @@ async def update_variant_image( object_type="app_variant", permission=Permission.CREATE_APPLICATION, ) - logger.debug(f"User has Permission to update variant image: {has_permission}") + logger.debug( + f"User has Permission to update variant image: {has_permission}" + ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) @@ -227,7 +237,7 @@ async def update_variant_image( {"detail": error_msg}, status_code=403, ) - + db_app_variant = await db_manager.fetch_app_variant_by_id( app_variant_id=variant_id ) @@ -298,7 +308,7 @@ async def start_variant( } else: envvars = {} if env_vars is None else env_vars.env_vars - + app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id=variant_id) if action.action == VariantActionEnum.START: url: URI = await app_manager.start_variant(app_variant_db, envvars) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 133a5124b9..8dcc559a05 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -44,8 +44,7 @@ async def start_variant( - db_app_variant: AppVariantDB, - env_vars: DockerEnvVars = None + db_app_variant: AppVariantDB, env_vars: DockerEnvVars = None ) -> URI: """ Starts a Docker container for a given app variant. @@ -110,9 +109,7 @@ async def start_variant( return URI(uri=deployment.uri) -async def update_variant_image( - app_variant_db: AppVariantDB, image: Image -): +async def update_variant_image(app_variant_db: AppVariantDB, image: Image): """Updates the image for app variant in the database. Arguments: @@ -141,8 +138,12 @@ async def update_variant_image( docker_id=image.docker_id, user=app_variant_db.user, deletable=True, - organization=app_variant_db.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa - workspace=app_variant_db.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + organization=app_variant_db.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa + workspace=app_variant_db.workspace + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa ) # Update base with new image await db_manager.update_base(app_variant_db.base, image=db_image) @@ -270,7 +271,9 @@ async def remove_app_related_resources(app_id: str): """ try: # Delete associated environments - environments: List[AppEnvironmentDB] = await db_manager.list_environments(app_id) + environments: List[AppEnvironmentDB] = await db_manager.list_environments( + app_id + ) for environment_db in environments: await db_manager.remove_environment(environment_db) logger.info(f"Successfully deleted environment {environment_db.name}.") @@ -320,9 +323,7 @@ async def remove_app(app_id: str): raise e from None -async def update_variant_parameters( - app_variant_id: str, parameters: Dict[str, Any] -): +async def update_variant_parameters(app_variant_id: str, parameters: Dict[str, Any]): """Updates the parameters for app variant in the database. Arguments: @@ -336,7 +337,9 @@ async def update_variant_parameters( logger.error(error_msg) raise ValueError(error_msg) try: - await db_manager.update_variant_parameters(app_variant_db=app_variant_db, parameters=parameters) + await db_manager.update_variant_parameters( + app_variant_db=app_variant_db, parameters=parameters + ) except Exception as e: logger.error( f"Error updating app variant {app_variant_db.app.app_name}/{app_variant_db.variant_name}" @@ -395,9 +398,7 @@ async def add_variant_based_on_image( # Check if app variant already exists logger.debug("Step 2: Checking if app variant already exists") - variants = await db_manager.list_app_variants_for_app_id( - app_id=str(app.id) - ) + variants = await db_manager.list_app_variants_for_app_id(app_id=str(app.id)) already_exists = any(av for av in variants if av.variant_name == variant_name) if already_exists: logger.error("App variant with the same name already exists") @@ -409,14 +410,22 @@ async def add_variant_based_on_image( if parsed_url.scheme and parsed_url.netloc: db_image = await db_manager.get_orga_image_instance_by_uri( template_uri=docker_id_or_template_uri, - organization_id=str(app.organization.id) if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa - workspace_id=str(app.workspace.id) if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + organization_id=str(app.organization.id) + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa + workspace_id=str(app.workspace.id) + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa ) else: db_image = await db_manager.get_orga_image_instance_by_docker_id( docker_id=docker_id_or_template_uri, - organization_id=str(app.organization.id) if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa - workspace_id=str(app.workspace.id) if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + organization_id=str(app.organization.id) + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa + workspace_id=str(app.workspace.id) + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa ) # Create new image if not exists @@ -428,8 +437,12 @@ async def add_variant_based_on_image( template_uri=docker_id_or_template_uri, deletable=not (is_template_image), user=user_instance, - organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa - workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + organization=app.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa + workspace=app.workspace + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa ) else: docker_id = docker_id_or_template_uri @@ -439,8 +452,12 @@ async def add_variant_based_on_image( tags=tags, deletable=not (is_template_image), user=user_instance, - organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa - workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + organization=app.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa + workspace=app.workspace + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa ) # Create config @@ -457,7 +474,9 @@ async def add_variant_based_on_image( ] # TODO: Change this in SDK2 to directly use base_name db_base = await db_manager.create_new_variant_base( app=app, - organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + organization=app.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa user=user_instance, base_name=base_name, # the first variant always has default base @@ -471,7 +490,9 @@ async def add_variant_based_on_image( variant_name=variant_name, image=db_image, user=user_instance, - organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + organization=app.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa parameters={}, base_name=base_name, diff --git a/agenta-backend/agenta_backend/services/container_manager.py b/agenta-backend/agenta_backend/services/container_manager.py index bee8047979..fba245cde6 100644 --- a/agenta-backend/agenta_backend/services/container_manager.py +++ b/agenta-backend/agenta_backend/services/container_manager.py @@ -120,9 +120,9 @@ def build_image_job( docker_id=image.id, tags=image.tags[0], ) - + return pydantic_image - + except docker.errors.BuildError as ex: log = "Error building Docker image:\n" log += str(ex) + "\n" diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 3b7df5b5aa..be1b87fe3d 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -34,12 +34,12 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.services import db_manager_ee - + from agenta_backend.commons.models.api.api_models import ( AppVariant_ as AppVariant, ImageExtended_ as ImageExtended, ) - + from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, UserDB_ as UserDB, @@ -127,19 +127,19 @@ async def add_testset_to_app_variant( app=app_db, user=user_db, ) - + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( org_id is not None and workspace_id is not None ), "organization and workspace must be provided together" - + organization_db = await db_manager_ee.get_organization(org_id) workspace_db = await db_manager_ee.get_workspace(workspace_id) - + testset_db.organization = organization_db testset_db.workspace = workspace_db - + await testset_db.create() except Exception as e: @@ -235,8 +235,8 @@ async def create_new_variant_base( user: UserDB, base_name: str, image: ImageDB, - organization = None, - workspace = None, + organization=None, + workspace=None, ) -> VariantBaseDB: """Create a new base. Args: @@ -256,16 +256,16 @@ async def create_new_variant_base( base_name=base_name, image=image, ) - + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None ), "organization and workspace must be provided together" - + base.organization = organization base.workspace = workspace - + await base.create() return base @@ -305,8 +305,8 @@ async def create_new_app_variant( base_name: str, config_name: str, parameters: Dict, - organization = None, - workspace = None, + organization=None, + workspace=None, ) -> AppVariantDB: """Create a new variant. Args: @@ -328,16 +328,16 @@ async def create_new_app_variant( config_name=config_name, parameters=parameters, ) - + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None ), "organization and workspace must be provided together" - + variant.organization = organization variant.workspace = workspace - + await variant.create() return variant @@ -346,8 +346,8 @@ async def create_image( image_type: str, user: UserDB, deletable: bool, - organization = None, - workspace = None, + organization=None, + workspace=None, template_uri: str = None, docker_id: str = None, tags: str = None, @@ -390,16 +390,16 @@ async def create_image( elif image_type == "image": image.type = "image" image.docker_id = docker_id - + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None ), "organization and workspace must be provided together" - + image.organization = organization image.workspace = workspace - + await image.create() return image @@ -411,8 +411,8 @@ async def create_deployment( container_id: str, uri: str, status: str, - organization = None, - workspace = None, + organization=None, + workspace=None, ) -> DeploymentDB: """Create a new deployment. Args: @@ -435,11 +435,11 @@ async def create_deployment( uri=uri, status=status, ) - + if FEATURE_FLAG in ["lcoud", "ee"]: deployment.organization = organization deployment.workspace = workspace - + await deployment.create() return deployment @@ -470,26 +470,26 @@ async def create_app_and_envs( app = await fetch_app_by_name_and_parameters( app_name, user_uid, - organization_id, + organization_id, workspace_id, ) if app is not None: raise ValueError("App with the same name already exists") app = AppDB(app_name=app_name, user=user_instance) - + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization_id is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None ), "org_id and workspace_id must be provided together" - + organization_db = await db_manager_ee.get_organization(organization_id) workspace_db = await db_manager_ee.get_workspace(workspace_id) - + app.organization = organization_db app.workspace = workspace_db - + await app.create() await initialize_environments(app) return app @@ -542,9 +542,7 @@ async def list_bases_for_app_id( return bases_db -async def list_variants_for_base( - base: VariantBaseDB -) -> List[AppVariantDB]: +async def list_variants_for_base(base: VariantBaseDB) -> List[AppVariantDB]: """ Lists all the app variants from the db for a base Args: @@ -575,7 +573,6 @@ async def get_user(user_uid: str) -> UserDB: user = await UserDB.find_one(UserDB.uid == user_uid) if user is None: if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"]: - # create user user_db = UserDB(uid="0") user = await user_db.create() @@ -661,19 +658,21 @@ async def get_orga_image_instance_by_docker_id( Returns: ImageDB: instance of image object """ - - query_expression = {'docker_id': docker_id} - + + query_expression = {"docker_id": docker_id} + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None ), "organization and workspace must be provided together" - - query_expression.update({ - 'organization.id': ObjectId(organization_id), - 'workspace.id': ObjectId(workspace_id), - }) + + query_expression.update( + { + "organization.id": ObjectId(organization_id), + "workspace.id": ObjectId(workspace_id), + } + ) image = await ImageDB.find_one(query_expression) return image @@ -695,19 +694,21 @@ async def get_orga_image_instance_by_uri( if not parsed_url.scheme and not parsed_url.netloc: raise ValueError(f"Invalid URL: {template_uri}") - - query_expression = {'template_uri': template_uri} - + + query_expression = {"template_uri": template_uri} + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None ), "organization and workspace must be provided together" - - query_expression.update({ - 'organization.id': ObjectId(organization_id), - 'workspace.id': ObjectId(workspace_id), - }) + + query_expression.update( + { + "organization.id": ObjectId(organization_id), + "workspace.id": ObjectId(workspace_id), + } + ) image = await ImageDB.find_one(query_expression) return image @@ -810,10 +811,12 @@ async def list_apps( if app_name is not None: app_db = await fetch_app_by_name_and_parameters( - app_name=app_name, user_uid=user_uid, organization_id=org_id, workspace_id=workspace_id + app_name=app_name, + user_uid=user_uid, + organization_id=org_id, + workspace_id=workspace_id, ) return [converters.app_db_to_pydantic(app_db)] - # elif (org_id is not None) or (workspace_id is not None): # TODO: Remember to enable this when workspace_id is provided after the RBAC is implemented # if not FEATURE_FLAG in ["cloud", "ee"]: @@ -821,12 +824,12 @@ async def list_apps( # {"error": "organization and/or workspace is only available in Cloud and EE"}, # status_code=400, # ) - + # # assert that if org_id is provided, workspace_id is also provided, and vice versa # assert ( # org_id is not None and workspace_id is not None # ), "org_id and workspace_id must be provided together" - + # user_org_workspace_data = await get_user_org_and_workspace_id(user_uid) # has_permission = await check_rbac_permission( # user_org_workspace_data=user_org_workspace_data, @@ -841,7 +844,7 @@ async def list_apps( # {"detail": error_msg}, # status_code=400, # ) - + # apps: List[AppDB] = await AppDB.find( # AppDB.organization.id == ObjectId(org_id), # AppDB.workspace.id == ObjectId(workspace_id), @@ -985,9 +988,7 @@ async def list_environments(app_id: str) -> List[AppEnvironmentDB]: return environments_db -async def initialize_environments( - app_db: AppDB -) -> List[AppEnvironmentDB]: +async def initialize_environments(app_db: AppDB) -> List[AppEnvironmentDB]: """ Initializes the environments for the app with the given database. @@ -1005,9 +1006,7 @@ async def initialize_environments( return environments -async def create_environment( - name: str, app_db: AppDB -) -> AppEnvironmentDB: +async def create_environment(name: str, app_db: AppDB) -> AppEnvironmentDB: """ Creates a new environment in the database. @@ -1019,22 +1018,18 @@ async def create_environment( Returns: AppEnvironmentDB: The newly created AppEnvironmentDB object. """ - environment_db = AppEnvironmentDB( - app=app_db, - name=name, - user=app_db.user - ) - + environment_db = AppEnvironmentDB(app=app_db, name=name, user=app_db.user) + if FEATURE_FLAG in ["cloud", "ee"]: environment_db.organization = app_db.organization environment_db.workspace = app_db.workspace - + await environment_db.create() return environment_db async def list_environments_by_variant( - app_variant: AppVariantDB + app_variant: AppVariantDB, ) -> List[AppEnvironmentDB]: """ Returns a list of environments for a given app variant. @@ -1522,7 +1517,7 @@ async def update_app_variant( async def fetch_app_by_name_and_parameters( app_name: str, user_uid: str, - organization_id: str = None, + organization_id: str = None, workspace_id: str = None, ): """Fetch an app by its name, organization id, and workspace id. @@ -1536,25 +1531,29 @@ async def fetch_app_by_name_and_parameters( AppDB: the instance of the app """ - query_expression = {'app_name': app_name} - + query_expression = {"app_name": app_name} + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None ), "organization_id and workspace_id must be provided together" - - query_expression.update({ - 'organization.id': ObjectId(organization_id), - 'workspace.id': ObjectId(workspace_id), - }) + + query_expression.update( + { + "organization.id": ObjectId(organization_id), + "workspace.id": ObjectId(workspace_id), + } + ) else: - query_expression.update({ - 'user.id': ObjectId(user_uid), - }) - + query_expression.update( + { + "user.id": ObjectId(user_uid), + } + ) + app_db = await AppDB.find_one(query_expression, fetch_links=True) - + return app_db @@ -1565,8 +1564,8 @@ async def create_new_evaluation( status: str, variant: AppVariantDB, evaluators_configs: List[str], - organization = None, - workspace = None, + organization=None, + workspace=None, ) -> EvaluationDB: """Create a new evaluation scenario. Returns: @@ -1583,16 +1582,16 @@ async def create_new_evaluation( created_at=datetime.now().isoformat(), updated_at=datetime.now().isoformat(), ) - + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None ), "organization and workspace must be provided together" - + evaluation.organization = organization evaluation.workspace = workspace - + await evaluation.create() return evaluation @@ -1608,8 +1607,8 @@ async def create_new_evaluation_scenario( note: Optional[str], evaluators_configs: List[EvaluatorConfigDB], results: List[EvaluationScenarioResult], - organization = None, - workspace = None, + organization=None, + workspace=None, ) -> EvaluationScenarioDB: """Create a new evaluation scenario. Returns: @@ -1629,16 +1628,16 @@ async def create_new_evaluation_scenario( created_at=datetime.utcnow(), updated_at=datetime.utcnow(), ) - + if FEATURE_FLAG in ["cloud", "ee"]: # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None ), "organization and workspace must be provided together" - + evaluation_scenario.organization = organization evaluation_scenario.workspace = workspace - + await evaluation_scenario.create() return evaluation_scenario @@ -1746,8 +1745,8 @@ async def create_evaluator_config( user: UserDB, name: str, evaluator_key: str, - organization = None, - workspace = None, + organization=None, + workspace=None, settings_values: Optional[Dict[str, Any]] = None, ) -> EvaluatorConfigDB: """Create a new evaluator configuration in the database.""" @@ -1759,14 +1758,13 @@ async def create_evaluator_config( evaluator_key=evaluator_key, settings_values=settings_values, ) - + if FEATURE_FLAG in ["cloud", "ee"]: - # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None ), "organization and workspace must be provided together" - + new_evaluator_config.organization = organization new_evaluator_config.workspace = workspace diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index dee580a31f..58eef14274 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -58,7 +58,9 @@ async def start_service( container_id=container_id, uri=uri, status="running", - organization=app_variant_db.organization if FEATURE_FLAG in ["cloud", "ee"] else None, + organization=app_variant_db.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, workspace=app_variant_db.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, ) return deployment diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 7401d63105..7423dfef86 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -60,14 +60,12 @@ class UpdateEvaluationScenarioError(Exception): pass -async def _fetch_human_evaluation( - evaluation_id: str -) -> HumanEvaluationDB: +async def _fetch_human_evaluation(evaluation_id: str) -> HumanEvaluationDB: # Fetch the evaluation by ID evaluation = await db_manager.fetch_human_evaluation_by_id( evaluation_id=evaluation_id ) - + # Check if the evaluation exists if evaluation is None: raise HTTPException( @@ -79,7 +77,7 @@ async def _fetch_human_evaluation( async def _fetch_human_evaluation_scenario( - evaluation_scenario_id: str + evaluation_scenario_id: str, ) -> HumanEvaluationDB: # Fetch the evaluation by ID evaluation_scenario = await db_manager.fetch_human_evaluation_scenario_by_id( @@ -237,7 +235,7 @@ async def update_human_evaluation_service( async def fetch_evaluation_scenarios_for_evaluation( - evaluation_id: str + evaluation_id: str, ) -> List[EvaluationScenario]: """ Fetch evaluation scenarios for a given evaluation ID. @@ -294,7 +292,7 @@ async def fetch_human_evaluation_scenarios_for_evaluation( async def update_human_evaluation_scenario( evaluation_scenario_id: str, evaluation_scenario_data: EvaluationScenarioUpdate, - evaluation_type: EvaluationType + evaluation_type: EvaluationType, ) -> None: """ Updates an evaluation scenario. @@ -376,7 +374,7 @@ async def update_evaluation_scenario_score_service( async def get_evaluation_scenario_score_service( - evaluation_scenario_id: str + evaluation_scenario_id: str, ) -> Dict[str, str]: """ Retrieve the score of a given evaluation scenario. @@ -423,7 +421,7 @@ async def fetch_list_evaluations( Returns: List[Evaluation]: A list of evaluations. """ - + evaluations_db = await EvaluationDB.find( EvaluationDB.app.id == ObjectId(app_id), fetch_links=True ).to_list() @@ -468,9 +466,7 @@ async def fetch_list_human_evaluations( ] -async def fetch_human_evaluation( - evaluation_id: str -) -> HumanEvaluation: +async def fetch_human_evaluation(evaluation_id: str) -> HumanEvaluation: """ Fetches a single evaluation based on its ID. @@ -480,15 +476,11 @@ async def fetch_human_evaluation( Returns: Evaluation: The fetched evaluation. """ - evaluation = await _fetch_human_evaluation( - evaluation_id=evaluation_id - ) + evaluation = await _fetch_human_evaluation(evaluation_id=evaluation_id) return await converters.human_evaluation_db_to_pydantic(evaluation) -async def delete_human_evaluations( - evaluation_ids: List[str] -) -> None: +async def delete_human_evaluations(evaluation_ids: List[str]) -> None: """ Delete evaluations by their IDs. @@ -499,9 +491,7 @@ async def delete_human_evaluations( HTTPException: If evaluation not found or access denied. """ for evaluation_id in evaluation_ids: - evaluation = await _fetch_human_evaluation( - evaluation_id=evaluation_id - ) + evaluation = await _fetch_human_evaluation(evaluation_id=evaluation_id) await evaluation.delete() @@ -614,9 +604,7 @@ async def create_new_evaluation( return await converters.evaluation_db_to_pydantic(evaluation_db) -async def retrieve_evaluation_results( - evaluation_id: str -) -> List[dict]: +async def retrieve_evaluation_results(evaluation_id: str) -> List[dict]: """Retrieve the aggregated results for a given evaluation. Args: diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 51126450fd..046512c020 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -10,9 +10,9 @@ if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, - EvaluatorConfigDB_ as EvaluatorConfigDB + EvaluatorConfigDB_ as EvaluatorConfigDB, ) -else: +else: from agenta_backend.models.db_models import AppDB, EvaluatorConfigDB from agenta_backend.models.converters import evaluator_config_db_to_pydantic from agenta_backend.models.api.evaluation_model import Evaluator, EvaluatorConfig @@ -91,7 +91,9 @@ async def create_evaluator_config( app = await db_manager.fetch_app_by_id(app_id) evaluator_config = await db_manager.create_evaluator_config( app=app, - organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, + organization=app.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa, workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, user=app.user, name=name, @@ -158,8 +160,12 @@ async def create_ready_to_use_evaluators(app: AppDB): for evaluator in direct_use_evaluators: await db_manager.create_evaluator_config( app=app, - organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, - workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, + organization=app.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa, + workspace=app.workspace + if FEATURE_FLAG in ["cloud", "ee"] + else None, # noqa, user=app.user, name=evaluator["name"], evaluator_key=evaluator["key"], diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index e30f2502e1..9e4f5db178 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -26,7 +26,7 @@ from agenta_backend.commons.models.db_models import AppDB_ as AppDB else: from agenta_backend.models.db_models import AppDB - + from agenta_backend.models.db_models import ( Result, AggregatedResult, @@ -182,8 +182,12 @@ def evaluate( EvaluationScenarioOutputDB(type="text", value=app_output.output) ], results=evaluators_results, - organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, - workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, + organization=app.organization + if FEATURE_FLAG in ["cloud", "ee"] + else None, + workspace=app.workspace + if FEATURE_FLAG in ["cloud", "ee"] + else None, ) ) From 51a547604f92227960cced7dbfd05e233cbd61f8 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 23 Jan 2024 16:21:04 +0100 Subject: [PATCH 33/99] fix feature flag mispelling --- agenta-backend/agenta_backend/services/db_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index be1b87fe3d..345eb8e91f 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -436,7 +436,7 @@ async def create_deployment( status=status, ) - if FEATURE_FLAG in ["lcoud", "ee"]: + if FEATURE_FLAG in ["cloud", "ee"]: deployment.organization = organization deployment.workspace = workspace From 4372df892a555572c98fe3d7cbdd4080d96999fe Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 23 Jan 2024 16:22:20 +0100 Subject: [PATCH 34/99] fix closing square bracket --- agenta-backend/agenta_backend/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/main.py b/agenta-backend/agenta_backend/main.py index 7e756dd626..4692cd27c2 100644 --- a/agenta-backend/agenta_backend/main.py +++ b/agenta-backend/agenta_backend/main.py @@ -84,7 +84,7 @@ async def lifespan(application: FastAPI, cache=True): app.include_router(user_profile.router, prefix="/profile") app.include_router(app_router.router, prefix="/apps", tags=["Apps"]) app.include_router(variants_router.router, prefix="/variants", tags=["Variants"]) -app.include_router(evaluation_router.router, prefix="/evaluations", tags=["Evaluations") +app.include_router(evaluation_router.router, prefix="/evaluations", tags=["Evaluations"]) app.include_router(human_evaluation_router.router, prefix="/human-evaluations", tags=["Human-Evaluations"]) app.include_router(evaluators_router.router, prefix="/evaluators", tags=["Evaluators"]) app.include_router(testset_router.router, prefix="/testsets", tags=["Testsets"]) From 01d6dbb711c1268f13b49ecea0049ab01102b292 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 23 Jan 2024 16:26:26 +0100 Subject: [PATCH 35/99] fix closing square brackets --- agenta-backend/agenta_backend/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/main.py b/agenta-backend/agenta_backend/main.py index 4692cd27c2..362e912ba7 100644 --- a/agenta-backend/agenta_backend/main.py +++ b/agenta-backend/agenta_backend/main.py @@ -89,8 +89,8 @@ async def lifespan(application: FastAPI, cache=True): app.include_router(evaluators_router.router, prefix="/evaluators", tags=["Evaluators"]) app.include_router(testset_router.router, prefix="/testsets", tags=["Testsets"]) app.include_router(container_router.router, prefix="/containers", tags=["Containers"]) -app.include_router(environment_router.router, prefix="/environments", tags=["Environments") -app.include_router(observability_router.router, prefix="/observability", tags=["Observability") +app.include_router(environment_router.router, prefix="/environments", tags=["Environments"]) +app.include_router(observability_router.router, prefix="/observability", tags=["Observability"]) app.include_router(bases_router.router, prefix="/bases", tags=["Bases"]) app.include_router(configs_router.router, prefix="/configs", tags=["Configs"]) From a2369e2026534b1db22fdd7c9cd86ec413447c4f Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 23 Jan 2024 19:08:01 +0100 Subject: [PATCH 36/99] cleanup --- .../agenta_backend/routers/app_router.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index ec0d7cb4b3..022907bc1b 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -464,7 +464,7 @@ async def create_app_and_variant_from_template( status_code=403, ) - print( + logger.debug( f"Step 5: Checking if app {payload.app_name} already exists" if FEATURE_FLAG in ["cloud", "ee"] else f"Step 1: Checking if app {payload.app_name} already exists" @@ -481,13 +481,12 @@ async def create_app_and_variant_from_template( f"App with name {app_name} already exists", ) - print( + logger.debug( "Step 6: Creating new app and initializing environments" if FEATURE_FLAG in ["cloud", "ee"] else "Step 2: Creating new app and initializing environments" ) if app is None: - print("Here next step") app = await db_manager.create_app_and_envs( app_name, request.state.user_id, @@ -495,7 +494,7 @@ async def create_app_and_variant_from_template( workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, ) - print( + logger.debug( "Step 7: Retrieve template from db" if FEATURE_FLAG in ["cloud", "ee"] else "Step 3: Retrieve template from db" @@ -504,7 +503,7 @@ async def create_app_and_variant_from_template( repo_name = os.environ.get("AGENTA_TEMPLATE_REPO", "agentaai/templates_v2") image_name = f"{repo_name}:{template_db.name}" - print( + logger.debug( "Step 8: Creating image instance and adding variant based on image" if FEATURE_FLAG in ["cloud", "ee"] else "Step 4: Creating image instance and adding variant based on image" @@ -522,7 +521,7 @@ async def create_app_and_variant_from_template( user_uid=request.state.user_id, ) - print( + logger.debug( "Step 9: Creating testset for app variant" if FEATURE_FLAG in ["cloud", "ee"] else "Step 5: Creating testset for app variant" @@ -536,14 +535,14 @@ async def create_app_and_variant_from_template( user_uid=request.state.user_id, ) - print( + logger.debug( "Step 10: We create ready-to use evaluators" if FEATURE_FLAG in ["cloud", "ee"] else "Step 6: We create ready-to use evaluators" ) await evaluator_manager.create_ready_to_use_evaluators(app=app) - print( + logger.debug( "Step 11: Starting variant and injecting environment variables" if FEATURE_FLAG in ["cloud", "ee"] else "Step 7: Starting variant and injecting environment variables" @@ -595,8 +594,8 @@ async def list_environments( try: if FEATURE_FLAG in ["cloud", "ee"]: has_permission = await check_action_access( - user_id=request.state.user_id, - object_uid=app_id, + user_uid=request.state.user_id, + object_id=app_id, object_type="app", permission=Permission.VIEW_APPLICATION, ) From 22427caf406945ac874e4bee6e7bb1a8329c7283 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 23 Jan 2024 19:08:22 +0100 Subject: [PATCH 37/99] rbac improvements --- .../agenta_backend/services/db_manager.py | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 345eb8e91f..9b98d87c9c 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -1,5 +1,6 @@ import os import logging +import traceback from pathlib import Path from datetime import datetime @@ -23,7 +24,6 @@ Result, ConfigDB, TemplateDB, - DeploymentDB, ConfigVersionDB, AggregatedResult, EvaluationScenarioResult, @@ -34,11 +34,9 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.services import db_manager_ee - - from agenta_backend.commons.models.api.api_models import ( - AppVariant_ as AppVariant, - ImageExtended_ as ImageExtended, - ) + from agenta_backend.commons.models.db_models import Permission + from agenta_backend.commons.utils.permissions import check_rbac_permission + from agenta_backend.commons.services.selectors import get_user_org_and_workspace_id from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, @@ -47,6 +45,7 @@ TestSetDB_ as TestSetDB, AppVariantDB_ as AppVariantDB, EvaluationDB_ as EvaluationDB, + DeploymentDB_ as DeploymentDB, VariantBaseDB_ as VariantBaseDB, AppEnvironmentDB_ as AppEnvironmentDB, EvaluatorConfigDB_ as EvaluatorConfigDB, @@ -54,11 +53,8 @@ EvaluationScenarioDB_ as EvaluationScenarioDB, HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, ) + else: - from agenta_backend.models.api.api_models import ( - AppVariant, - ImageExtended, - ) from agenta_backend.models.db_models import ( AppDB, @@ -67,6 +63,7 @@ TestSetDB, AppVariantDB, EvaluationDB, + DeploymentDB, VariantBaseDB, AppEnvironmentDB, EvaluatorConfigDB, @@ -427,21 +424,24 @@ async def create_deployment( Returns: DeploymentDB: The created deployment. """ - deployment = DeploymentDB( - app=app, - user=user, - container_name=container_name, - container_id=container_id, - uri=uri, - status=status, - ) + try: + deployment = DeploymentDB( + app=app, + user=user, + container_name=container_name, + container_id=container_id, + uri=uri, + status=status, + ) - if FEATURE_FLAG in ["cloud", "ee"]: - deployment.organization = organization - deployment.workspace = workspace + if FEATURE_FLAG in ["cloud", "ee"]: + deployment.organization = organization + deployment.workspace = workspace - await deployment.create() - return deployment + await deployment.create() + return deployment + except Exception as e: + raise Exception(f"Error while creating deployment: {e}") async def create_app_and_envs( @@ -818,38 +818,38 @@ async def list_apps( ) return [converters.app_db_to_pydantic(app_db)] - # elif (org_id is not None) or (workspace_id is not None): # TODO: Remember to enable this when workspace_id is provided after the RBAC is implemented - # if not FEATURE_FLAG in ["cloud", "ee"]: - # return JSONResponse( - # {"error": "organization and/or workspace is only available in Cloud and EE"}, - # status_code=400, - # ) - - # # assert that if org_id is provided, workspace_id is also provided, and vice versa - # assert ( - # org_id is not None and workspace_id is not None - # ), "org_id and workspace_id must be provided together" - - # user_org_workspace_data = await get_user_org_and_workspace_id(user_uid) - # has_permission = await check_rbac_permission( - # user_org_workspace_data=user_org_workspace_data, - # workspace_id=ObjectId(workspace_id), - # organization_id=ObjectId(org_id), - # permission=Permission.VIEW_APPLICATION, - # ) - # logger.debug(f"User has Permission to list apps: {has_permission}") - # if not has_permission: - # error_msg = f"You do not have access to perform this action. Please contact your organization admin." - # return JSONResponse( - # {"detail": error_msg}, - # status_code=400, - # ) - - # apps: List[AppDB] = await AppDB.find( - # AppDB.organization.id == ObjectId(org_id), - # AppDB.workspace.id == ObjectId(workspace_id), - # ).to_list() - # return [converters.app_db_to_pydantic(app) for app in apps] + elif (org_id is not None) or (workspace_id is not None): + if not FEATURE_FLAG in ["cloud", "ee"]: + return JSONResponse( + {"error": "organization and/or workspace is only available in Cloud and EE"}, + status_code=400, + ) + + # assert that if org_id is provided, workspace_id is also provided, and vice versa + assert ( + org_id is not None and workspace_id is not None + ), "org_id and workspace_id must be provided together" + + user_org_workspace_data = await get_user_org_and_workspace_id(user_uid) + has_permission = await check_rbac_permission( + user_org_workspace_data=user_org_workspace_data, + workspace_id=ObjectId(workspace_id), + organization_id=ObjectId(org_id), + permission=Permission.VIEW_APPLICATION, + ) + logger.debug(f"User has Permission to list apps: {has_permission}") + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=400, + ) + + apps: List[AppDB] = await AppDB.find( + AppDB.organization.id == ObjectId(org_id), + AppDB.workspace.id == ObjectId(workspace_id), + ).to_list() + return [converters.app_db_to_pydantic(app) for app in apps] else: apps = await AppDB.find(AppDB.user.id == user.id).to_list() From d1e51b902a2743bc4d6cfe2fa8437101632702df Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Wed, 24 Jan 2024 15:32:12 +0500 Subject: [PATCH 38/99] bug fixes and removed org id from everywhere --- .gitignore | 5 +- .../agenta_backend/routers/app_router.py | 8 +- .../agenta_backend/services/app_manager.py | 7 +- .../agenta_backend/services/db_manager.py | 25 +++--- .../services/deployment_manager.py | 8 +- .../agenta_backend/services/docker_utils.py | 13 +-- .../src/components/AppSelector/AppCard.tsx | 9 +-- .../components/AppSelector/AppSelector.tsx | 3 - agenta-web/src/components/Sidebar/Sidebar.tsx | 80 ++---------------- agenta-web/src/contexts/profile.context.tsx | 81 ++----------------- agenta-web/src/lib/Types.ts | 8 -- agenta-web/src/lib/helpers/utils.ts | 15 ---- agenta-web/src/lib/services/api.ts | 17 +--- 13 files changed, 64 insertions(+), 215 deletions(-) diff --git a/.gitignore b/.gitignore index b32eb68f0d..a27893b1a4 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,7 @@ agenta-web/cypress/screenshots/ agenta-web/cypress/videos/ .nextjs_cache/ -rabbitmq_data \ No newline at end of file +rabbitmq_data + +# docker compose override +docker-compose.*override.yaml \ No newline at end of file diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 022907bc1b..cd0dd4cbd6 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -391,11 +391,14 @@ async def remove_app(app_id: str, request: Request): ) else: - await app_manager.remove_app(app_id=app_id, user_uid=request.state.user_id) + await app_manager.remove_app(app_id=app_id) except DockerException as e: detail = f"Docker error while trying to remove the app: {str(e)}" raise HTTPException(status_code=500, detail=detail) except Exception as e: + import traceback + + traceback.print_exc() detail = f"Unexpected error while trying to remove the app: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -567,6 +570,9 @@ async def create_app_and_variant_from_template( return await converters.app_variant_db_to_output(app_variant_db) except Exception as e: + import traceback + + traceback.print_exc() logger.debug(f"Error: Exception caught - {str(e)}") raise HTTPException(status_code=500, detail=str(e)) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 8dcc559a05..5ffb589329 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -70,8 +70,8 @@ async def start_variant( db_app_variant.image.docker_id, db_app_variant.image.tags, db_app_variant.app.app_name, - db_app_variant.organization, - db_app_variant.workspace, + db_app_variant.organization if FEATURE_FLAG in ["cloud", "ee"] else None, + db_app_variant.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, ) logger.debug("App name is %s", db_app_variant.app.app_name) # update the env variables @@ -99,6 +99,9 @@ async def start_variant( deployment=deployment.id, ) except Exception as e: + import traceback + + traceback.print_exc() logger.error( f"Error starting Docker container for app variant {db_app_variant.app.app_name}/{db_app_variant.variant_name}: {str(e)}" ) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 9b98d87c9c..ecabde1609 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -53,9 +53,8 @@ EvaluationScenarioDB_ as EvaluationScenarioDB, HumanEvaluationScenarioDB_ as HumanEvaluationScenarioDB, ) - -else: +else: from agenta_backend.models.db_models import ( AppDB, UserDB, @@ -821,7 +820,9 @@ async def list_apps( elif (org_id is not None) or (workspace_id is not None): if not FEATURE_FLAG in ["cloud", "ee"]: return JSONResponse( - {"error": "organization and/or workspace is only available in Cloud and EE"}, + { + "error": "organization and/or workspace is only available in Cloud and EE" + }, status_code=400, ) @@ -884,11 +885,17 @@ async def check_is_last_variant_for_image(db_app_variant: AppVariantDB) -> bool: true if it's the last variant, false otherwise """ - count_variants = await AppVariantDB.find( - AppVariantDB.organization.id == db_app_variant.organization.id, - AppVariantDB.workspace.id == db_app_variant.workspace.id, - AppVariantDB.base.id == db_app_variant.base.id, - ).count() + query_expression = {"base.id": db_app_variant.base.id} + + if FEATURE_FLAG in ["cloud", "ee"]: + query_expression.update( + { + "organization.id": db_app_variant.organization.id, + "workspace.id": db_app_variant.workspace.id, + } + ) + + count_variants = await AppVariantDB.find(query_expression).count() return count_variants == 1 @@ -1548,7 +1555,7 @@ async def fetch_app_by_name_and_parameters( else: query_expression.update( { - "user.id": ObjectId(user_uid), + "user.uid": user_uid, } ) diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index 58eef14274..1da1a8db1f 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -28,8 +28,12 @@ async def start_service( True if successful, False otherwise. """ - uri_path = f"{app_variant_db.organization.id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" - container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.organization.id}" + if FEATURE_FLAG in ["cloud", "ee"]: + uri_path = f"{app_variant_db.organization.id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" + container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.organization.id}" + else: + uri_path = f"{app_variant_db.user.id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" + container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.user.id}" logger.debug("Starting service with the following parameters:") logger.debug(f"image_name: {app_variant_db.image.tags}") logger.debug(f"uri_path: {uri_path}") diff --git a/agenta-backend/agenta_backend/services/docker_utils.py b/agenta-backend/agenta_backend/services/docker_utils.py index df5623f14b..9dd8d5ad9c 100644 --- a/agenta-backend/agenta_backend/services/docker_utils.py +++ b/agenta-backend/agenta_backend/services/docker_utils.py @@ -63,11 +63,11 @@ def list_images() -> List[Image]: def start_container( image_name: str, uri_path: str, container_name: str, env_vars: DockerEnvVars ) -> Dict: - logger.debug("Starting container with the following parameters:") - logger.debug(f"image_name: {image_name}") - logger.debug(f"uri_path: {uri_path}") - logger.debug(f"container_name: {container_name}") - logger.debug(f"env_vars: {env_vars}") + print("Starting container with the following parameters:") + print(f"image_name: {image_name}") + print(f"uri_path: {uri_path}") + print(f"container_name: {container_name}") + print(f"env_vars: {env_vars}") try: image = client.images.get(f"{image_name}") @@ -134,6 +134,9 @@ def start_container( logs = failed_container.logs().decode("utf-8") raise Exception(f"Docker Logs: {logs}") from error except Exception as e: + import traceback + + traceback.print_exc() logger.error( f"Failed to fetch logs: {str(e)} \n Exception Error: {str(error)}" ) diff --git a/agenta-web/src/components/AppSelector/AppCard.tsx b/agenta-web/src/components/AppSelector/AppCard.tsx index a05652cc60..e6cec7519f 100644 --- a/agenta-web/src/components/AppSelector/AppCard.tsx +++ b/agenta-web/src/components/AppSelector/AppCard.tsx @@ -7,7 +7,6 @@ import {renameVariablesCapitalizeAll} from "@/lib/helpers/utils" import {createUseStyles} from "react-jss" import {getGradientFromStr} from "@/lib/helpers/colors" import {ListAppsItem} from "@/lib/Types" -import {useProfileData, Role} from "@/contexts/profile.context" import {useAppsData} from "@/contexts/app.context" const useStyles = createUseStyles({ @@ -78,8 +77,6 @@ const AppCard: React.FC<{ }> = ({app}) => { const [visibleDelete, setVisibleDelete] = useState(false) const [confirmLoading, setConfirmLoading] = useState(false) - const {role} = useProfileData() - const isOwner = role === Role.OWNER const {mutate} = useAppsData() const showDeleteModal = () => { @@ -108,11 +105,7 @@ const AppCard: React.FC<{ <> ] - : undefined - } + actions={[]} > { - const router = useRouter() const posthog = usePostHogAg() const {appTheme} = useAppTheme() const classes = useStyles({themeMode: appTheme} as StyleProps) @@ -111,7 +110,6 @@ const AppSelector: React.FC = () => { const [statusModalOpen, setStatusModalOpen] = useState(false) const [fetchingTemplate, setFetchingTemplate] = useState(false) const [newApp, setNewApp] = useState("") - const {selectedOrg} = useProfileData() const {apps, error, isLoading, mutate} = useAppsData() const [statusData, setStatusData] = useState<{status: string; details?: any; appId?: string}>({ status: "", @@ -190,7 +188,6 @@ const AppSelector: React.FC = () => { await createAndStartTemplate({ appName: newApp, templateId: template_id, - orgId: selectedOrg?.id!, providerKey: isDemo() ? "" : getApikeys(), timeout, onStatusChange: async (status, details, appId) => { diff --git a/agenta-web/src/components/Sidebar/Sidebar.tsx b/agenta-web/src/components/Sidebar/Sidebar.tsx index 5ad3ba8d0d..e5ac743e14 100644 --- a/agenta-web/src/components/Sidebar/Sidebar.tsx +++ b/agenta-web/src/components/Sidebar/Sidebar.tsx @@ -10,10 +10,9 @@ import { PhoneOutlined, SettingOutlined, LogoutOutlined, - ApartmentOutlined, FormOutlined, } from "@ant-design/icons" -import {Layout, Menu, Space, Tooltip, theme, Avatar} from "antd" +import {Layout, Menu, Space, Tooltip, theme} from "antd" import Logo from "../Logo/Logo" import Link from "next/link" @@ -22,8 +21,7 @@ import {ErrorBoundary} from "react-error-boundary" import {createUseStyles} from "react-jss" import AlertPopup from "../AlertPopup/AlertPopup" import {useProfileData} from "@/contexts/profile.context" -import {getColorFromStr} from "@/lib/helpers/colors" -import {getInitials, isDemo} from "@/lib/helpers/utils" +import {dynamicComponent, isDemo} from "@/lib/helpers/utils" import {useSession} from "@/hooks/useSession" type StyleProps = { @@ -76,44 +74,10 @@ const useStyles = createUseStyles({ menuLinks: { width: "100%", }, - menuItemNoBg: { - textOverflow: "unset !important", - "& .ant-menu-submenu-title": {display: "flex", alignItems: "center"}, - "& .ant-select-selector": { - padding: "0 !important", - }, - "&> span": { - display: "inline-block", - marginTop: 4, - }, - "& .ant-select-selection-item": { - "&> span > span": { - width: 120, - marginRight: 10, - }, - }, - }, - orgLabel: { - display: "flex", - alignItems: "center", - gap: "6px", - justifyContent: "flex-start", - "&> div": { - width: 18, - height: 18, - aspectRatio: "1/1", - borderRadius: "50%", - }, - "&> span": { - overflow: "hidden", - textOverflow: "ellipsis", - whiteSpace: "nowrap", - }, - }, }) const Sidebar: React.FC = () => { - const {appTheme, toggleAppTheme} = useAppTheme() + const {appTheme} = useAppTheme() const { token: {colorBgContainer}, } = theme.useToken() @@ -137,7 +101,7 @@ const Sidebar: React.FC = () => { initialSelectedKeys = ["apps"] } const [selectedKeys, setSelectedKeys] = useState(initialSelectedKeys) - const {user, orgs, selectedOrg, changeSelectedOrg, reset} = useProfileData() + const {user} = useProfileData() const [collapsed, setCollapsed] = useState(false) useEffect(() => { @@ -160,6 +124,8 @@ const Sidebar: React.FC = () => { }) } + const OrgsListSubMenu = dynamicComponent("OrgsListSubMenu/OrgsListSubMenu") + return (
{ Book Onboarding Call - {selectedOrg && ( - } - > - {orgs.map((org, index) => ( - - {getInitials(org.name)} - - } - onClick={() => changeSelectedOrg(org.id)} - > - {org.name} - - ))} - - )} + {user?.username && ( void) => void reset: () => void refetch: (onSuccess?: () => void) => void } const initialValues: ProfileContextType = { user: null, - orgs: [], - selectedOrg: null, - role: null, loading: false, - changeSelectedOrg: () => {}, reset: () => {}, refetch: () => {}, } @@ -56,27 +30,16 @@ export const getProfileValues = () => profileContextValues const ProfileContextProvider: React.FC = ({children}) => { const posthog = usePostHogAg() - const router = useRouter() - const [user, setUser] = useState(null) - const [orgs, setOrgs] = useState([]) - const [selectedOrg, setSelectedOrg] = useStateCallback(null) + const [user, setUser] = useStateCallback(null) const [loading, setLoading] = useState(false) const {logout, doesSessionExist} = useSession() const fetcher = useCallback((onSuccess?: () => void) => { setLoading(true) - Promise.all([getProfile(), getOrgsList()]) - .then(([profile, orgs]) => { + getProfile() + .then((profile) => { posthog.identify() - setUser(profile.data) - setOrgs(orgs.data) - setSelectedOrg( - orgs.data.find((org: Org) => org.id === localStorage.getItem(LS_ORG_KEY)) || - orgs.data.find((org: Org) => org.owner === profile.data.id) || - orgs.data[0] || - null, - onSuccess, - ) + setUser(profile.data, onSuccess) }) .catch((error) => { console.error(error) @@ -85,52 +48,24 @@ const ProfileContextProvider: React.FC = ({children}) => { .finally(() => setLoading(false)) }, []) - useUpdateEffect(() => { - localStorage.setItem(LS_ORG_KEY, selectedOrg?.id || "") - }, [selectedOrg?.id]) - useEffect(() => { - // fetch profile and orgs list only if user is logged in + // fetch profile only if user is logged in if (doesSessionExist) { fetcher() } }, [doesSessionExist]) - const changeSelectedOrg: ProfileContextType["changeSelectedOrg"] = (orgId, onSuccess) => { - setSelectedOrg( - orgs.find((org) => org.id === orgId) || selectedOrg, - onSuccess || - (() => { - router.push("/apps") - }), - ) - } - const reset = () => { setUser(initialValues.user) - setOrgs(initialValues.orgs) - setSelectedOrg(initialValues.selectedOrg) } - const role = useMemo( - () => (loading ? null : selectedOrg?.owner === user?.id ? Role.OWNER : Role.MEMBER), - [selectedOrg, user, loading], - ) - profileContextValues.user = user - profileContextValues.orgs = orgs - profileContextValues.selectedOrg = selectedOrg - profileContextValues.changeSelectedOrg = changeSelectedOrg return ( @@ -280,13 +279,6 @@ export interface User { email: string } -export interface Org { - id: string - name: string - description?: string - owner: string -} - export enum ChatRole { System = "system", User = "user", diff --git a/agenta-web/src/lib/helpers/utils.ts b/agenta-web/src/lib/helpers/utils.ts index 5c53716bb6..6c3c13720b 100644 --- a/agenta-web/src/lib/helpers/utils.ts +++ b/agenta-web/src/lib/helpers/utils.ts @@ -181,21 +181,6 @@ export const stringToNumberInRange = (text: string, min: number, max: number) => return result } -export const getInitials = (str: string, limit = 2) => { - let initialText = "E" - - try { - initialText = str - ?.split(" ") - .slice(0, limit) - ?.reduce((acc, curr) => acc + (curr[0] || "")?.toUpperCase(), "") - } catch (error) { - console.log("Error using getInitials", error) - } - - return initialText -} - export const isDemo = () => { if (process.env.NEXT_PUBLIC_FF) { return ["cloud", "ee"].includes(process.env.NEXT_PUBLIC_FF) diff --git a/agenta-web/src/lib/services/api.ts b/agenta-web/src/lib/services/api.ts index 229a3cb259..f31452816f 100644 --- a/agenta-web/src/lib/services/api.ts +++ b/agenta-web/src/lib/services/api.ts @@ -527,16 +527,12 @@ export const updateEvaluationScenarioScore = async ( } export const useApps = () => { - const {selectedOrg} = useProfileData() - const {data, error, isLoading, mutate} = useSWR( - `${getAgentaApiUrl()}/api/apps/?org_id=${selectedOrg?.id}`, - selectedOrg?.id ? fetcher : () => {}, //doon't fetch if org is not selected - ) + const {data, error, isLoading, mutate} = useSWR(`${getAgentaApiUrl()}/api/apps/`, fetcher) return { data: (data || []) as ListAppsItem[], error, - isLoading: selectedOrg?.id ? isLoading : true, + isLoading, mutate, } } @@ -547,12 +543,6 @@ export const getProfile = async (ignoreAxiosError: boolean = false) => { } as any) } -export const getOrgsList = async (ignoreAxiosError: boolean = false) => { - return axios.get(`${getAgentaApiUrl()}/api/organizations/`, { - _ignoreError: ignoreAxiosError, - } as any) -} - export const getTemplates = async () => { const response = await axios.get(`${getAgentaApiUrl()}/api/containers/templates/`) return response.data @@ -606,14 +596,12 @@ export const createAndStartTemplate = async ({ appName, providerKey, templateId, - orgId, timeout, onStatusChange, }: { appName: string providerKey: string templateId: string - orgId: string timeout?: number onStatusChange?: ( status: "creating_app" | "starting_app" | "success" | "bad_request" | "timeout" | "error", @@ -632,7 +620,6 @@ export const createAndStartTemplate = async ({ env_vars: { OPENAI_API_KEY: providerKey, }, - organization_id: orgId, }, true, ) From 495ede5772c757d2cbd1afb6596d4e6379ca737c Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 24 Jan 2024 12:58:10 +0100 Subject: [PATCH 39/99] Fix for non-org use in oss --- .../agenta_backend/models/converters.py | 30 +++++++++++++------ .../agenta_backend/services/app_manager.py | 2 +- .../agenta_backend/services/db_manager.py | 8 +++-- .../agenta_backend/services/docker_utils.py | 10 +++---- 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 81471b967e..5f932717da 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -221,17 +221,21 @@ def evaluation_scenario_db_to_pydantic( def app_variant_db_to_pydantic( app_variant_db: AppVariantDB, previous_variant_name: str = None ) -> AppVariant: - return AppVariant( + app_variant = AppVariant( app_id=str(app_variant_db.app.id), app_name=app_variant_db.app.app_name, variant_name=app_variant_db.variant_name, parameters=app_variant_db.config.parameters, previous_variant_name=app_variant_db.previous_variant_name, - organization_id=str(app_variant_db.organization.id), - workspace_id=str(app_variant_db.workspace.id), base_name=app_variant_db.base_name, config_name=app_variant_db.config_name, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + app_variant.organization_id = str(app_variant_db.organization.id) + app_variant.workspace_id = str(app_variant_db.workspace.id) + + return app_variant async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantResponse: @@ -244,14 +248,12 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantRe deployment = None uri = None logger.info(f"uri: {uri} deployment: {app_variant_db.base.deployment} {deployment}") - return AppVariantResponse( + variant_response = AppVariantResponse( app_id=str(app_variant_db.app.id), app_name=str(app_variant_db.app.app_name), variant_name=app_variant_db.variant_name, variant_id=str(app_variant_db.id), user_id=str(app_variant_db.user.id), - organization_id=str(app_variant_db.organization.id), - workspace_id=str(app_variant_db.workspace.id), parameters=app_variant_db.config.parameters, previous_variant_name=app_variant_db.previous_variant_name, base_name=app_variant_db.base_name, @@ -260,6 +262,12 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantRe config_id=str(app_variant_db.config.id), uri=uri, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + variant_response.organization_id = str(app_variant_db.organization.id) + variant_response.workspace_id = str(app_variant_db.workspace.id) + + return variant_response async def environment_db_to_output( @@ -293,13 +301,17 @@ def app_db_to_pydantic(app_db: AppDB) -> App: def image_db_to_pydantic(image_db: ImageDB) -> ImageExtended: - return ImageExtended( - organization_id=str(image_db.organization.id), - workspace_id=str(image_db.workspace.id), + image = ImageExtended( docker_id=image_db.docker_id, tags=image_db.tags, id=str(image_db.id), ) + + if FEATURE_FLAG in ["cloud", "ee"]: + image.organization_id = str(image_db.organization.id) + image.workspace_id = str(image_db.workspace.id) + + return image def templates_db_to_pydantic(templates_db: List[TemplateDB]) -> List[Template]: diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 5ffb589329..413be26683 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -64,7 +64,7 @@ async def start_variant( RuntimeError: If there is an error starting the Docker container. """ try: - logger.debug( + print( "Starting variant %s with image name %s and tags %s and app_name %s and organization %s", db_app_variant.variant_name, db_app_variant.image.docker_id, diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index ecabde1609..c643ed5dd9 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -385,6 +385,7 @@ async def create_image( image.template_uri = template_uri elif image_type == "image": image.type = "image" + image.tags = tags image.docker_id = docker_id if FEATURE_FLAG in ["cloud", "ee"]: @@ -775,8 +776,6 @@ async def add_variant_from_base_and_config( variant_name=new_variant_name, image=base_db.image, user=user_db, - organization=previous_app_variant_db.organization, - workspace=previous_app_variant_db.workspace, parameters=parameters, previous_variant_name=previous_app_variant_db.variant_name, # TODO: Remove in future base_name=base_db.base_name, @@ -785,6 +784,11 @@ async def add_variant_from_base_and_config( config=config_db, is_deleted=False, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + db_app_variant.organization = previous_app_variant_db.organization + db_app_variant.workspace = previous_app_variant_db.workspace + await db_app_variant.create() return db_app_variant diff --git a/agenta-backend/agenta_backend/services/docker_utils.py b/agenta-backend/agenta_backend/services/docker_utils.py index 9dd8d5ad9c..9fced69465 100644 --- a/agenta-backend/agenta_backend/services/docker_utils.py +++ b/agenta-backend/agenta_backend/services/docker_utils.py @@ -63,11 +63,11 @@ def list_images() -> List[Image]: def start_container( image_name: str, uri_path: str, container_name: str, env_vars: DockerEnvVars ) -> Dict: - print("Starting container with the following parameters:") - print(f"image_name: {image_name}") - print(f"uri_path: {uri_path}") - print(f"container_name: {container_name}") - print(f"env_vars: {env_vars}") + logger.debug("Starting container with the following parameters:") + logger.debug(f"image_name: {image_name}") + logger.debug(f"uri_path: {uri_path}") + logger.debug(f"container_name: {container_name}") + logger.debug(f"env_vars: {env_vars}") try: image = client.images.get(f"{image_name}") From da862e2aa70b27d73df263622e8d0cd70d473074 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 24 Jan 2024 13:01:54 +0100 Subject: [PATCH 40/99] format black --- agenta-backend/agenta_backend/main.py | 18 ++++++++++++++---- .../agenta_backend/models/converters.py | 12 ++++++------ .../agenta_backend/services/db_manager.py | 4 ++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/agenta-backend/agenta_backend/main.py b/agenta-backend/agenta_backend/main.py index 362e912ba7..04c98bb5ae 100644 --- a/agenta-backend/agenta_backend/main.py +++ b/agenta-backend/agenta_backend/main.py @@ -84,13 +84,23 @@ async def lifespan(application: FastAPI, cache=True): app.include_router(user_profile.router, prefix="/profile") app.include_router(app_router.router, prefix="/apps", tags=["Apps"]) app.include_router(variants_router.router, prefix="/variants", tags=["Variants"]) -app.include_router(evaluation_router.router, prefix="/evaluations", tags=["Evaluations"]) -app.include_router(human_evaluation_router.router, prefix="/human-evaluations", tags=["Human-Evaluations"]) +app.include_router( + evaluation_router.router, prefix="/evaluations", tags=["Evaluations"] +) +app.include_router( + human_evaluation_router.router, + prefix="/human-evaluations", + tags=["Human-Evaluations"], +) app.include_router(evaluators_router.router, prefix="/evaluators", tags=["Evaluators"]) app.include_router(testset_router.router, prefix="/testsets", tags=["Testsets"]) app.include_router(container_router.router, prefix="/containers", tags=["Containers"]) -app.include_router(environment_router.router, prefix="/environments", tags=["Environments"]) -app.include_router(observability_router.router, prefix="/observability", tags=["Observability"]) +app.include_router( + environment_router.router, prefix="/environments", tags=["Environments"] +) +app.include_router( + observability_router.router, prefix="/observability", tags=["Observability"] +) app.include_router(bases_router.router, prefix="/bases", tags=["Bases"]) app.include_router(configs_router.router, prefix="/configs", tags=["Configs"]) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 5f932717da..9f7e7649da 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -230,11 +230,11 @@ def app_variant_db_to_pydantic( base_name=app_variant_db.base_name, config_name=app_variant_db.config_name, ) - + if FEATURE_FLAG in ["cloud", "ee"]: app_variant.organization_id = str(app_variant_db.organization.id) app_variant.workspace_id = str(app_variant_db.workspace.id) - + return app_variant @@ -262,11 +262,11 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantRe config_id=str(app_variant_db.config.id), uri=uri, ) - + if FEATURE_FLAG in ["cloud", "ee"]: variant_response.organization_id = str(app_variant_db.organization.id) variant_response.workspace_id = str(app_variant_db.workspace.id) - + return variant_response @@ -306,11 +306,11 @@ def image_db_to_pydantic(image_db: ImageDB) -> ImageExtended: tags=image_db.tags, id=str(image_db.id), ) - + if FEATURE_FLAG in ["cloud", "ee"]: image.organization_id = str(image_db.organization.id) image.workspace_id = str(image_db.workspace.id) - + return image diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index c643ed5dd9..f9850c621a 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -784,11 +784,11 @@ async def add_variant_from_base_and_config( config=config_db, is_deleted=False, ) - + if FEATURE_FLAG in ["cloud", "ee"]: db_app_variant.organization = previous_app_variant_db.organization db_app_variant.workspace = previous_app_variant_db.workspace - + await db_app_variant.create() return db_app_variant From b209a90ba1d840645b2714caa524d1616e836450 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Thu, 25 Jan 2024 09:02:49 +0100 Subject: [PATCH 41/99] fix bugs --- agenta-backend/agenta_backend/services/db_manager.py | 2 +- .../agenta_backend/services/evaluation_service.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index f9850c621a..dcb802557b 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -1695,7 +1695,7 @@ async def fetch_evaluator_config(evaluator_config_id: str): try: evaluator_config: EvaluatorConfigDB = await EvaluatorConfigDB.find_one( - EvaluatorConfigDB.id == ObjectId(evaluator_config_id, fetch_links=True) + EvaluatorConfigDB.id == ObjectId(evaluator_config_id), fetch_links=True ) return evaluator_config except Exception as e: diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 6b3389ad04..bb5ba651a4 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -542,8 +542,6 @@ async def create_new_human_evaluation( # Initialize and save evaluation instance to database eval_instance = HumanEvaluationDB( app=app, - organization=app.organization, # Assuming user has an organization_id attribute - workspace=app.workspace, user=user, status=payload.status, evaluation_type=payload.evaluation_type, @@ -552,6 +550,11 @@ async def create_new_human_evaluation( created_at=current_time, updated_at=current_time, ) + + if FEATURE_FLAG in ["cloud", "ee"]: + eval_instance.organization = app.organization + eval_instance.workspace = app.workspace + newEvaluation = await eval_instance.create() if newEvaluation is None: raise HTTPException( From 44c9a97a7bdfe9bcbd234f1ed5dfe005301f7b3a Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Thu, 25 Jan 2024 09:06:41 +0100 Subject: [PATCH 42/99] add org & workspace in cloud --- .../agenta_backend/services/evaluation_service.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index bb5ba651a4..f8ea4b7c3d 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -160,12 +160,15 @@ async def prepare_csvdata_and_create_evaluation_scenario( eval_scenario_instance = HumanEvaluationScenarioDB( **evaluation_scenario_payload, user=user, - organization=app.organization, - workspace=app.workspace, evaluation=new_evaluation, inputs=list_of_scenario_input, outputs=[], ) + + if FEATURE_FLAG in ["cloud", "ee"]: + eval_scenario_instance.organization = app.organization + eval_scenario_instance.workspace = app.workspace + await eval_scenario_instance.create() From 2c27771db36df56df07ad3585b1a26889acd2698 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Thu, 25 Jan 2024 14:05:44 +0100 Subject: [PATCH 43/99] Refactor tests in line with RBAC: remove organizations --- .../tests/variants_main_router/conftest.py | 29 ++--------- .../test_app_variant_router.py | 12 +---- .../test_observability_router.py | 12 +---- .../test_variant_evaluators_router.py | 1 - .../test_organization_router.py | 50 ------------------- 5 files changed, 6 insertions(+), 98 deletions(-) delete mode 100644 agenta-backend/agenta_backend/tests/variants_organization_router/test_organization_router.py diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py b/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py index 2401f352b3..8205447869 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py @@ -10,9 +10,7 @@ ImageDB, ConfigDB, AppVariantDB, - OrganizationDB, ) -from agenta_backend.services import selectors import httpx @@ -39,12 +37,6 @@ async def get_first_user_object(): create_user = UserDB(uid="0") await create_user.create() - org = OrganizationDB(type="default", owner=str(create_user.id)) - await org.create() - - create_user.organizations.append(org.id) - await create_user.save() - return create_user return user @@ -60,12 +52,6 @@ async def get_second_user_object(): ) await create_user.create() - org = OrganizationDB(type="default", owner=str(create_user.id)) - await org.create() - - create_user.organizations.append(org.id) - await create_user.save() - return create_user return user @@ -73,16 +59,14 @@ async def get_second_user_object(): @pytest.fixture() async def get_first_user_app(get_first_user_object): user = await get_first_user_object - organization = await selectors.get_user_own_org(user.uid) - app = AppDB(app_name="myapp", organization=organization, user=user) + app = AppDB(app_name="myapp", user=user) await app.create() db_image = ImageDB( docker_id="sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", tags="agentaai/templates_v2:local_test_prompt", user=user, - organization=organization, ) await db_image.create() @@ -93,7 +77,7 @@ async def get_first_user_app(get_first_user_object): await db_config.create() db_base = VariantBaseDB( - base_name="app", image=db_image, organization=organization, user=user, app=app + base_name="app", image=db_image, user=user, app=app ) await db_base.create() @@ -102,7 +86,6 @@ async def get_first_user_app(get_first_user_object): variant_name="app", image=db_image, user=user, - organization=organization, parameters={}, base_name="app", config_name="default", @@ -110,7 +93,7 @@ async def get_first_user_app(get_first_user_object): config=db_config, ) await appvariant.create() - return appvariant, user, organization, app, db_image, db_config, db_base + return appvariant, user, app, db_image, db_config, db_base @pytest.fixture() @@ -231,12 +214,6 @@ def fetch_single_prompt_template(fetch_templates): return fetch_templates[1] -@pytest.fixture() -async def fetch_user_organization(): - organization = await OrganizationDB.find().to_list() - return {"org_id": str(organization[0].id)} - - @pytest.fixture() def app_from_template(): return { diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py index f75a8c9b3d..c8ca74efa7 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py @@ -5,7 +5,7 @@ from bson import ObjectId from agenta_backend.routers import app_router -from agenta_backend.services import selectors, db_manager +from agenta_backend.services import db_manager from agenta_backend.models.db_models import ( AppDB, VariantBaseDB, @@ -36,13 +36,11 @@ @pytest.mark.asyncio async def test_create_app(get_first_user_object): user = await get_first_user_object - organization = await selectors.get_user_own_org(user.uid) response = await test_client.post( f"{BACKEND_API_HOST}/apps/", json={ "app_name": "app_variant_test", - "organization_id": str(organization.id), }, timeout=timeout, ) @@ -61,14 +59,12 @@ async def test_list_apps(): @pytest.mark.asyncio async def test_create_app_variant(get_first_user_object): user = await get_first_user_object - organization = await selectors.get_user_own_org(user.uid) app = await AppDB.find_one(AppDB.app_name == "app_variant_test") db_image = ImageDB( docker_id="sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", tags="agentaai/templates_v2:local_test_prompt", user=user, - organization=organization, ) await db_image.create() @@ -81,7 +77,6 @@ async def test_create_app_variant(get_first_user_object): db_base = VariantBaseDB( base_name="app", app=app, - organization=organization, user=user, image=db_image, ) @@ -92,7 +87,6 @@ async def test_create_app_variant(get_first_user_object): variant_name="app", image=db_image, user=user, - organization=organization, parameters={}, base_name="app", config_name="default", @@ -120,11 +114,9 @@ async def test_list_app_variants(): @pytest.mark.asyncio async def test_delete_app_without_permission(get_second_user_object): user2 = await get_second_user_object - user2_organization = await selectors.get_user_own_org(user2.uid) user2_app = AppDB( app_name="test_app_by_user2", - organization=user2_organization, user=user2, ) await user2_app.create() @@ -149,7 +141,7 @@ async def test_list_environments(): @pytest.mark.asyncio async def test_get_variant_by_env(get_first_user_app): - _, _, _, app, _, _, _ = await get_first_user_app + _, _, app, _, _, _ = await get_first_user_app environments = await db_manager.list_environments(app_id=str(app.id)) for environment in environments: diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_observability_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_observability_router.py index 1160e4d58c..51debb38bf 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_observability_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_observability_router.py @@ -9,13 +9,10 @@ SpanDB, UserDB, TraceDB, - OrganizationDB, ImageDB, AppVariantDB, VariantBaseDB, ) -from agenta_backend.services import selectors - import httpx @@ -44,11 +41,8 @@ async def test_create_spans_endpoint(spans_db_data): @pytest.mark.asyncio async def test_create_image_in_db(image_create_data): user_db = await UserDB.find_one(UserDB.uid == "0") - organization_db = await OrganizationDB.find_one( - OrganizationDB.owner == str(user_db.id) - ) - image_db = ImageDB(**image_create_data, user=user_db, organization=organization_db) + image_db = ImageDB(**image_create_data, user=user_db) await image_db.create() assert image_db.user.id == user_db.id @@ -58,12 +52,10 @@ async def test_create_image_in_db(image_create_data): @pytest.mark.asyncio async def test_create_appvariant_in_db(app_variant_create_data): user_db = await UserDB.find_one(UserDB.uid == "0") - organization_db = await selectors.get_user_own_org(user_db.uid) image_db = await ImageDB.find_one(ImageDB.user.id == user_db.id) app = AppDB( app_name="test_app", - organization=organization_db, user=user_db, ) await app.create() @@ -76,7 +68,6 @@ async def test_create_appvariant_in_db(app_variant_create_data): db_base = VariantBaseDB( app=app, - organization=organization_db, user=user_db, base_name="app", image=image_db, @@ -88,7 +79,6 @@ async def test_create_appvariant_in_db(app_variant_create_data): app=app, image=image_db, user=user_db, - organization=organization_db, base_name="app", config_name="default", base=db_base, diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py index d17603e211..3eaaf68e65 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py @@ -6,7 +6,6 @@ from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum from agenta_backend.models.db_models import ( AppDB, - ConfigDB, TestSetDB, AppVariantDB, EvaluationDB, diff --git a/agenta-backend/agenta_backend/tests/variants_organization_router/test_organization_router.py b/agenta-backend/agenta_backend/tests/variants_organization_router/test_organization_router.py deleted file mode 100644 index 222c9dbe9c..0000000000 --- a/agenta-backend/agenta_backend/tests/variants_organization_router/test_organization_router.py +++ /dev/null @@ -1,50 +0,0 @@ -import os - -from agenta_backend.services import selectors -from agenta_backend.models.db_models import UserDB -from agenta_backend.models.api.organization_models import OrganizationOutput - -import httpx -import pytest - - -# Initialize http client -test_client = httpx.AsyncClient() -timeout = httpx.Timeout(timeout=5, read=None, write=5) - -# Set global variables -ENVIRONMENT = os.environ.get("ENVIRONMENT") -if ENVIRONMENT == "development": - BACKEND_API_HOST = "http://host.docker.internal/api" -elif ENVIRONMENT == "github": - BACKEND_API_HOST = "http://agenta-backend-test:8000" - - -@pytest.mark.asyncio -async def test_list_organizations(): - response = await test_client.get(f"{BACKEND_API_HOST}/organizations/") - - assert response.status_code == 200 - assert len(response.json()) == 1 - - -@pytest.mark.asyncio -async def test_get_user_organization(): - user = await UserDB.find_one(UserDB.uid == "0") - user_org = await selectors.get_user_own_org(user.uid) - - response = await test_client.get(f"{BACKEND_API_HOST}/organizations/own/") - - assert response.status_code == 200 - assert response.json() == OrganizationOutput( - id=str(user_org.id), name=user_org.name - ) - - -@pytest.mark.asyncio -async def test_user_does_not_have_an_organization(): - user = UserDB(uid="0123", username="john_doe", email="johndoe@email.com") - await user.create() - - user_org = await selectors.get_user_own_org(user.uid) - assert user_org == None From c35864d83626208b5956385ac0b5dfcdeb18b710 Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Thu, 25 Jan 2024 18:12:26 +0500 Subject: [PATCH 44/99] permission denied modal and minor fixes --- agenta-web/src/lib/helpers/errorHandler.ts | 11 +++++++++++ agenta-web/src/lib/helpers/utils.ts | 7 +++++++ agenta-web/src/pages/settings/index.tsx | 15 +++++++-------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/agenta-web/src/lib/helpers/errorHandler.ts b/agenta-web/src/lib/helpers/errorHandler.ts index 8389005f04..dbe8ec69ee 100644 --- a/agenta-web/src/lib/helpers/errorHandler.ts +++ b/agenta-web/src/lib/helpers/errorHandler.ts @@ -1,3 +1,4 @@ +import AlertPopup from "@/components/AlertPopup/AlertPopup" import {message} from "antd" export const getErrorMessage = (error: any, fallback = "An unknown error occurred!") => { @@ -15,6 +16,16 @@ export const getErrorMessage = (error: any, fallback = "An unknown error occurre } export const globalErrorHandler = (error: any) => { + if (error.response?.status === 403) { + AlertPopup({ + title: "Permission Denied", + message: + "You don't have permission to perform this action. Please contact your organization admin.", + cancelText: null, + }) + return + } + const errorMsg = getErrorMessage(error) console.error(errorMsg, error) message.error(errorMsg) diff --git a/agenta-web/src/lib/helpers/utils.ts b/agenta-web/src/lib/helpers/utils.ts index 6c3c13720b..ded3297ae3 100644 --- a/agenta-web/src/lib/helpers/utils.ts +++ b/agenta-web/src/lib/helpers/utils.ts @@ -387,3 +387,10 @@ export const redirectIfNoLLMKeys = () => { } return false } + +export const snakeToTitle = (str: string) => { + return str + .split("_") + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(" ") +} diff --git a/agenta-web/src/pages/settings/index.tsx b/agenta-web/src/pages/settings/index.tsx index d2c7e43151..e4bc75a8d7 100644 --- a/agenta-web/src/pages/settings/index.tsx +++ b/agenta-web/src/pages/settings/index.tsx @@ -1,10 +1,9 @@ import Secrets from "@/components/pages/settings/Secrets/Secrets" import ProtectedRoute from "@/components/ProtectedRoute/ProtectedRoute" import {useQueryParam} from "@/hooks/useQuery" -import {isFeatureEnabled} from "@/lib/helpers/featureFlag" import {dynamicComponent, isDemo} from "@/lib/helpers/utils" import {ApartmentOutlined, KeyOutlined, LockOutlined} from "@ant-design/icons" -import {Tabs, Typography} from "antd" +import {Space, Tabs, Typography} from "antd" import {createUseStyles} from "react-jss" const useStyles = createUseStyles({ @@ -36,10 +35,10 @@ const Settings: React.FC = () => { const items = [ { label: ( - + Workspace - + ), key: "workspace", children: , @@ -47,20 +46,20 @@ const Settings: React.FC = () => { }, { label: ( - + LLM Keys - + ), key: "secrets", children: , }, { label: ( - + API Keys - + ), key: "apiKeys", children: , From c424049d72a0e66769e806ae40daedb96d57526c Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Thu, 25 Jan 2024 14:43:52 +0100 Subject: [PATCH 45/99] set rbac error status code to 403 --- .../agenta_backend/routers/app_router.py | 12 +-- .../routers/evaluators_router.py | 98 ++++++++++++++++--- .../routers/human_evaluation_router.py | 20 ++-- 3 files changed, 100 insertions(+), 30 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index cd0dd4cbd6..bf8d770efe 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -98,7 +98,7 @@ async def list_app_variants( error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) app_variants = await db_manager.list_app_variants(app_id=app_id) @@ -150,7 +150,7 @@ async def get_variant_by_env( error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) # Fetch the app variant using the provided app_id and environment @@ -242,7 +242,7 @@ async def create_app( error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -342,7 +342,7 @@ async def add_variant_from_image( error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) app = await db_manager.fetch_app_by_id(app_id) @@ -387,7 +387,7 @@ async def remove_app(app_id: str, request: Request): error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) else: @@ -610,7 +610,7 @@ async def list_environments( error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) environments_db = await db_manager.list_environments(app_id=app_id) diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index b1bbcbbc08..39cfe8eb2b 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -51,7 +51,7 @@ async def get_evaluators_endpoint(): @router.get("/configs/", response_model=List[EvaluatorConfig]) -async def get_evaluator_configs(app_id: str): +async def get_evaluator_configs(app_id: str, request: Request): """Endpoint to fetch evaluator configurations for a specific app. Args: @@ -62,6 +62,21 @@ async def get_evaluator_configs(app_id: str): """ try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=app_id, + object_type="app", + permission=Permission.VIEW_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + evaluators_configs = await evaluator_manager.get_evaluators_configs(app_id) return evaluators_configs except Exception as e: @@ -107,6 +122,7 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): @router.post("/configs/", response_model=EvaluatorConfig) async def create_new_evaluator_config( payload: NewEvaluatorConfig, + request: Request ): """Endpoint to fetch evaluator configurations for a specific app. @@ -116,19 +132,38 @@ async def create_new_evaluator_config( Returns: EvaluatorConfigDB: Evaluator configuration api model. """ + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=payload.app_id, + object_type="app", + permission=Permission.CREATE_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) - evaluator_config = await evaluator_manager.create_evaluator_config( - app_id=payload.app_id, - name=payload.name, - evaluator_key=payload.evaluator_key, - settings_values=payload.settings_values, - ) - return evaluator_config + evaluator_config = await evaluator_manager.create_evaluator_config( + app_id=payload.app_id, + name=payload.name, + evaluator_key=payload.evaluator_key, + settings_values=payload.settings_values, + ) + return evaluator_config + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Error creating evaluator configuration: {str(e)}" + ) @router.put("/configs/{evaluator_config_id}/", response_model=EvaluatorConfig) async def update_evaluator_config( - evaluator_config_id: str, payload: UpdateEvaluatorConfig + evaluator_config_id: str, payload: UpdateEvaluatorConfig, request: Request ): """Endpoint to update evaluator configurations for a specific app. @@ -136,14 +171,34 @@ async def update_evaluator_config( List[EvaluatorConfigDB]: A list of evaluator configuration objects. """ - evaluators_configs = await evaluator_manager.update_evaluator_config( - evaluator_config_id=evaluator_config_id, updates=payload - ) - return evaluators_configs + try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluator_config_id, + object_type="evaluator_config", + permission=Permission.EDIT_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + evaluators_configs = await evaluator_manager.update_evaluator_config( + evaluator_config_id=evaluator_config_id, updates=payload + ) + return evaluators_configs + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Error updating evaluator configuration: {str(e)}" + ) @router.delete("/configs/{evaluator_config_id}/", response_model=bool) -async def delete_evaluator_config(evaluator_config_id: str): +async def delete_evaluator_config(evaluator_config_id: str, request: Request): """Endpoint to delete a specific evaluator configuration. Args: @@ -153,6 +208,21 @@ async def delete_evaluator_config(evaluator_config_id: str): bool: True if deletion was successful, False otherwise. """ try: + if FEATURE_FLAG in ["cloud", "ee"]: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object_id=evaluator_config_id, + object_type="evaluator_config", + permission=Permission.DELETE_EVALUATION, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + success = await evaluator_manager.delete_evaluator_config(evaluator_config_id) return success except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index b2cb881411..de0b1b74c1 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -74,7 +74,7 @@ async def create_evaluation( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) app = await db_manager.fetch_app_by_id(app_id=payload.app_id) @@ -120,7 +120,7 @@ async def fetch_list_human_evaluations( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) return await evaluation_service.fetch_list_human_evaluations(app_id) @@ -153,7 +153,7 @@ async def fetch_human_evaluation( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) return await evaluation_service.fetch_human_evaluation(evaluation_id) @@ -194,7 +194,7 @@ async def fetch_evaluation_scenarios( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) eval_scenarios = ( @@ -234,7 +234,7 @@ async def update_human_evaluation( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) await update_human_evaluation_service(evaluation_id, update_data) @@ -277,7 +277,7 @@ async def update_evaluation_scenario_router( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) await update_human_evaluation_scenario( @@ -317,7 +317,7 @@ async def get_evaluation_scenario_score_router( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) return await get_evaluation_scenario_score_service(evaluation_scenario_id) @@ -351,7 +351,7 @@ async def update_evaluation_scenario_score_router( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) await update_evaluation_scenario_score_service( @@ -388,7 +388,7 @@ async def fetch_results( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) evaluation = await evaluation_service._fetch_human_evaluation(evaluation_id) @@ -433,7 +433,7 @@ async def delete_evaluations( error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) await evaluation_service.delete_human_evaluations( From b72261e54d2bf8210dbae95e919598f090bdd331 Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Thu, 25 Jan 2024 21:52:39 +0500 Subject: [PATCH 46/99] apps fetching separation for cloud and oss --- agenta-web/src/contexts/app.context.tsx | 14 +++++++++++++- agenta-web/src/lib/helpers/errorHandler.ts | 1 + agenta-web/src/lib/services/api.ts | 19 +++---------------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/agenta-web/src/contexts/app.context.tsx b/agenta-web/src/contexts/app.context.tsx index fdd1fca9e5..35db25f318 100644 --- a/agenta-web/src/contexts/app.context.tsx +++ b/agenta-web/src/contexts/app.context.tsx @@ -1,7 +1,9 @@ import {ListAppsItem} from "@/lib/Types" -import {useApps} from "@/lib/services/api" +import {getAgentaApiUrl} from "@/lib/helpers/utils" +import {axiosFetcher} from "@/lib/services/api" import {useRouter} from "next/router" import {PropsWithChildren, createContext, useContext, useMemo} from "react" +import useSWR from "swr" type AppContextType = { currentApp: ListAppsItem | null @@ -19,6 +21,16 @@ const initialValues: AppContextType = { mutate: () => {}, } +const useApps = () => { + const {data, error, isLoading, mutate} = useSWR(`${getAgentaApiUrl()}/api/apps/`, axiosFetcher) + return { + data: (data || []) as ListAppsItem[], + error, + isLoading, + mutate, + } +} + export const AppContext = createContext(initialValues) export const useAppsData = () => useContext(AppContext) diff --git a/agenta-web/src/lib/helpers/errorHandler.ts b/agenta-web/src/lib/helpers/errorHandler.ts index dbe8ec69ee..f2271205ee 100644 --- a/agenta-web/src/lib/helpers/errorHandler.ts +++ b/agenta-web/src/lib/helpers/errorHandler.ts @@ -22,6 +22,7 @@ export const globalErrorHandler = (error: any) => { message: "You don't have permission to perform this action. Please contact your organization admin.", cancelText: null, + okText: "Ok", }) return } diff --git a/agenta-web/src/lib/services/api.ts b/agenta-web/src/lib/services/api.ts index f31452816f..018d5f6c17 100644 --- a/agenta-web/src/lib/services/api.ts +++ b/agenta-web/src/lib/services/api.ts @@ -14,7 +14,6 @@ import { Environment, CreateCustomEvaluation, ExecuteCustomEvalCode, - ListAppsItem, AICritiqueCreate, ChatMessage, KeyValuePair, @@ -24,13 +23,12 @@ import { fromEvaluationScenarioResponseToEvaluationScenario, } from "../transformers" import {EvaluationFlow, EvaluationType} from "../enums" -import {delay, getAgentaApiUrl, removeKeys, shortPoll} from "../helpers/utils" -import {useProfileData} from "@/contexts/profile.context" +import {getAgentaApiUrl, removeKeys, shortPoll} from "../helpers/utils" /** * Raw interface for the parameters parsed from the openapi.json */ -const fetcher = (url: string) => axios.get(url).then((res) => res.data) +export const axiosFetcher = (url: string) => axios.get(url).then((res) => res.data) export async function fetchVariants( appId: string, @@ -250,7 +248,7 @@ export async function removeVariant(variantId: string) { export const useLoadTestsetsList = (appId: string) => { const {data, error, mutate, isLoading} = useSWR( `${getAgentaApiUrl()}/api/testsets/?app_id=${appId}`, - fetcher, + axiosFetcher, {revalidateOnFocus: false}, ) @@ -526,17 +524,6 @@ export const updateEvaluationScenarioScore = async ( return response } -export const useApps = () => { - const {data, error, isLoading, mutate} = useSWR(`${getAgentaApiUrl()}/api/apps/`, fetcher) - - return { - data: (data || []) as ListAppsItem[], - error, - isLoading, - mutate, - } -} - export const getProfile = async (ignoreAxiosError: boolean = false) => { return axios.get(`${getAgentaApiUrl()}/api/profile/`, { _ignoreError: ignoreAxiosError, From a011fb7828cc1787d0278417972d40fb6d8edf43 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Thu, 25 Jan 2024 19:46:14 +0100 Subject: [PATCH 47/99] fix delete testet permission --- agenta-backend/agenta_backend/routers/testset_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index e34bda070e..0a8c4fa040 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -441,7 +441,7 @@ async def delete_testsets( user_uid=request.state.user_id, object_id=testset_id, object_type="testset", - permission=Permission.VIEW_TESTSET, + permission=Permission.DELETE_TESTSET, ) logger.debug(f"User has Permission to delete Testset: {has_permission}") if not has_permission: From 5aca614ab52502ba71f085646cb82dfd25eaefd4 Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Tue, 30 Jan 2024 11:44:54 +0500 Subject: [PATCH 48/99] dynamic modules | pass org id when creating app --- agenta-web/src/components/Sidebar/Sidebar.tsx | 3 +- agenta-web/src/contexts/app.context.tsx | 22 +++++++++++-- agenta-web/src/lib/Types.ts | 1 + agenta-web/src/lib/helpers/dynamic.ts | 32 +++++++++++++++++++ agenta-web/src/lib/helpers/utils.ts | 8 ----- agenta-web/src/lib/services/api.ts | 8 +++++ agenta-web/src/pages/settings/index.tsx | 3 +- 7 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 agenta-web/src/lib/helpers/dynamic.ts diff --git a/agenta-web/src/components/Sidebar/Sidebar.tsx b/agenta-web/src/components/Sidebar/Sidebar.tsx index e5ac743e14..d8812351fe 100644 --- a/agenta-web/src/components/Sidebar/Sidebar.tsx +++ b/agenta-web/src/components/Sidebar/Sidebar.tsx @@ -21,8 +21,9 @@ import {ErrorBoundary} from "react-error-boundary" import {createUseStyles} from "react-jss" import AlertPopup from "../AlertPopup/AlertPopup" import {useProfileData} from "@/contexts/profile.context" -import {dynamicComponent, isDemo} from "@/lib/helpers/utils" +import {isDemo} from "@/lib/helpers/utils" import {useSession} from "@/hooks/useSession" +import {dynamicComponent} from "@/lib/helpers/dynamic" type StyleProps = { themeMode: "system" | "dark" | "light" diff --git a/agenta-web/src/contexts/app.context.tsx b/agenta-web/src/contexts/app.context.tsx index 35db25f318..acf44a0215 100644 --- a/agenta-web/src/contexts/app.context.tsx +++ b/agenta-web/src/contexts/app.context.tsx @@ -1,9 +1,10 @@ import {ListAppsItem} from "@/lib/Types" -import {getAgentaApiUrl} from "@/lib/helpers/utils" +import {getAgentaApiUrl, isDemo} from "@/lib/helpers/utils" import {axiosFetcher} from "@/lib/services/api" import {useRouter} from "next/router" -import {PropsWithChildren, createContext, useContext, useMemo} from "react" +import {PropsWithChildren, createContext, useContext, useEffect, useMemo, useState} from "react" import useSWR from "swr" +import {dynamicContext} from "@/lib/helpers/dynamic" type AppContextType = { currentApp: ListAppsItem | null @@ -22,7 +23,22 @@ const initialValues: AppContextType = { } const useApps = () => { - const {data, error, isLoading, mutate} = useSWR(`${getAgentaApiUrl()}/api/apps/`, axiosFetcher) + const [useOrgData, setUseOrgData] = useState(() => () => "") + + useEffect(() => { + dynamicContext("org.context", {useOrgData}).then((context) => { + setUseOrgData(() => context.useOrgData) + }) + }, []) + + const {selectedOrg} = useOrgData() + const {data, error, isLoading, mutate} = useSWR( + `${getAgentaApiUrl()}/api/apps/` + + (isDemo() + ? `?org_id=${selectedOrg?.id}&workspace_id=${selectedOrg?.default_workspace.id}` + : ""), + isDemo() ? (selectedOrg?.id ? axiosFetcher : () => {}) : axiosFetcher, + ) return { data: (data || []) as ListAppsItem[], error, diff --git a/agenta-web/src/lib/Types.ts b/agenta-web/src/lib/Types.ts index bf1ff8231b..ff6ad0ac8d 100644 --- a/agenta-web/src/lib/Types.ts +++ b/agenta-web/src/lib/Types.ts @@ -251,6 +251,7 @@ export interface AppTemplate { env_vars?: { OPENAI_API_KEY: string | null } + organization_id?: string } export type GenericObject = Record diff --git a/agenta-web/src/lib/helpers/dynamic.ts b/agenta-web/src/lib/helpers/dynamic.ts new file mode 100644 index 0000000000..19f5a003ce --- /dev/null +++ b/agenta-web/src/lib/helpers/dynamic.ts @@ -0,0 +1,32 @@ +import dynamic from "next/dynamic" + +export function dynamicComponent(path: string, fallback: any = () => null) { + return dynamic(() => import(`@/components/${path}`), { + loading: fallback, + ssr: false, + }) +} + +export async function dynamicContext(path: string, fallback?: any) { + try { + return await import(`@/contexts/${path}`) + } catch (error) { + return fallback + } +} + +export async function dynamicHook(path: string, fallback: any = () => null) { + try { + return await import(`@/hooks/${path}`) + } catch (error) { + return fallback + } +} + +export async function dynamicService(path: string, fallback?: any) { + try { + return await import(`@/services/${path}`) + } catch (error) { + return fallback + } +} diff --git a/agenta-web/src/lib/helpers/utils.ts b/agenta-web/src/lib/helpers/utils.ts index ded3297ae3..87f73088fa 100644 --- a/agenta-web/src/lib/helpers/utils.ts +++ b/agenta-web/src/lib/helpers/utils.ts @@ -1,5 +1,4 @@ import {v4 as uuidv4} from "uuid" -import dynamic from "next/dynamic" import {EvaluationType} from "../enums" import {GenericObject} from "../Types" import promiseRetry from "promise-retry" @@ -188,13 +187,6 @@ export const isDemo = () => { return false } -export function dynamicComponent(path: string, fallback: any = () => null) { - return dynamic(() => import(`@/components/${path}`), { - loading: fallback, - ssr: false, - }) -} - export const removeKeys = (obj: GenericObject, keys: string[]) => { let newObj = Object.assign({}, obj) for (let key of keys) { diff --git a/agenta-web/src/lib/services/api.ts b/agenta-web/src/lib/services/api.ts index 018d5f6c17..9b91630897 100644 --- a/agenta-web/src/lib/services/api.ts +++ b/agenta-web/src/lib/services/api.ts @@ -24,6 +24,7 @@ import { } from "../transformers" import {EvaluationFlow, EvaluationType} from "../enums" import {getAgentaApiUrl, removeKeys, shortPoll} from "../helpers/utils" +import {dynamicContext} from "../helpers/dynamic" /** * Raw interface for the parameters parsed from the openapi.json */ @@ -597,6 +598,12 @@ export const createAndStartTemplate = async ({ ) => void }) => { try { + const {getOrgValues} = await dynamicContext("org.context", { + getOrgValues: () => ({ + selectedOrg: {id: undefined}, + }), + }) + const {selectedOrg} = getOrgValues() onStatusChange?.("creating_app") let app try { @@ -607,6 +614,7 @@ export const createAndStartTemplate = async ({ env_vars: { OPENAI_API_KEY: providerKey, }, + organization_id: selectedOrg.id, }, true, ) diff --git a/agenta-web/src/pages/settings/index.tsx b/agenta-web/src/pages/settings/index.tsx index e4bc75a8d7..5fba15eb9b 100644 --- a/agenta-web/src/pages/settings/index.tsx +++ b/agenta-web/src/pages/settings/index.tsx @@ -1,7 +1,8 @@ import Secrets from "@/components/pages/settings/Secrets/Secrets" import ProtectedRoute from "@/components/ProtectedRoute/ProtectedRoute" import {useQueryParam} from "@/hooks/useQuery" -import {dynamicComponent, isDemo} from "@/lib/helpers/utils" +import {dynamicComponent} from "@/lib/helpers/dynamic" +import {isDemo} from "@/lib/helpers/utils" import {ApartmentOutlined, KeyOutlined, LockOutlined} from "@ant-design/icons" import {Space, Tabs, Typography} from "antd" import {createUseStyles} from "react-jss" From 862666454ad88a7e257fd95f3a4fff005d3da83f Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 09:33:38 +0100 Subject: [PATCH 49/99] require workspace and org id when creation app from template in cloud --- .../agenta_backend/routers/app_router.py | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index bf8d770efe..55b38fe048 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -434,26 +434,17 @@ async def create_app_and_variant_from_template( request.state.user_id ) - logger.debug("Step 2: Setting organization ID") - if payload.organization_id is None: - organization = await get_user_own_org(user_org_workspace_data["uid"]) - organization_id = str(organization.id) - else: - organization_id = payload.organization_id - organization = await db_manager_ee.get_organization(organization_id) - - logger.debug("Step 3: Setting workspace ID") - if payload.workspace_id is None: - workspace = await get_org_default_workspace(organization) - workspace_id = str(workspace.id) - else: - workspace_id = payload.workspace_id - - logger.debug("Step 4: Checking user has permission to create app") + logger.debug("Step 2: Checking that workspace ID and organization ID are provided") + if payload.organization_id is None or payload.workspace_id is None: + raise Exception( + "Organization ID and Workspace ID must be provided to create app from template", + ) + + logger.debug("Step 3: Checking user has permission to create app") has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, - workspace=workspace, - organization=organization, + workspace_id=payload.workspace_id, + organization_id=payload.organization_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -468,7 +459,7 @@ async def create_app_and_variant_from_template( ) logger.debug( - f"Step 5: Checking if app {payload.app_name} already exists" + f"Step 4: Checking if app {payload.app_name} already exists" if FEATURE_FLAG in ["cloud", "ee"] else f"Step 1: Checking if app {payload.app_name} already exists" ) @@ -476,8 +467,8 @@ async def create_app_and_variant_from_template( app = await db_manager.fetch_app_by_name_and_parameters( app_name, request.state.user_id, - organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, - workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, + payload.organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, + payload.workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, ) if app is not None: raise Exception( @@ -485,7 +476,7 @@ async def create_app_and_variant_from_template( ) logger.debug( - "Step 6: Creating new app and initializing environments" + "Step 5: Creating new app and initializing environments" if FEATURE_FLAG in ["cloud", "ee"] else "Step 2: Creating new app and initializing environments" ) @@ -493,12 +484,12 @@ async def create_app_and_variant_from_template( app = await db_manager.create_app_and_envs( app_name, request.state.user_id, - organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, - workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, + payload.organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, + payload.workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, ) logger.debug( - "Step 7: Retrieve template from db" + "Step 6: Retrieve template from db" if FEATURE_FLAG in ["cloud", "ee"] else "Step 3: Retrieve template from db" ) @@ -507,7 +498,7 @@ async def create_app_and_variant_from_template( image_name = f"{repo_name}:{template_db.name}" logger.debug( - "Step 8: Creating image instance and adding variant based on image" + "Step 7: Creating image instance and adding variant based on image" if FEATURE_FLAG in ["cloud", "ee"] else "Step 4: Creating image instance and adding variant based on image" ) @@ -525,28 +516,28 @@ async def create_app_and_variant_from_template( ) logger.debug( - "Step 9: Creating testset for app variant" + "Step 8: Creating testset for app variant" if FEATURE_FLAG in ["cloud", "ee"] else "Step 5: Creating testset for app variant" ) await db_manager.add_testset_to_app_variant( app_id=str(app.id), - org_id=organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, - workspace_id=workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, + org_id=payload.organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace_id=payload.workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, template_name=template_db.name, app_name=app.app_name, user_uid=request.state.user_id, ) logger.debug( - "Step 10: We create ready-to use evaluators" + "Step 9: We create ready-to use evaluators" if FEATURE_FLAG in ["cloud", "ee"] else "Step 6: We create ready-to use evaluators" ) await evaluator_manager.create_ready_to_use_evaluators(app=app) logger.debug( - "Step 11: Starting variant and injecting environment variables" + "Step 10: Starting variant and injecting environment variables" if FEATURE_FLAG in ["cloud", "ee"] else "Step 7: Starting variant and injecting environment variables" ) From 73ba8022717339b3fe7ffce04ecbc83f588a3c8d Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Tue, 30 Jan 2024 13:53:43 +0500 Subject: [PATCH 50/99] pass workspace_id when creating an app --- agenta-web/src/lib/Types.ts | 1 + agenta-web/src/lib/services/api.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/agenta-web/src/lib/Types.ts b/agenta-web/src/lib/Types.ts index ff6ad0ac8d..eb8be44758 100644 --- a/agenta-web/src/lib/Types.ts +++ b/agenta-web/src/lib/Types.ts @@ -252,6 +252,7 @@ export interface AppTemplate { OPENAI_API_KEY: string | null } organization_id?: string + workspace_id?: string } export type GenericObject = Record diff --git a/agenta-web/src/lib/services/api.ts b/agenta-web/src/lib/services/api.ts index 9b91630897..2dfa1a6525 100644 --- a/agenta-web/src/lib/services/api.ts +++ b/agenta-web/src/lib/services/api.ts @@ -600,7 +600,7 @@ export const createAndStartTemplate = async ({ try { const {getOrgValues} = await dynamicContext("org.context", { getOrgValues: () => ({ - selectedOrg: {id: undefined}, + selectedOrg: {id: undefined, default_workspace: {id: undefined}}, }), }) const {selectedOrg} = getOrgValues() @@ -615,6 +615,7 @@ export const createAndStartTemplate = async ({ OPENAI_API_KEY: providerKey, }, organization_id: selectedOrg.id, + workspace_id: selectedOrg.default_workspace.id, }, true, ) From d515f5164f5b6e6b9ada034a7d8f72fe6dfc84dd Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 14:13:43 +0100 Subject: [PATCH 51/99] merge oss/rbac_migrations --- .../oss/20240126100524_models_revamp.py | 772 ++++++++++++++++++ .../20240126144938_drop_organization_model.py | 42 + 2 files changed, 814 insertions(+) create mode 100644 agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126100524_models_revamp.py create mode 100644 agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py diff --git a/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126100524_models_revamp.py b/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126100524_models_revamp.py new file mode 100644 index 0000000000..d87f359662 --- /dev/null +++ b/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126100524_models_revamp.py @@ -0,0 +1,772 @@ +from enum import Enum +from uuid import uuid4 +from datetime import datetime +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field +from beanie import Document, Link, PydanticObjectId + +from beanie import iterative_migration + + +# Common Models (BaseModels) +class ConfigVersionDB(BaseModel): + version: int + parameters: Dict[str, Any] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + +class Result(BaseModel): + type: str + value: Any + + +class EvaluationScenarioResult(BaseModel): + evaluator_config: PydanticObjectId + result: Result + + +class AggregatedResult(BaseModel): + evaluator_config: PydanticObjectId + result: Result + + +class EvaluationScenarioInputDB(BaseModel): + name: str + type: str + value: str + + +class EvaluationScenarioOutputDB(BaseModel): + type: str + value: Any + + +class HumanEvaluationScenarioInput(BaseModel): + input_name: str + input_value: str + + +class HumanEvaluationScenarioOutput(BaseModel): + variant_id: str + variant_output: str + + +class SpanDB(Document): + parent_span_id: Optional[str] + meta: Optional[Dict[str, Any]] + event_name: str # Function or execution name + event_type: Optional[str] + start_time: datetime + duration: Optional[int] + status: str # initiated, completed, stopped, cancelled + end_time: datetime = Field(default=datetime.utcnow()) + inputs: Optional[List[str]] + outputs: Optional[List[str]] + prompt_template: Optional[str] + tokens_input: Optional[int] + tokens_output: Optional[int] + token_total: Optional[int] + cost: Optional[float] + tags: Optional[List[str]] + + class Settings: + name = "spans" + + +class Feedback(BaseModel): + uid: str = Field(default=str(uuid4())) + user_id: str + feedback: Optional[str] + score: Optional[float] + meta: Optional[Dict[str, Any]] + created_at: datetime + updated_at: datetime = Field(default=datetime.utcnow()) + + + + + + +# Old DB Models +class InvitationDB(BaseModel): + token: str = Field(unique=True) + email: str + expiration_date: datetime = Field(default="0") + used: bool = False + + +class OldOrganizationDB(Document): + name: str = Field(default="agenta") + description: str = Field(default="") + type: Optional[str] + owner: str # user id + members: Optional[List[PydanticObjectId]] + invitations: Optional[List[InvitationDB]] = [] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "organizations" + + +class OldUserDB(Document): + uid: str = Field(default="0", unique=True, index=True) + username: str = Field(default="agenta") + email: str = Field(default="demo@agenta.ai", unique=True) + organizations: Optional[List[PydanticObjectId]] = [] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "users" + + +class OldImageDB(Document): + """Defines the info needed to get an image and connect it to the app variant""" + + type: Optional[str] = Field(default="image") + template_uri: Optional[str] + docker_id: Optional[str] = Field(index=True) + tags: Optional[str] + deletable: bool = Field(default=True) + user: Link[OldUserDB] + organization: Link[OldOrganizationDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "docker_images" + + +class OldAppDB(Document): + app_name: str + organization: Link[OldOrganizationDB] + user: Link[OldUserDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "app_db" + + +class OldDeploymentDB(Document): + app: Link[OldAppDB] + organization: Link[OldOrganizationDB] + user: Link[OldUserDB] + container_name: Optional[str] + container_id: Optional[str] + uri: Optional[str] + status: str + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "deployments" + + +class OldVariantBaseDB(Document): + app: Link[OldAppDB] + organization: Link[OldOrganizationDB] + user: Link[OldUserDB] + base_name: str + image: Link[OldImageDB] + deployment: Optional[PydanticObjectId] # Link to deployment + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "bases" + + +class ConfigDB(Document): + config_name: str + current_version: int = Field(default=1) + parameters: Dict[str, Any] = Field(default=dict) + version_history: List[ConfigVersionDB] = Field(default=[]) + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "configs" + + +class OldAppVariantDB(Document): + app: Link[OldAppDB] + variant_name: str + image: Link[OldImageDB] + user: Link[OldUserDB] + organization: Link[OldOrganizationDB] + parameters: Dict[str, Any] = Field(default=dict) # TODO: deprecated. remove + previous_variant_name: Optional[str] # TODO: deprecated. remove + base_name: Optional[str] + base: Link[OldVariantBaseDB] + config_name: Optional[str] + config: Link[ConfigDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + is_deleted: bool = Field( # TODO: deprecated. remove + default=False + ) # soft deletion for using the template variants + + class Settings: + name = "app_variants" + + +class OldAppEnvironmentDB(Document): + app: Link[OldAppDB] + name: str + user: Link[OldUserDB] + organization: Link[OldOrganizationDB] + deployed_app_variant: Optional[PydanticObjectId] + deployment: Optional[PydanticObjectId] # reference to deployment + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "environments" + + +class TemplateDB(Document): + type: Optional[str] = Field(default="image") + template_uri: Optional[str] + tag_id: Optional[int] + name: str = Field(unique=True) # tag name of image + repo_name: Optional[str] + title: str + description: str + size: Optional[int] + digest: Optional[str] # sha256 hash of image digest + last_pushed: Optional[datetime] + + class Settings: + name = "templates" + + +class OldTestSetDB(Document): + name: str + app: Link[OldAppDB] + csvdata: List[Dict[str, str]] + user: Link[OldUserDB] + organization: Link[OldOrganizationDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "testsets" + + +class OldEvaluatorConfigDB(Document): + app: Link[OldAppDB] + organization: Link[OldOrganizationDB] + user: Link[OldUserDB] + name: str + evaluator_key: str + settings_values: Dict[str, Any] = Field(default=dict) + created_at: datetime = Field(default=datetime.utcnow()) + updated_at: datetime = Field(default=datetime.utcnow()) + + class Settings: + name = "evaluators_configs" + + +class OldHumanEvaluationDB(Document): + app: Link[OldAppDB] + organization: Link[OldOrganizationDB] + user: Link[OldUserDB] + status: str + evaluation_type: str + variants: List[PydanticObjectId] + testset: Link[OldTestSetDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "human_evaluations" + + +class OldHumanEvaluationScenarioDB(Document): + user: Link[OldUserDB] + organization: Link[OldOrganizationDB] + evaluation: Link[OldHumanEvaluationDB] + inputs: List[HumanEvaluationScenarioInput] + outputs: List[HumanEvaluationScenarioOutput] + vote: Optional[str] + score: Optional[Any] + correct_answer: Optional[str] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + is_pinned: Optional[bool] + note: Optional[str] + + class Settings: + name = "human_evaluations_scenarios" + + +class OldEvaluationDB(Document): + app: Link[OldAppDB] + organization: Link[OldOrganizationDB] + user: Link[OldUserDB] + status: str = Field(default="EVALUATION_INITIALIZED") + testset: Link[OldTestSetDB] + variant: PydanticObjectId + evaluators_configs: List[PydanticObjectId] + aggregated_results: List[AggregatedResult] + created_at: datetime = Field(default=datetime.utcnow()) + updated_at: datetime = Field(default=datetime.utcnow()) + + class Settings: + name = "new_evaluations" + + +class OldEvaluationScenarioDB(Document): + user: Link[OldUserDB] + organization: Link[OldOrganizationDB] + evaluation: Link[OldEvaluationDB] + variant_id: PydanticObjectId + inputs: List[EvaluationScenarioInputDB] + outputs: List[EvaluationScenarioOutputDB] + correct_answer: Optional[str] + is_pinned: Optional[bool] + note: Optional[str] + evaluators_configs: List[PydanticObjectId] + results: List[EvaluationScenarioResult] + created_at: datetime = Field(default=datetime.utcnow()) + updated_at: datetime = Field(default=datetime.utcnow()) + + class Settings: + name = "new_evaluation_scenarios" + + +class TraceDB(Document): + app_id: Optional[str] + variant_id: str + spans: List[PydanticObjectId] + start_time: datetime + end_time: datetime = Field(default=datetime.utcnow()) + cost: Optional[float] + latency: float + status: str # initiated, completed, stopped, cancelled, failed + token_consumption: Optional[int] + user: Link[OldUserDB] + tags: Optional[List[str]] + feedbacks: Optional[List[Feedback]] + + class Settings: + name = "traces" + + + + + + + +# New DB Models +class NewUserDB(Document): + uid: str = Field(default="0", unique=True, index=True) + username: str = Field(default="agenta") + email: str = Field(default="demo@agenta.ai", unique=True) + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "users" + + +class NewImageDB(Document): + """Defines the info needed to get an image and connect it to the app variant""" + + type: Optional[str] = Field(default="image") + template_uri: Optional[str] + docker_id: Optional[str] = Field(index=True) + tags: Optional[str] + deletable: bool = Field(default=True) + user: Link[NewUserDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "docker_images" + + +class NewAppDB(Document): + app_name: str + user: Link[NewUserDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "app_db" + + +class NewDeploymentDB(Document): + app: Link[NewAppDB] + user: Link[NewUserDB] + container_name: Optional[str] + container_id: Optional[str] + uri: Optional[str] + status: str + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "deployments" + + +class NewVariantBaseDB(Document): + app: Link[NewAppDB] + user: Link[NewUserDB] + base_name: str + image: Link[NewImageDB] + deployment: Optional[PydanticObjectId] # Link to deployment + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "bases" + + +class ConfigDB(Document): + config_name: str + current_version: int = Field(default=1) + parameters: Dict[str, Any] = Field(default=dict) + version_history: List[ConfigVersionDB] = Field(default=[]) + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "configs" + + +class NewAppVariantDB(Document): + app: Link[NewAppDB] + variant_name: str + image: Link[NewImageDB] + user: Link[NewUserDB] + parameters: Dict[str, Any] = Field(default=dict) # TODO: deprecated. remove + previous_variant_name: Optional[str] # TODO: deprecated. remove + base_name: Optional[str] + base: Link[NewVariantBaseDB] + config_name: Optional[str] + config: Link[ConfigDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + is_deleted: bool = Field( # TODO: deprecated. remove + default=False + ) # soft deletion for using the template variants + + class Settings: + name = "app_variants" + + +class NewAppEnvironmentDB(Document): + app: Link[NewAppDB] + name: str + user: Link[NewUserDB] + deployed_app_variant: Optional[PydanticObjectId] + deployment: Optional[PydanticObjectId] # reference to deployment + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "environments" + + +class TemplateDB(Document): + type: Optional[str] = Field(default="image") + template_uri: Optional[str] + tag_id: Optional[int] + name: str = Field(unique=True) # tag name of image + repo_name: Optional[str] + title: str + description: str + size: Optional[int] + digest: Optional[str] # sha256 hash of image digest + last_pushed: Optional[datetime] + + class Settings: + name = "templates" + + +class NewTestSetDB(Document): + name: str + app: Link[NewAppDB] + csvdata: List[Dict[str, str]] + user: Link[NewUserDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "testsets" + + +class NewEvaluatorConfigDB(Document): + app: Link[NewAppDB] + user: Link[NewUserDB] + name: str + evaluator_key: str + settings_values: Dict[str, Any] = Field(default=dict) + created_at: datetime = Field(default=datetime.utcnow()) + updated_at: datetime = Field(default=datetime.utcnow()) + + class Settings: + name = "evaluators_configs" + + +class NewHumanEvaluationDB(Document): + app: Link[NewAppDB] + user: Link[NewUserDB] + status: str + evaluation_type: str + variants: List[PydanticObjectId] + testset: Link[NewTestSetDB] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "human_evaluations" + + +class NewHumanEvaluationScenarioDB(Document): + user: Link[NewUserDB] + evaluation: Link[NewHumanEvaluationDB] + inputs: List[HumanEvaluationScenarioInput] + outputs: List[HumanEvaluationScenarioOutput] + vote: Optional[str] + score: Optional[Any] + correct_answer: Optional[str] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + is_pinned: Optional[bool] + note: Optional[str] + + class Settings: + name = "human_evaluations_scenarios" + + +class NewEvaluationDB(Document): + app: Link[NewAppDB] + user: Link[NewUserDB] + status: str = Field(default="EVALUATION_INITIALIZED") + testset: Link[NewTestSetDB] + variant: PydanticObjectId + evaluators_configs: List[PydanticObjectId] + aggregated_results: List[AggregatedResult] + created_at: datetime = Field(default=datetime.utcnow()) + updated_at: datetime = Field(default=datetime.utcnow()) + + class Settings: + name = "new_evaluations" + + +class NewEvaluationScenarioDB(Document): + user: Link[NewUserDB] + evaluation: Link[NewEvaluationDB] + variant_id: PydanticObjectId + inputs: List[EvaluationScenarioInputDB] + outputs: List[EvaluationScenarioOutputDB] + correct_answer: Optional[str] + is_pinned: Optional[bool] + note: Optional[str] + evaluators_configs: List[PydanticObjectId] + results: List[EvaluationScenarioResult] + created_at: datetime = Field(default=datetime.utcnow()) + updated_at: datetime = Field(default=datetime.utcnow()) + + class Settings: + name = "new_evaluation_scenarios" + + +class TraceDB(Document): + app_id: Optional[str] + variant_id: str + spans: List[PydanticObjectId] + start_time: datetime + end_time: datetime = Field(default=datetime.utcnow()) + cost: Optional[float] + latency: float + status: str # initiated, completed, stopped, cancelled, failed + token_consumption: Optional[int] + user: Link[NewUserDB] + tags: Optional[List[str]] + feedbacks: Optional[List[Feedback]] + + class Settings: + name = "traces" + + + +class Forward: + + @iterative_migration( + document_models=[ + OldUserDB, + NewUserDB, + ] + ) + async def remove_organization_from_user_model( + self, input_document: OldUserDB, output_document: NewUserDB + ): + data = input_document.dict(exclude={"organizations"}) + new_document = NewUserDB(**data) + + + @iterative_migration( + document_models=[ + OldAppDB, + NewAppDB, + ] + ) + async def remove_organization_from_app_model( + self, input_document: OldAppDB, output_document: NewAppDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewAppDB(**data) + + + @iterative_migration( + document_models=[ + OldImageDB, + NewImageDB, + ] + ) + async def remove_organization_from_image_model( + self, input_document: OldImageDB, output_document: NewImageDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewImageDB(**data) + + + @iterative_migration( + document_models=[ + OldTestSetDB, + NewTestSetDB, + ] + ) + async def remove_organization_from_testset_model( + self, input_document: OldTestSetDB, output_document: NewTestSetDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewTestSetDB(**data) + + + @iterative_migration( + document_models=[ + OldVariantBaseDB, + NewVariantBaseDB, + ] + ) + async def remove_organization_from_variant_base_model( + self, input_document: OldVariantBaseDB, output_document: NewVariantBaseDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewVariantBaseDB(**data) + + + @iterative_migration( + document_models=[ + OldAppVariantDB, + NewVariantBaseDB, + ] + ) + async def remove_organization_from_app_variant_model( + self, input_document: OldAppVariantDB, output_document: NewAppVariantDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewAppVariantDB(**data) + + + @iterative_migration( + document_models=[ + OldEvaluationDB, + NewEvaluationDB, + ] + ) + async def remove_organization_from_evaluation_model( + self, input_document: OldEvaluationDB, output_document: NewEvaluationDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewEvaluationDB(**data) + + + @iterative_migration( + document_models=[ + OldDeploymentDB, + NewDeploymentDB, + ] + ) + async def remove_organization_from_deployment_model( + self, input_document: OldDeploymentDB, output_document: NewDeploymentDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewDeploymentDB(**data) + + + @iterative_migration( + document_models=[ + OldAppEnvironmentDB, + NewAppEnvironmentDB, + ] + ) + async def remove_organization_from_app_environment_model( + self, input_document: OldAppEnvironmentDB, output_document: NewAppEnvironmentDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewAppEnvironmentDB(**data) + + + @iterative_migration( + document_models=[ + OldEvaluatorConfigDB, + NewEvaluatorConfigDB, + ] + ) + async def remove_organization_from_evaluator_config_model( + self, input_document: OldEvaluatorConfigDB, output_document: NewEvaluatorConfigDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewEvaluatorConfigDB(**data) + + + @iterative_migration( + document_models=[ + OldHumanEvaluationDB, + NewHumanEvaluationDB, + ] + ) + async def remove_organization_from_human_evaluation_model( + self, input_document: OldHumanEvaluationDB, output_document: NewHumanEvaluationDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewHumanEvaluationDB(**data) + + + @iterative_migration( + document_models=[ + OldEvaluationScenarioDB, + NewEvaluationScenarioDB, + ] + ) + async def remove_organization_from_evaluation_scenario_model( + self, input_document: OldEvaluationScenarioDB, output_document: NewEvaluationScenarioDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewEvaluationScenarioDB(**data) + + @iterative_migration( + document_models=[ + OldHumanEvaluationScenarioDB, + NewHumanEvaluationScenarioDB, + ] + ) + async def remove_organization_from_app_environment_model( + self, input_document: OldHumanEvaluationScenarioDB, output_document: NewHumanEvaluationScenarioDB + ): + data = input_document.dict(exclude={"organization"}) + new_document = NewHumanEvaluationScenarioDB(**data) + + + +class Backward: + pass diff --git a/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py b/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py new file mode 100644 index 0000000000..8049ab94af --- /dev/null +++ b/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py @@ -0,0 +1,42 @@ +from datetime import datetime +from typing import List, Optional +from pydantic import BaseModel, Field + +from beanie import Document, PydanticObjectId, free_fall_migration, Session + +class InvitationDB(BaseModel): + token: str = Field(unique=True) + email: str + expiration_date: datetime = Field(default="0") + used: bool = False + +class OldOrganizationDB(Document): + name: str = Field(default="agenta") + description: str = Field(default="") + type: Optional[str] + owner: str # user id + members: Optional[List[PydanticObjectId]] + invitations: Optional[List[InvitationDB]] = [] + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "organizations" + + +class Forward: + + @free_fall_migration(document_models=[OldOrganizationDB]) + async def drop_old_organization_db(self, session: Session): + # Wrap deletion loop in a with_transaction context for potential rollback + async with session.start_transaction(): + async for old_organization in OldOrganizationDB.find_all(): + await old_organization.delete() + + # Commit the transaction if everything succeeds + await session.commit() + + + +class Backward: + pass From 9cec9469a154b387f8f350ffd38d8d55a61ab32c Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 14:14:46 +0100 Subject: [PATCH 52/99] cleanup --- agenta-backend/agenta_backend/routers/app_router.py | 2 +- .../agenta_backend/routers/container_router.py | 2 +- .../routers/human_evaluation_router.py | 13 ++----------- .../agenta_backend/routers/testset_router.py | 7 ++----- .../agenta_backend/services/db_manager.py | 2 +- .../agenta_backend/services/evaluation_service.py | 2 -- 6 files changed, 7 insertions(+), 21 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 55b38fe048..f0ab3d1db4 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -47,7 +47,7 @@ check_action_access, check_rbac_permission, ) - from agenta_backend.commons.models.db_models import Permission, WorkspaceRole + from agenta_backend.commons.models.db_models import Permission if FEATURE_FLAG in ["cloud"]: diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 500af783d1..bcd62219eb 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -10,7 +10,7 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.models.db_models import WorkspaceRole, Permission + from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access from agenta_backend.commons.models.api.api_models import Image_ as Image else: diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index de0b1b74c1..d0ff4f4f6d 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -34,17 +34,8 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.services.selectors import ( # noqa pylint: disable-all - get_user_org_and_workspace_id, - ) - from agenta_backend.commons.utils.permissions import ( # noqa pylint: disable-all - check_action_access, - check_rbac_permission, - ) - from agenta_backend.commons.models.db_models import ( # noqa pylint: disable-all - Permission, - WorkspaceRole, - ) + from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all + from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all router = APIRouter() diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 0a8c4fa040..b91971cb6b 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -26,15 +26,12 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: - from agenta_backend.commons.utils.permissions import ( - check_action_access, - ) # noqa pylint: disable-all + from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all from agenta_backend.commons.models.db_models import ( Permission, - ) # noqa pylint: disable-all - from agenta_backend.commons.models.db_models import ( TestSetDB_ as TestSetDB, ) # noqa pylint: disable-all + else: from agenta_backend.models.db_models import TestSetDB diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index dcb802557b..a0b97ec483 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -34,11 +34,11 @@ FEATURE_FLAG = os.environ["FEATURE_FLAG"] if FEATURE_FLAG in ["cloud", "ee"]: from agenta_backend.commons.services import db_manager_ee - from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_rbac_permission from agenta_backend.commons.services.selectors import get_user_org_and_workspace_id from agenta_backend.commons.models.db_models import ( + Permission, AppDB_ as AppDB, UserDB_ as UserDB, ImageDB_ as ImageDB, diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index f8ea4b7c3d..842533ec6e 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -28,7 +28,6 @@ from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, UserDB_ as UserDB, - AppVariantDB_ as AppVariantDB, EvaluationDB_ as EvaluationDB, HumanEvaluationDB_ as HumanEvaluationDB, EvaluationScenarioDB_ as EvaluationScenarioDB, @@ -38,7 +37,6 @@ from agenta_backend.models.db_models import ( AppDB, UserDB, - AppVariantDB, EvaluationDB, HumanEvaluationDB, EvaluationScenarioDB, From b0b59686470d3aadc09d98770a805ed93ce17561 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 15:30:09 +0100 Subject: [PATCH 53/99] refactor use of Feature Flag --- agenta-backend/agenta_backend/main.py | 10 +- .../agenta_backend/models/converters.py | 11 +-- .../agenta_backend/models/db_engine.py | 6 +- .../agenta_backend/routers/app_router.py | 91 ++++++++++--------- .../agenta_backend/routers/bases_router.py | 8 +- .../agenta_backend/routers/configs_router.py | 22 ++--- .../routers/container_router.py | 61 +++++++------ .../routers/environment_router.py | 8 +- .../routers/evaluation_router.py | 22 ++--- .../routers/evaluators_router.py | 19 ++-- .../routers/human_evaluation_router.py | 29 +++--- .../agenta_backend/routers/testset_router.py | 20 ++-- .../agenta_backend/routers/variants_router.py | 7 +- .../agenta_backend/services/app_manager.py | 64 +++++++------ .../services/container_manager.py | 5 +- .../agenta_backend/services/db_manager.py | 41 +++++---- .../services/deployment_manager.py | 11 +-- .../services/evaluation_service.py | 12 +-- .../services/evaluator_manager.py | 12 +-- .../services/results_service.py | 12 +-- agenta-backend/agenta_backend/utils/common.py | 40 +++++++- 21 files changed, 268 insertions(+), 243 deletions(-) diff --git a/agenta-backend/agenta_backend/main.py b/agenta-backend/agenta_backend/main.py index 04c98bb5ae..e5006957df 100644 --- a/agenta-backend/agenta_backend/main.py +++ b/agenta-backend/agenta_backend/main.py @@ -1,4 +1,3 @@ -import os import asyncio from contextlib import asynccontextmanager @@ -19,10 +18,11 @@ configs_router, health_router, ) +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.db_engine import DBEngine from agenta_backend.open_api import open_api_tags_metadata -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.services import templates_manager else: from agenta_backend.services import templates_manager @@ -70,12 +70,12 @@ async def lifespan(application: FastAPI, cache=True): allow_headers=allow_headers, ) -if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"]: +if not isCloudEE: from agenta_backend.services.auth_helper import authentication_middleware app.middleware("http")(authentication_middleware) -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: +if isCloudEE: import agenta_backend.cloud.main as cloud app, allow_headers = cloud.extend_main(app) @@ -104,7 +104,7 @@ async def lifespan(application: FastAPI, cache=True): app.include_router(bases_router.router, prefix="/bases", tags=["Bases"]) app.include_router(configs_router.router, prefix="/configs", tags=["Configs"]) -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: +if isCloudEE: import agenta_backend.cloud.main as cloud app = cloud.extend_app_schema(app) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 9f7e7649da..000205e2a1 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -1,10 +1,10 @@ """Converts db models to pydantic models """ -import os import json import logging from typing import List from agenta_backend.services import db_manager +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.api.user_models import User from agenta_backend.models.api.observability_models import ( @@ -23,8 +23,7 @@ EvaluationScenarioOutput, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, UserDB_ as UserDB, @@ -231,7 +230,7 @@ def app_variant_db_to_pydantic( config_name=app_variant_db.config_name, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: app_variant.organization_id = str(app_variant_db.organization.id) app_variant.workspace_id = str(app_variant_db.workspace.id) @@ -263,7 +262,7 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantRe uri=uri, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: variant_response.organization_id = str(app_variant_db.organization.id) variant_response.workspace_id = str(app_variant_db.workspace.id) @@ -307,7 +306,7 @@ def image_db_to_pydantic(image_db: ImageDB) -> ImageExtended: id=str(image_db.id), ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: image.organization_id = str(image_db.organization.id) image.workspace_id = str(image_db.workspace.id) diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index 778c630958..32926f6086 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -6,7 +6,9 @@ from beanie import init_beanie, Document from motor.motor_asyncio import AsyncIOMotorClient -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: +from agenta_backend.utils.common import isCloudEE + +if isCloudEE: from agenta_backend.commons.models.db_models import ( APIKeyDB, WorkspaceDB, @@ -70,7 +72,7 @@ HumanEvaluationScenarioDB, ] -if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: +if isCloudEE: document_models = document_models + [OrganizationDB, WorkspaceDB, APIKeyDB] diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index f0ab3d1db4..abc7c9f829 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -1,13 +1,20 @@ import os import logging + from typing import List, Optional from docker.errors import DockerException from fastapi.responses import JSONResponse -from agenta_backend.config import settings from fastapi import HTTPException, Request -from agenta_backend.models import converters from beanie import PydanticObjectId as ObjectId -from agenta_backend.utils.common import APIRouter + +from agenta_backend.config import settings +from agenta_backend.models import converters +from agenta_backend.utils.common import ( + isEE, + isCloud, + APIRouter, + isCloudEE, +) from agenta_backend.services import ( db_manager, @@ -21,8 +28,7 @@ AddVariantFromImagePayload, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.api.api_models import ( Image_ as Image, CreateApp_ as CreateApp, @@ -36,7 +42,7 @@ AppVariantResponse, CreateAppVariant, ) -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.services import db_manager_ee from agenta_backend.commons.services.selectors import ( get_user_own_org, @@ -50,11 +56,11 @@ from agenta_backend.commons.models.db_models import Permission -if FEATURE_FLAG in ["cloud"]: +if isCloud: from agenta_backend.cloud.services import ( lambda_deployment_manager as deployment_manager, ) # noqa pylint: disable-all -elif FEATURE_FLAG in ["ee"]: +elif isEE: from agenta_backend.ee.services import ( deployment_manager, ) # noqa pylint: disable-all @@ -86,7 +92,7 @@ async def list_app_variants( List[AppVariantResponse]: A list of app variants for the given app ID. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -136,7 +142,7 @@ async def get_variant_by_env( AppVariantResponse: The retrieved app variant. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -191,7 +197,7 @@ async def create_app( HTTPException: If there is an error creating the app or the user does not have permission to access the app. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: try: user_org_workspace_data = await get_user_org_and_workspace_id( request.state.user_id @@ -250,8 +256,8 @@ async def create_app( app_db = await db_manager.create_app_and_envs( payload.app_name, request.state.user_id, - organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, - workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, + organization_id if isCloudEE else None, + workspace_id if isCloudEE else None, ) return CreateAppOutput(app_id=str(app_db.id), app_name=str(app_db.app_name)) except Exception as e: @@ -313,7 +319,7 @@ async def add_variant_from_image( dict: The newly added variant. """ - if FEATURE_FLAG not in ["cloud", "ee"]: + if not isCloudEE: image = Image( type="image", docker_id=payload.docker_id, @@ -328,11 +334,12 @@ async def add_variant_from_image( raise HTTPException(status_code=404, detail="Image not found") try: - if FEATURE_FLAG in ["cloud", "ee"]: + app = await db_manager.fetch_app_by_id(app_id) + + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + object = app, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -345,8 +352,6 @@ async def add_variant_from_image( status_code=403, ) - app = await db_manager.fetch_app_by_id(app_id) - variant_db = await app_manager.add_variant_based_on_image( app=app, variant_name=payload.variant_name, @@ -375,11 +380,12 @@ async def remove_app(app_id: str, request: Request): app -- App to remove """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + app = await db_manager.fetch_app_by_id(app_id) + + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + object = app, permission=Permission.DELETE_APPLICATION, ) logger.debug(f"User has Permission to delete app: {has_permission}") @@ -391,14 +397,11 @@ async def remove_app(app_id: str, request: Request): ) else: - await app_manager.remove_app(app_id=app_id) + await app_manager.remove_app(app) except DockerException as e: detail = f"Docker error while trying to remove the app: {str(e)}" raise HTTPException(status_code=500, detail=detail) except Exception as e: - import traceback - - traceback.print_exc() detail = f"Unexpected error while trying to remove the app: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -427,7 +430,7 @@ async def create_app_and_variant_from_template( try: logger.debug("Start: Creating app and variant from template") - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # Get user and org id logger.debug("Step 1: Getting user and organization ID") user_org_workspace_data: dict = await get_user_org_and_workspace_id( @@ -460,15 +463,15 @@ async def create_app_and_variant_from_template( logger.debug( f"Step 4: Checking if app {payload.app_name} already exists" - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else f"Step 1: Checking if app {payload.app_name} already exists" ) app_name = payload.app_name.lower() app = await db_manager.fetch_app_by_name_and_parameters( app_name, request.state.user_id, - payload.organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, - payload.workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, + payload.organization_id if isCloudEE else None, + payload.workspace_id if isCloudEE else None, ) if app is not None: raise Exception( @@ -477,20 +480,20 @@ async def create_app_and_variant_from_template( logger.debug( "Step 5: Creating new app and initializing environments" - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else "Step 2: Creating new app and initializing environments" ) if app is None: app = await db_manager.create_app_and_envs( app_name, request.state.user_id, - payload.organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, - payload.workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, + payload.organization_id if isCloudEE else None, + payload.workspace_id if isCloudEE else None, ) logger.debug( "Step 6: Retrieve template from db" - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else "Step 3: Retrieve template from db" ) template_db = await db_manager.get_template(payload.template_id) @@ -499,16 +502,16 @@ async def create_app_and_variant_from_template( logger.debug( "Step 7: Creating image instance and adding variant based on image" - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else "Step 4: Creating image instance and adding variant based on image" ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, variant_name="app.default", docker_id_or_template_uri=template_db.template_uri - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else template_db.digest, - tags=f"{image_name}" if FEATURE_FLAG not in ["cloud", "ee"] else None, + tags=f"{image_name}" if not isCloudEE else None, base_name="app", config_name="default", is_template_image=True, @@ -517,13 +520,13 @@ async def create_app_and_variant_from_template( logger.debug( "Step 8: Creating testset for app variant" - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else "Step 5: Creating testset for app variant" ) await db_manager.add_testset_to_app_variant( app_id=str(app.id), - org_id=payload.organization_id if FEATURE_FLAG in ["cloud", "ee"] else None, - workspace_id=payload.workspace_id if FEATURE_FLAG in ["cloud", "ee"] else None, + org_id=payload.organization_id if isCloudEE else None, + workspace_id=payload.workspace_id if isCloudEE else None, template_name=template_db.name, app_name=app.app_name, user_uid=request.state.user_id, @@ -531,17 +534,17 @@ async def create_app_and_variant_from_template( logger.debug( "Step 9: We create ready-to use evaluators" - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else "Step 6: We create ready-to use evaluators" ) await evaluator_manager.create_ready_to_use_evaluators(app=app) logger.debug( "Step 10: Starting variant and injecting environment variables" - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else "Step 7: Starting variant and injecting environment variables" ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: if not os.environ["OPENAI_API_KEY"]: raise Exception( "Unable to start app container. Please file an issue by clicking on the button below.", @@ -589,7 +592,7 @@ async def list_environments( """ logger.debug(f"Listing environments for app: {app_id}") try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index 2f9e629337..b806077e12 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -1,4 +1,3 @@ -import os import logging from typing import List, Optional @@ -7,11 +6,10 @@ from agenta_backend.models import converters from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import BaseOutput -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -42,7 +40,7 @@ async def list_bases( HTTPException: If there was an error retrieving the bases. """ try: - if os.environ["FEATURE_FLAG"] in ["cloud", "ee"] and app_id is not None: + if isCloudEE and app_id is not None: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index 289e6fd318..3a48e4ccc2 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -1,10 +1,9 @@ -import os import logging from typing import Optional from fastapi.responses import JSONResponse from fastapi import Request, HTTPException -from agenta_backend.utils.common import APIRouter +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import ( SaveConfigPayload, @@ -15,8 +14,7 @@ app_manager, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -32,11 +30,12 @@ async def save_config( request: Request, ): try: - if FEATURE_FLAG in ["cloud", "ee"]: + base_db = await db_manager.fetch_base_by_id(payload.base_id) + + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=payload.base_id, - object_type="base", + object = base_db, permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: @@ -47,7 +46,6 @@ async def save_config( status_code=403, ) - base_db = await db_manager.fetch_base_by_id(payload.base_id) variants_db = await db_manager.list_variants_for_base(base_db) variant_to_overwrite = None for variant_db in variants_db: @@ -92,12 +90,13 @@ async def get_config( environment_name: Optional[str] = None, ): try: + base_db = await db_manager.fetch_base_by_id(base_id) + # detemine whether the user has access to the base - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=base_id, - object_type="base", + object = base_db, permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: @@ -108,7 +107,6 @@ async def get_config( status_code=403, ) - base_db = await db_manager.fetch_base_by_id(base_id) # in case environment_name is provided, find the variant deployed if environment_name: app_environments = await db_manager.list_environments( diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index bcd62219eb..2f56cb76af 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -1,4 +1,3 @@ -import os import logging from typing import List, Optional, Union @@ -6,10 +5,9 @@ from fastapi import Request, UploadFile, HTTPException from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter +from agenta_backend.utils.common import APIRouter, isCloudEE, isCloud, isEE -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access from agenta_backend.commons.models.api.api_models import Image_ as Image @@ -17,9 +15,9 @@ from agenta_backend.models.api.api_models import Image -if FEATURE_FLAG in ["cloud"]: +if isCloud: from agenta_backend.cloud.services import container_manager -elif FEATURE_FLAG in ["ee"]: +elif isEE: from agenta_backend.ee.services import container_manager else: from agenta_backend.services import container_manager @@ -57,31 +55,34 @@ async def build_image( Returns: Image: The Docker image that was built. """ - if FEATURE_FLAG in ["cloud", "ee"]: - has_permission = await check_action_access( - user_uid=request.state.user_id, - object_id=app_id, - object_type="app", - permission=Permission.CREATE_APPLICATION, - ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + try: + app_db = await db_manager.fetch_app_by_id(app_id) + + # Check app access + if isCloudEE: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object = app_db, + permission=Permission.CREATE_APPLICATION, ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + + image_result = await container_manager.build_image( + app_db=app_db, + base_name=base_name, + tar_file=tar_file, + ) - # Check app access - app_db = await db_manager.fetch_app_by_id(app_id) - - image_result = await container_manager.build_image( - app_db=app_db, - base_name=base_name, - tar_file=tar_file, - ) - - return image_result + return image_result + except Exception as ex: + return JSONResponse({"message": str(ex)}, status_code=500) @router.post("/restart_container/", operation_id="restart_container") @@ -153,7 +154,7 @@ async def construct_app_container_url( # assert that one of base_id or variant_id is provided assert base_id or variant_id, "Please provide either base_id or variant_id" - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=base_id if base_id else variant_id, diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 577071ab81..d18acb0eea 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -1,15 +1,13 @@ -import os import logging from fastapi.responses import JSONResponse from fastapi import Request, HTTPException from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import DeployToEnvironmentPayload -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -34,7 +32,7 @@ async def deploy_to_environment( HTTPException: If the deployment fails. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=payload.variant_id, diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 2738ead0d7..fbcbe47044 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -1,4 +1,3 @@ -import os import secrets import logging @@ -6,8 +5,8 @@ from fastapi.responses import JSONResponse from fastapi import HTTPException, Request, status, Response -from agenta_backend.utils.common import APIRouter from agenta_backend.tasks.evaluations import evaluate +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import evaluation_service, db_manager from agenta_backend.models.api.evaluation_model import ( @@ -23,8 +22,7 @@ check_ai_critique_inputs, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -45,7 +43,7 @@ async def create_evaluation( _description_ """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=payload.app_id, @@ -113,7 +111,7 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -150,7 +148,7 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -196,7 +194,7 @@ async def fetch_evaluation_scenarios( """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -239,7 +237,7 @@ async def fetch_list_evaluations( List[Evaluation]: A list of evaluations. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -278,7 +276,7 @@ async def fetch_evaluation( Evaluation: The fetched evaluation. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -317,7 +315,7 @@ async def delete_evaluations( """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: for evaluation_id in delete_evaluations.evaluations_ids: has_permission = await check_action_access( user_uid=request.state.user_id, @@ -382,7 +380,7 @@ async def fetch_evaluation_scenarios( try: evaluations_ids_list = evaluations_ids.split(",") - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: for evaluation_id in evaluations_ids_list: has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 39cfe8eb2b..71ae528a6e 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -1,13 +1,11 @@ -import os -import json import logging -from typing import List +from typing import List from fastapi import HTTPException, Request from fastapi.responses import JSONResponse -from agenta_backend.utils.common import APIRouter from agenta_backend.services import evaluator_manager +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.evaluation_model import ( Evaluator, @@ -16,8 +14,7 @@ UpdateEvaluatorConfig, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -62,7 +59,7 @@ async def get_evaluator_configs(app_id: str, request: Request): """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -94,7 +91,7 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluator_config_id, @@ -133,7 +130,7 @@ async def create_new_evaluator_config( EvaluatorConfigDB: Evaluator configuration api model. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=payload.app_id, @@ -172,7 +169,7 @@ async def update_evaluator_config( """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluator_config_id, @@ -208,7 +205,7 @@ async def delete_evaluator_config(evaluator_config_id: str, request: Request): bool: True if deletion was successful, False otherwise. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluator_config_id, diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index d0ff4f4f6d..5b5f0e7ea4 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -1,10 +1,6 @@ -import os -import secrets from typing import List, Dict - from fastapi.responses import JSONResponse -from fastapi.encoders import jsonable_encoder -from agenta_backend.utils.common import APIRouter +from agenta_backend.utils.common import APIRouter, isCloudEE from fastapi import HTTPException, Body, Request, status, Response from agenta_backend.models import converters @@ -32,8 +28,7 @@ update_human_evaluation_service, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all @@ -54,7 +49,7 @@ async def create_evaluation( _description_ """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=payload.app_id, @@ -100,7 +95,7 @@ async def fetch_list_human_evaluations( List[HumanEvaluation]: A list of evaluations. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -133,7 +128,7 @@ async def fetch_human_evaluation( HumanEvaluation: The fetched evaluation. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -174,7 +169,7 @@ async def fetch_evaluation_scenarios( """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -214,7 +209,7 @@ async def update_human_evaluation( None: A 204 No Content status code, indicating that the update was successful. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -257,7 +252,7 @@ async def update_evaluation_scenario_router( None: 204 No Content status code upon successful update. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_scenario_id, @@ -297,7 +292,7 @@ async def get_evaluation_scenario_score_router( Dictionary containing the scenario ID and its score. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_scenario_id, @@ -331,7 +326,7 @@ async def update_evaluation_scenario_score_router( None: 204 No Content status code upon successful update. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_scenario_id, @@ -368,7 +363,7 @@ async def fetch_results( """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -412,7 +407,7 @@ async def delete_evaluations( """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: for evaluation_id in delete_evaluations.evaluations_ids: has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index b91971cb6b..45b89bef7f 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -1,5 +1,4 @@ import io -import os import csv import json import logging @@ -11,8 +10,8 @@ from pydantic import ValidationError from fastapi.responses import JSONResponse from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter from agenta_backend.services.db_manager import get_user +from agenta_backend.utils.common import APIRouter, isCloudEE from fastapi import HTTPException, UploadFile, File, Form, Request from agenta_backend.models.converters import testset_db_to_pydantic @@ -24,8 +23,7 @@ TestSetOutputResponse, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all from agenta_backend.commons.models.db_models import ( Permission, @@ -64,7 +62,7 @@ async def upload_file( dict: The result of the upload process. """ - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -148,7 +146,7 @@ async def import_testset( Returns: dict: The result of the import process. """ - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -235,7 +233,7 @@ async def create_testset( str: The id of the test set created. """ - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -294,7 +292,7 @@ async def update_testset( Returns: str: The id of the test set updated. """ - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=testset_id, @@ -349,7 +347,7 @@ async def get_testsets( Raises: - `HTTPException` with status code 404 if no testsets are found. """ - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -395,7 +393,7 @@ async def get_single_testset( Returns: The requested testset if found, else an HTTPException. """ - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=testset_id, @@ -432,7 +430,7 @@ async def delete_testsets( Returns: A list of the deleted testsets' IDs. """ - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: for testset_id in delete_testsets.testset_ids: has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index e5f2980c26..1fd7d2b8bd 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -6,15 +6,14 @@ from fastapi.responses import JSONResponse from agenta_backend.models import converters from fastapi import HTTPException, Request, Body -from agenta_backend.utils.common import APIRouter +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import ( app_manager, db_manager, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.utils.permissions import ( check_action_access, ) # noqa pylint: disable-all @@ -297,7 +296,7 @@ async def start_variant( logger.debug("Starting variant %s", variant_id) # Inject env vars to docker container - if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: + if isCloudEE: if not os.environ["OPENAI_API_KEY"]: raise HTTPException( status_code=400, diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 413be26683..f09727ebf6 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -21,20 +21,25 @@ evaluator_manager, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] +from agenta_backend.utils.common import ( + isEE, + isOssEE, + isCloud, + isCloudEE, +) -if FEATURE_FLAG in ["cloud"]: +if isCloud: from agenta_backend.cloud.services import ( lambda_deployment_manager as deployment_manager, ) # noqa pylint: disable-all -elif FEATURE_FLAG in ["ee"]: +elif isEE: from agenta_backend.ee.services import ( deployment_manager, ) # noqa pylint: disable-all else: from agenta_backend.services import deployment_manager -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.services import ( api_key_service, ) # noqa pylint: disable-all @@ -70,8 +75,8 @@ async def start_variant( db_app_variant.image.docker_id, db_app_variant.image.tags, db_app_variant.app.app_name, - db_app_variant.organization if FEATURE_FLAG in ["cloud", "ee"] else None, - db_app_variant.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, + db_app_variant.organization if isCloudEE else None, + db_app_variant.workspace if isCloudEE else None, ) logger.debug("App name is %s", db_app_variant.app.app_name) # update the env variables @@ -86,7 +91,7 @@ async def start_variant( env_vars.update( {"AGENTA_BASE_ID": str(db_app_variant.base.id), "AGENTA_HOST": domain_name} ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: api_key = await api_key_service.create_api_key( str(db_app_variant.user.uid), expiration_date=None, hidden=True ) @@ -130,7 +135,7 @@ async def update_variant_image(app_variant_db: AppVariantDB, image: Image): await deployment_manager.stop_and_delete_service(deployment) await db_manager.remove_deployment(deployment) - if FEATURE_FLAG in ["ee", "oss"]: + if isOssEE: await deployment_manager.remove_image(app_variant_db.base.image) await db_manager.remove_image(app_variant_db.base.image) @@ -142,10 +147,10 @@ async def update_variant_image(app_variant_db: AppVariantDB, image: Image): user=app_variant_db.user, deletable=True, organization=app_variant_db.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa workspace=app_variant_db.workspace - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa ) # Update base with new image @@ -224,7 +229,7 @@ async def terminate_and_remove_app_variant( # If image deletable is True, remove docker image and image db if image.deletable: try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: await deployment_manager.remove_repository(image.tags) else: await deployment_manager.remove_image(image) @@ -293,7 +298,7 @@ async def remove_app_related_resources(app_id: str): raise e from None -async def remove_app(app_id: str): +async def remove_app(app: AppDB): """Removes all app variants from db, if it is the last one using an image, then deletes the image from the db, shutdowns the container, deletes it and remove the image from the registry @@ -302,13 +307,12 @@ async def remove_app(app_id: str): app_name -- the app name to remove """ # checks if it is the last app variant using its image - app = await db_manager.fetch_app_by_id(app_id) if app is None: - error_msg = f"Failed to delete app {app_id}: Not found in DB." + error_msg = f"Failed to delete app {app.id}: Not found in DB." logger.error(error_msg) raise ValueError(error_msg) try: - app_variants = await db_manager.list_app_variants(app_id) + app_variants = await db_manager.list_app_variants(app.id) for app_variant_db in app_variants: await terminate_and_remove_app_variant(app_variant_db=app_variant_db) logger.info( @@ -317,11 +321,11 @@ async def remove_app(app_id: str): if len(app_variants) == 0: # Failsafe in case something went wrong before logger.debug("remove_app_related_resources") - await remove_app_related_resources(app_id) + await remove_app_related_resources(str(app.id)) except Exception as e: logger.error( - f"An error occurred while deleting app {app_id} and its associated resources: {str(e)}" + f"An error occurred while deleting app {app.id} and its associated resources: {str(e)}" ) raise e from None @@ -391,7 +395,7 @@ async def add_variant_based_on_image( ): raise ValueError("App variant, variant name or docker_id/template_uri is None") - if FEATURE_FLAG not in ["cloud", "ee"]: + if not isCloudEE: if tags in [None, ""]: raise ValueError("OSS: Tags is None") @@ -414,20 +418,20 @@ async def add_variant_based_on_image( db_image = await db_manager.get_orga_image_instance_by_uri( template_uri=docker_id_or_template_uri, organization_id=str(app.organization.id) - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa workspace_id=str(app.workspace.id) - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa ) else: db_image = await db_manager.get_orga_image_instance_by_docker_id( docker_id=docker_id_or_template_uri, organization_id=str(app.organization.id) - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa workspace_id=str(app.workspace.id) - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa ) @@ -441,10 +445,10 @@ async def add_variant_based_on_image( deletable=not (is_template_image), user=user_instance, organization=app.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa workspace=app.workspace - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa ) else: @@ -456,10 +460,10 @@ async def add_variant_based_on_image( deletable=not (is_template_image), user=user_instance, organization=app.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa workspace=app.workspace - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa ) @@ -478,9 +482,9 @@ async def add_variant_based_on_image( db_base = await db_manager.create_new_variant_base( app=app, organization=app.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa - workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace=app.workspace if isCloudEE else None, # noqa user=user_instance, base_name=base_name, # the first variant always has default base image=db_image, @@ -494,9 +498,9 @@ async def add_variant_based_on_image( image=db_image, user=user_instance, organization=app.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa - workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa + workspace=app.workspace if isCloudEE else None, # noqa parameters={}, base_name=base_name, config_name=config_name, diff --git a/agenta-backend/agenta_backend/services/container_manager.py b/agenta-backend/agenta_backend/services/container_manager.py index fba245cde6..ba1dc61d85 100644 --- a/agenta-backend/agenta_backend/services/container_manager.py +++ b/agenta-backend/agenta_backend/services/container_manager.py @@ -20,11 +20,10 @@ AppDB, ) from agenta_backend.services import docker_utils +from agenta_backend.utils.common import isCloud client = docker.from_env() -FEATURE_FLAG = os.environ["FEATURE_FLAG"] - logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -101,7 +100,7 @@ def build_image_job( shutil.unpack_archive(tar_path, temp_dir) try: - if os.environ["FEATURE_FLAG"] in ["cloud"]: + if isCloud: dockerfile = "Dockerfile.cloud" else: dockerfile = "Dockerfile" diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index a0b97ec483..1ab472d9ca 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -10,6 +10,8 @@ from fastapi import HTTPException from fastapi.responses import JSONResponse +from agenta_backend.models import converters +from agenta_backend.utils.common import isCloudEE from agenta_backend.services.json_importer_helper import get_json from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum @@ -18,7 +20,6 @@ Template, ) -from agenta_backend.models import converters from agenta_backend.models.db_models import ( Result, @@ -31,8 +32,8 @@ EvaluationScenarioOutputDB, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: + +if isCloudEE: from agenta_backend.commons.services import db_manager_ee from agenta_backend.commons.utils.permissions import check_rbac_permission from agenta_backend.commons.services.selectors import get_user_org_and_workspace_id @@ -124,7 +125,7 @@ async def add_testset_to_app_variant( user=user_db, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( org_id is not None and workspace_id is not None @@ -253,7 +254,7 @@ async def create_new_variant_base( image=image, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -325,7 +326,7 @@ async def create_new_app_variant( parameters=parameters, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -388,7 +389,7 @@ async def create_image( image.tags = tags image.docker_id = docker_id - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -434,7 +435,7 @@ async def create_deployment( status=status, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: deployment.organization = organization deployment.workspace = workspace @@ -478,7 +479,7 @@ async def create_app_and_envs( app = AppDB(app_name=app_name, user=user_instance) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization_id is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None @@ -572,7 +573,7 @@ async def get_user(user_uid: str) -> UserDB: user = await UserDB.find_one(UserDB.uid == user_uid) if user is None: - if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"]: + if not isCloudEE: # create user user_db = UserDB(uid="0") user = await user_db.create() @@ -661,7 +662,7 @@ async def get_orga_image_instance_by_docker_id( query_expression = {"docker_id": docker_id} - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None @@ -697,7 +698,7 @@ async def get_orga_image_instance_by_uri( query_expression = {"template_uri": template_uri} - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None @@ -785,7 +786,7 @@ async def add_variant_from_base_and_config( is_deleted=False, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: db_app_variant.organization = previous_app_variant_db.organization db_app_variant.workspace = previous_app_variant_db.workspace @@ -822,7 +823,7 @@ async def list_apps( return [converters.app_db_to_pydantic(app_db)] elif (org_id is not None) or (workspace_id is not None): - if not FEATURE_FLAG in ["cloud", "ee"]: + if not isCloudEE: return JSONResponse( { "error": "organization and/or workspace is only available in Cloud and EE" @@ -891,7 +892,7 @@ async def check_is_last_variant_for_image(db_app_variant: AppVariantDB) -> bool: query_expression = {"base.id": db_app_variant.base.id} - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: query_expression.update( { "organization.id": db_app_variant.organization.id, @@ -1031,7 +1032,7 @@ async def create_environment(name: str, app_db: AppDB) -> AppEnvironmentDB: """ environment_db = AppEnvironmentDB(app=app_db, name=name, user=app_db.user) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: environment_db.organization = app_db.organization environment_db.workspace = app_db.workspace @@ -1544,7 +1545,7 @@ async def fetch_app_by_name_and_parameters( query_expression = {"app_name": app_name} - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None @@ -1594,7 +1595,7 @@ async def create_new_evaluation( updated_at=datetime.now().isoformat(), ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -1640,7 +1641,7 @@ async def create_new_evaluation_scenario( updated_at=datetime.utcnow(), ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -1770,7 +1771,7 @@ async def create_evaluator_config( settings_values=settings_values, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index 1da1a8db1f..4c3ea9ee39 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -3,6 +3,7 @@ from typing import Dict from agenta_backend.config import settings +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.api.api_models import Image from agenta_backend.models.db_models import AppVariantDB, DeploymentDB, ImageDB from agenta_backend.services import db_manager, docker_utils @@ -11,8 +12,6 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] - async def start_service( app_variant_db: AppVariantDB, env_vars: Dict[str, str] @@ -28,7 +27,7 @@ async def start_service( True if successful, False otherwise. """ - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: uri_path = f"{app_variant_db.organization.id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.organization.id}" else: @@ -63,9 +62,9 @@ async def start_service( uri=uri, status="running", organization=app_variant_db.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, - workspace=app_variant_db.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, + workspace=app_variant_db.workspace if isCloudEE else None, ) return deployment @@ -81,7 +80,7 @@ async def remove_image(image: Image): None """ try: - if os.environ["FEATURE_FLAG"] not in ["cloud", "ee"] and image.deletable: + if not isCloudEE and image.deletable: docker_utils.delete_image(image.docker_id) logger.info(f"Image {image.docker_id} deleted") except RuntimeError as e: diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 842533ec6e..de29e8d557 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -8,6 +8,7 @@ from agenta_backend.models import converters from agenta_backend.services import db_manager +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.api.evaluation_model import ( Evaluation, @@ -23,8 +24,7 @@ NewHumanEvaluation, ) -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, UserDB_ as UserDB, @@ -163,7 +163,7 @@ async def prepare_csvdata_and_create_evaluation_scenario( outputs=[], ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: eval_scenario_instance.organization = app.organization eval_scenario_instance.workspace = app.workspace @@ -552,7 +552,7 @@ async def create_new_human_evaluation( updated_at=current_time, ) - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE: eval_instance.organization = app.organization eval_instance.workspace = app.workspace @@ -603,8 +603,8 @@ async def create_new_evaluation( status=EvaluationStatusEnum.EVALUATION_STARTED, variant=variant_id, evaluators_configs=evaluator_config_ids, - organization=app.organization if FEATURE_FLAG in ["cloud", "ee"] else None, - workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, + organization=app.organization if isCloudEE else None, + workspace=app.workspace if isCloudEE else None, ) return await converters.evaluation_db_to_pydantic(evaluation_db) diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 046512c020..28abca4486 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -5,9 +5,9 @@ from fastapi.responses import JSONResponse from agenta_backend.services import db_manager +from agenta_backend.utils.common import isCloudEE -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, EvaluatorConfigDB_ as EvaluatorConfigDB, @@ -92,9 +92,9 @@ async def create_evaluator_config( evaluator_config = await db_manager.create_evaluator_config( app=app, organization=app.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa, - workspace=app.workspace if FEATURE_FLAG in ["cloud", "ee"] else None, # noqa, + workspace=app.workspace if isCloudEE else None, # noqa, user=app.user, name=name, evaluator_key=evaluator_key, @@ -161,10 +161,10 @@ async def create_ready_to_use_evaluators(app: AppDB): await db_manager.create_evaluator_config( app=app, organization=app.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa, workspace=app.workspace - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE else None, # noqa, user=app.user, name=evaluator["name"], diff --git a/agenta-backend/agenta_backend/services/results_service.py b/agenta-backend/agenta_backend/services/results_service.py index c6e3d1bca8..14fb559de5 100644 --- a/agenta-backend/agenta_backend/services/results_service.py +++ b/agenta-backend/agenta_backend/services/results_service.py @@ -1,7 +1,10 @@ -import os +from beanie import PydanticObjectId as ObjectId + +from agenta_backend.services import db_manager +from agenta_backend.utils.common import isCloudEE +from agenta_backend.models.api.evaluation_model import EvaluationType -FEATURE_FLAG = os.environ["FEATURE_FLAG"] -if FEATURE_FLAG in ["cloud", "ee"]: +if isCloudEE: from agenta_backend.commons.models.db_models import ( HumanEvaluationDB_ as HumanEvaluationDB, EvaluationScenarioDB_ as EvaluationScenarioDB, @@ -13,10 +16,7 @@ EvaluationScenarioDB, HumanEvaluationScenarioDB, ) -from agenta_backend.services import db_manager -from agenta_backend.models.api.evaluation_model import EvaluationType -from beanie import PydanticObjectId as ObjectId async def fetch_results_for_evaluation(evaluation: HumanEvaluationDB): diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index ee1860e6bd..46f1dbf84d 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -1,11 +1,10 @@ +import os import logging from typing import Any, Callable from fastapi.types import DecoratedCallable from fastapi import APIRouter as FastAPIRouter -from agenta_backend.services import db_manager # To fix circular import - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -49,3 +48,40 @@ def decorator(func: DecoratedCallable) -> DecoratedCallable: return add_path(func) return decorator + + +async def check_action_access( + user_uid: str, + object: dict = None, + object_id: str = None, + object_type: str = None, + permission = None, + role: str = None, +) -> bool: + """ + Validate that a user has access. + + Args: + user_id (str): The user's ID. + object_id (str): The ID of the object to check. + type (str): The type of the object to check. + permission (Permission): The permission to check. + role (str): The role to check. + + Returns: + bool: True. + """ + + return True + +def isCloudEE(): + return os.environ["FEATURE_FLAG"] in ["cloud", "ee"] + +def isCloud(): + return os.environ["FEATURE_FLAG"] == "cloud" + +def isEE(): + return os.environ["FEATURE_FLAG"] == "ee" + +def isOssEE(): + return os.environ["FEATURE_FLAG"] in ["oss", "ee"] From 515e85f14e81f1aef804581023d4d72cc522da1d Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 16:11:22 +0100 Subject: [PATCH 54/99] remove session --- .../oss/20240126144938_drop_organization_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py b/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py index 8049ab94af..43687d4894 100644 --- a/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py +++ b/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py @@ -2,7 +2,7 @@ from typing import List, Optional from pydantic import BaseModel, Field -from beanie import Document, PydanticObjectId, free_fall_migration, Session +from beanie import Document, PydanticObjectId, free_fall_migration class InvitationDB(BaseModel): token: str = Field(unique=True) @@ -27,7 +27,7 @@ class Settings: class Forward: @free_fall_migration(document_models=[OldOrganizationDB]) - async def drop_old_organization_db(self, session: Session): + async def drop_old_organization_db(self, session): # Wrap deletion loop in a with_transaction context for potential rollback async with session.start_transaction(): async for old_organization in OldOrganizationDB.find_all(): From 9de7346df84c64dfc855768c2ac59697c26a6332 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 16:25:40 +0100 Subject: [PATCH 55/99] fix paranthesis --- agenta-backend/agenta_backend/main.py | 10 +-- .../agenta_backend/models/converters.py | 10 +-- .../agenta_backend/models/db_engine.py | 6 +- .../agenta_backend/routers/app_router.py | 66 +++++++++---------- .../agenta_backend/routers/bases_router.py | 6 +- .../agenta_backend/routers/configs_router.py | 8 +-- .../routers/container_router.py | 12 ++-- .../routers/environment_router.py | 6 +- .../routers/evaluation_router.py | 20 +++--- .../routers/evaluators_router.py | 14 ++-- .../routers/human_evaluation_router.py | 24 +++---- .../agenta_backend/routers/testset_router.py | 18 ++--- .../agenta_backend/routers/variants_router.py | 6 +- .../agenta_backend/services/app_manager.py | 54 +++++++-------- .../services/container_manager.py | 4 +- .../agenta_backend/services/db_manager.py | 38 +++++------ .../services/deployment_manager.py | 10 +-- .../services/evaluation_service.py | 12 ++-- .../services/evaluator_manager.py | 12 ++-- .../services/results_service.py | 4 +- agenta-backend/agenta_backend/utils/common.py | 8 +-- 21 files changed, 174 insertions(+), 174 deletions(-) diff --git a/agenta-backend/agenta_backend/main.py b/agenta-backend/agenta_backend/main.py index e5006957df..790940500e 100644 --- a/agenta-backend/agenta_backend/main.py +++ b/agenta-backend/agenta_backend/main.py @@ -18,11 +18,11 @@ configs_router, health_router, ) -from agenta_backend.utils.common import isCloudEE +from agenta_backend.utils.common import isCloudEE() from agenta_backend.models.db_engine import DBEngine from agenta_backend.open_api import open_api_tags_metadata -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.services import templates_manager else: from agenta_backend.services import templates_manager @@ -70,12 +70,12 @@ async def lifespan(application: FastAPI, cache=True): allow_headers=allow_headers, ) -if not isCloudEE: +if not isCloudEE(): from agenta_backend.services.auth_helper import authentication_middleware app.middleware("http")(authentication_middleware) -if isCloudEE: +if isCloudEE(): import agenta_backend.cloud.main as cloud app, allow_headers = cloud.extend_main(app) @@ -104,7 +104,7 @@ async def lifespan(application: FastAPI, cache=True): app.include_router(bases_router.router, prefix="/bases", tags=["Bases"]) app.include_router(configs_router.router, prefix="/configs", tags=["Configs"]) -if isCloudEE: +if isCloudEE(): import agenta_backend.cloud.main as cloud app = cloud.extend_app_schema(app) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 000205e2a1..9621f9ebd3 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -4,7 +4,7 @@ import logging from typing import List from agenta_backend.services import db_manager -from agenta_backend.utils.common import isCloudEE +from agenta_backend.utils.common import isCloudEE() from agenta_backend.models.api.user_models import User from agenta_backend.models.api.observability_models import ( @@ -23,7 +23,7 @@ EvaluationScenarioOutput, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, UserDB_ as UserDB, @@ -230,7 +230,7 @@ def app_variant_db_to_pydantic( config_name=app_variant_db.config_name, ) - if isCloudEE: + if isCloudEE(): app_variant.organization_id = str(app_variant_db.organization.id) app_variant.workspace_id = str(app_variant_db.workspace.id) @@ -262,7 +262,7 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantRe uri=uri, ) - if isCloudEE: + if isCloudEE(): variant_response.organization_id = str(app_variant_db.organization.id) variant_response.workspace_id = str(app_variant_db.workspace.id) @@ -306,7 +306,7 @@ def image_db_to_pydantic(image_db: ImageDB) -> ImageExtended: id=str(image_db.id), ) - if isCloudEE: + if isCloudEE(): image.organization_id = str(image_db.organization.id) image.workspace_id = str(image_db.workspace.id) diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index 32926f6086..3dd13606c7 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -6,9 +6,9 @@ from beanie import init_beanie, Document from motor.motor_asyncio import AsyncIOMotorClient -from agenta_backend.utils.common import isCloudEE +from agenta_backend.utils.common import isCloudEE() -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import ( APIKeyDB, WorkspaceDB, @@ -72,7 +72,7 @@ HumanEvaluationScenarioDB, ] -if isCloudEE: +if isCloudEE(): document_models = document_models + [OrganizationDB, WorkspaceDB, APIKeyDB] diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index abc7c9f829..a903b57b72 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -10,10 +10,10 @@ from agenta_backend.config import settings from agenta_backend.models import converters from agenta_backend.utils.common import ( - isEE, - isCloud, + isEE(), + isCloud(), APIRouter, - isCloudEE, + isCloudEE(), ) from agenta_backend.services import ( @@ -28,7 +28,7 @@ AddVariantFromImagePayload, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.api.api_models import ( Image_ as Image, CreateApp_ as CreateApp, @@ -42,7 +42,7 @@ AppVariantResponse, CreateAppVariant, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.services import db_manager_ee from agenta_backend.commons.services.selectors import ( get_user_own_org, @@ -56,11 +56,11 @@ from agenta_backend.commons.models.db_models import Permission -if isCloud: +if isCloud(): from agenta_backend.cloud.services import ( lambda_deployment_manager as deployment_manager, ) # noqa pylint: disable-all -elif isEE: +elif isEE(): from agenta_backend.ee.services import ( deployment_manager, ) # noqa pylint: disable-all @@ -92,7 +92,7 @@ async def list_app_variants( List[AppVariantResponse]: A list of app variants for the given app ID. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -142,7 +142,7 @@ async def get_variant_by_env( AppVariantResponse: The retrieved app variant. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -197,7 +197,7 @@ async def create_app( HTTPException: If there is an error creating the app or the user does not have permission to access the app. """ try: - if isCloudEE: + if isCloudEE(): try: user_org_workspace_data = await get_user_org_and_workspace_id( request.state.user_id @@ -256,8 +256,8 @@ async def create_app( app_db = await db_manager.create_app_and_envs( payload.app_name, request.state.user_id, - organization_id if isCloudEE else None, - workspace_id if isCloudEE else None, + organization_id if isCloudEE() else None, + workspace_id if isCloudEE() else None, ) return CreateAppOutput(app_id=str(app_db.id), app_name=str(app_db.app_name)) except Exception as e: @@ -319,7 +319,7 @@ async def add_variant_from_image( dict: The newly added variant. """ - if not isCloudEE: + if not isCloudEE(): image = Image( type="image", docker_id=payload.docker_id, @@ -336,7 +336,7 @@ async def add_variant_from_image( try: app = await db_manager.fetch_app_by_id(app_id) - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object = app, @@ -382,7 +382,7 @@ async def remove_app(app_id: str, request: Request): try: app = await db_manager.fetch_app_by_id(app_id) - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object = app, @@ -430,7 +430,7 @@ async def create_app_and_variant_from_template( try: logger.debug("Start: Creating app and variant from template") - if isCloudEE: + if isCloudEE(): # Get user and org id logger.debug("Step 1: Getting user and organization ID") user_org_workspace_data: dict = await get_user_org_and_workspace_id( @@ -463,15 +463,15 @@ async def create_app_and_variant_from_template( logger.debug( f"Step 4: Checking if app {payload.app_name} already exists" - if isCloudEE + if isCloudEE() else f"Step 1: Checking if app {payload.app_name} already exists" ) app_name = payload.app_name.lower() app = await db_manager.fetch_app_by_name_and_parameters( app_name, request.state.user_id, - payload.organization_id if isCloudEE else None, - payload.workspace_id if isCloudEE else None, + payload.organization_id if isCloudEE() else None, + payload.workspace_id if isCloudEE() else None, ) if app is not None: raise Exception( @@ -480,20 +480,20 @@ async def create_app_and_variant_from_template( logger.debug( "Step 5: Creating new app and initializing environments" - if isCloudEE + if isCloudEE() else "Step 2: Creating new app and initializing environments" ) if app is None: app = await db_manager.create_app_and_envs( app_name, request.state.user_id, - payload.organization_id if isCloudEE else None, - payload.workspace_id if isCloudEE else None, + payload.organization_id if isCloudEE() else None, + payload.workspace_id if isCloudEE() else None, ) logger.debug( "Step 6: Retrieve template from db" - if isCloudEE + if isCloudEE() else "Step 3: Retrieve template from db" ) template_db = await db_manager.get_template(payload.template_id) @@ -502,16 +502,16 @@ async def create_app_and_variant_from_template( logger.debug( "Step 7: Creating image instance and adding variant based on image" - if isCloudEE + if isCloudEE() else "Step 4: Creating image instance and adding variant based on image" ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, variant_name="app.default", docker_id_or_template_uri=template_db.template_uri - if isCloudEE + if isCloudEE() else template_db.digest, - tags=f"{image_name}" if not isCloudEE else None, + tags=f"{image_name}" if not isCloudEE() else None, base_name="app", config_name="default", is_template_image=True, @@ -520,13 +520,13 @@ async def create_app_and_variant_from_template( logger.debug( "Step 8: Creating testset for app variant" - if isCloudEE + if isCloudEE() else "Step 5: Creating testset for app variant" ) await db_manager.add_testset_to_app_variant( app_id=str(app.id), - org_id=payload.organization_id if isCloudEE else None, - workspace_id=payload.workspace_id if isCloudEE else None, + org_id=payload.organization_id if isCloudEE() else None, + workspace_id=payload.workspace_id if isCloudEE() else None, template_name=template_db.name, app_name=app.app_name, user_uid=request.state.user_id, @@ -534,17 +534,17 @@ async def create_app_and_variant_from_template( logger.debug( "Step 9: We create ready-to use evaluators" - if isCloudEE + if isCloudEE() else "Step 6: We create ready-to use evaluators" ) await evaluator_manager.create_ready_to_use_evaluators(app=app) logger.debug( "Step 10: Starting variant and injecting environment variables" - if isCloudEE + if isCloudEE() else "Step 7: Starting variant and injecting environment variables" ) - if isCloudEE: + if isCloudEE(): if not os.environ["OPENAI_API_KEY"]: raise Exception( "Unable to start app container. Please file an issue by clicking on the button below.", @@ -592,7 +592,7 @@ async def list_environments( """ logger.debug(f"Listing environments for app: {app_id}") try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index b806077e12..e4f47b6ece 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -6,10 +6,10 @@ from agenta_backend.models import converters from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils.common import APIRouter, isCloudEE() from agenta_backend.models.api.api_models import BaseOutput -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -40,7 +40,7 @@ async def list_bases( HTTPException: If there was an error retrieving the bases. """ try: - if isCloudEE and app_id is not None: + if isCloudEE() and app_id is not None: has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index 3a48e4ccc2..98e83c9af2 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -3,7 +3,7 @@ from typing import Optional from fastapi.responses import JSONResponse from fastapi import Request, HTTPException -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils.common import APIRouter, isCloudEE() from agenta_backend.models.api.api_models import ( SaveConfigPayload, @@ -14,7 +14,7 @@ app_manager, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -32,7 +32,7 @@ async def save_config( try: base_db = await db_manager.fetch_base_by_id(payload.base_id) - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object = base_db, @@ -93,7 +93,7 @@ async def get_config( base_db = await db_manager.fetch_base_by_id(base_id) # detemine whether the user has access to the base - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object = base_db, diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 2f56cb76af..2cd27827ae 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -5,9 +5,9 @@ from fastapi import Request, UploadFile, HTTPException from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter, isCloudEE, isCloud, isEE +from agenta_backend.utils.common import APIRouter, isCloudEE(), isCloud(), isEE() -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access from agenta_backend.commons.models.api.api_models import Image_ as Image @@ -15,9 +15,9 @@ from agenta_backend.models.api.api_models import Image -if isCloud: +if isCloud(): from agenta_backend.cloud.services import container_manager -elif isEE: +elif isEE(): from agenta_backend.ee.services import container_manager else: from agenta_backend.services import container_manager @@ -59,7 +59,7 @@ async def build_image( app_db = await db_manager.fetch_app_by_id(app_id) # Check app access - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object = app_db, @@ -154,7 +154,7 @@ async def construct_app_container_url( # assert that one of base_id or variant_id is provided assert base_id or variant_id, "Please provide either base_id or variant_id" - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=base_id if base_id else variant_id, diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index d18acb0eea..e9a0bec46a 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -4,10 +4,10 @@ from fastapi import Request, HTTPException from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils.common import APIRouter, isCloudEE() from agenta_backend.models.api.api_models import DeployToEnvironmentPayload -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -32,7 +32,7 @@ async def deploy_to_environment( HTTPException: If the deployment fails. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=payload.variant_id, diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index fbcbe47044..31b9a6728d 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -6,7 +6,7 @@ from fastapi import HTTPException, Request, status, Response from agenta_backend.tasks.evaluations import evaluate -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils.common import APIRouter, isCloudEE() from agenta_backend.services import evaluation_service, db_manager from agenta_backend.models.api.evaluation_model import ( @@ -22,7 +22,7 @@ check_ai_critique_inputs, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -43,7 +43,7 @@ async def create_evaluation( _description_ """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=payload.app_id, @@ -111,7 +111,7 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -148,7 +148,7 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -194,7 +194,7 @@ async def fetch_evaluation_scenarios( """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -237,7 +237,7 @@ async def fetch_list_evaluations( List[Evaluation]: A list of evaluations. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -276,7 +276,7 @@ async def fetch_evaluation( Evaluation: The fetched evaluation. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -315,7 +315,7 @@ async def delete_evaluations( """ try: - if isCloudEE: + if isCloudEE(): for evaluation_id in delete_evaluations.evaluations_ids: has_permission = await check_action_access( user_uid=request.state.user_id, @@ -380,7 +380,7 @@ async def fetch_evaluation_scenarios( try: evaluations_ids_list = evaluations_ids.split(",") - if isCloudEE: + if isCloudEE(): for evaluation_id in evaluations_ids_list: has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 71ae528a6e..4bfe315846 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse from agenta_backend.services import evaluator_manager -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils.common import APIRouter, isCloudEE() from agenta_backend.models.api.evaluation_model import ( Evaluator, @@ -14,7 +14,7 @@ UpdateEvaluatorConfig, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import Permission from agenta_backend.commons.utils.permissions import check_action_access @@ -59,7 +59,7 @@ async def get_evaluator_configs(app_id: str, request: Request): """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -91,7 +91,7 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluator_config_id, @@ -130,7 +130,7 @@ async def create_new_evaluator_config( EvaluatorConfigDB: Evaluator configuration api model. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=payload.app_id, @@ -169,7 +169,7 @@ async def update_evaluator_config( """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluator_config_id, @@ -205,7 +205,7 @@ async def delete_evaluator_config(evaluator_config_id: str, request: Request): bool: True if deletion was successful, False otherwise. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluator_config_id, diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 5b5f0e7ea4..0f5428d83e 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -1,6 +1,6 @@ from typing import List, Dict from fastapi.responses import JSONResponse -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils.common import APIRouter, isCloudEE() from fastapi import HTTPException, Body, Request, status, Response from agenta_backend.models import converters @@ -28,7 +28,7 @@ update_human_evaluation_service, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all @@ -49,7 +49,7 @@ async def create_evaluation( _description_ """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=payload.app_id, @@ -95,7 +95,7 @@ async def fetch_list_human_evaluations( List[HumanEvaluation]: A list of evaluations. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -128,7 +128,7 @@ async def fetch_human_evaluation( HumanEvaluation: The fetched evaluation. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -169,7 +169,7 @@ async def fetch_evaluation_scenarios( """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -209,7 +209,7 @@ async def update_human_evaluation( None: A 204 No Content status code, indicating that the update was successful. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -252,7 +252,7 @@ async def update_evaluation_scenario_router( None: 204 No Content status code upon successful update. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_scenario_id, @@ -292,7 +292,7 @@ async def get_evaluation_scenario_score_router( Dictionary containing the scenario ID and its score. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_scenario_id, @@ -326,7 +326,7 @@ async def update_evaluation_scenario_score_router( None: 204 No Content status code upon successful update. """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_scenario_id, @@ -363,7 +363,7 @@ async def fetch_results( """ try: - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -407,7 +407,7 @@ async def delete_evaluations( """ try: - if isCloudEE: + if isCloudEE(): for evaluation_id in delete_evaluations.evaluations_ids: has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 45b89bef7f..f422e850f7 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse from agenta_backend.services import db_manager from agenta_backend.services.db_manager import get_user -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils.common import APIRouter, isCloudEE() from fastapi import HTTPException, UploadFile, File, Form, Request from agenta_backend.models.converters import testset_db_to_pydantic @@ -23,7 +23,7 @@ TestSetOutputResponse, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all from agenta_backend.commons.models.db_models import ( Permission, @@ -62,7 +62,7 @@ async def upload_file( dict: The result of the upload process. """ - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -146,7 +146,7 @@ async def import_testset( Returns: dict: The result of the import process. """ - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -233,7 +233,7 @@ async def create_testset( str: The id of the test set created. """ - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -292,7 +292,7 @@ async def update_testset( Returns: str: The id of the test set updated. """ - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=testset_id, @@ -347,7 +347,7 @@ async def get_testsets( Raises: - `HTTPException` with status code 404 if no testsets are found. """ - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=app_id, @@ -393,7 +393,7 @@ async def get_single_testset( Returns: The requested testset if found, else an HTTPException. """ - if isCloudEE: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=testset_id, @@ -430,7 +430,7 @@ async def delete_testsets( Returns: A list of the deleted testsets' IDs. """ - if isCloudEE: + if isCloudEE(): for testset_id in delete_testsets.testset_ids: has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 1fd7d2b8bd..984b76df1e 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -6,14 +6,14 @@ from fastapi.responses import JSONResponse from agenta_backend.models import converters from fastapi import HTTPException, Request, Body -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils.common import APIRouter, isCloudEE() from agenta_backend.services import ( app_manager, db_manager, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.utils.permissions import ( check_action_access, ) # noqa pylint: disable-all @@ -296,7 +296,7 @@ async def start_variant( logger.debug("Starting variant %s", variant_id) # Inject env vars to docker container - if isCloudEE: + if isCloudEE(): if not os.environ["OPENAI_API_KEY"]: raise HTTPException( status_code=400, diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index f09727ebf6..26810e6433 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -22,24 +22,24 @@ ) from agenta_backend.utils.common import ( - isEE, - isOssEE, - isCloud, - isCloudEE, + isEE(), + isOssEE(), + isCloud(), + isCloudEE(), ) -if isCloud: +if isCloud(): from agenta_backend.cloud.services import ( lambda_deployment_manager as deployment_manager, ) # noqa pylint: disable-all -elif isEE: +elif isEE(): from agenta_backend.ee.services import ( deployment_manager, ) # noqa pylint: disable-all else: from agenta_backend.services import deployment_manager -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.services import ( api_key_service, ) # noqa pylint: disable-all @@ -75,8 +75,8 @@ async def start_variant( db_app_variant.image.docker_id, db_app_variant.image.tags, db_app_variant.app.app_name, - db_app_variant.organization if isCloudEE else None, - db_app_variant.workspace if isCloudEE else None, + db_app_variant.organization if isCloudEE() else None, + db_app_variant.workspace if isCloudEE() else None, ) logger.debug("App name is %s", db_app_variant.app.app_name) # update the env variables @@ -91,7 +91,7 @@ async def start_variant( env_vars.update( {"AGENTA_BASE_ID": str(db_app_variant.base.id), "AGENTA_HOST": domain_name} ) - if isCloudEE: + if isCloudEE(): api_key = await api_key_service.create_api_key( str(db_app_variant.user.uid), expiration_date=None, hidden=True ) @@ -135,7 +135,7 @@ async def update_variant_image(app_variant_db: AppVariantDB, image: Image): await deployment_manager.stop_and_delete_service(deployment) await db_manager.remove_deployment(deployment) - if isOssEE: + if isOssEE(): await deployment_manager.remove_image(app_variant_db.base.image) await db_manager.remove_image(app_variant_db.base.image) @@ -147,10 +147,10 @@ async def update_variant_image(app_variant_db: AppVariantDB, image: Image): user=app_variant_db.user, deletable=True, organization=app_variant_db.organization - if isCloudEE + if isCloudEE() else None, # noqa workspace=app_variant_db.workspace - if isCloudEE + if isCloudEE() else None, # noqa ) # Update base with new image @@ -229,7 +229,7 @@ async def terminate_and_remove_app_variant( # If image deletable is True, remove docker image and image db if image.deletable: try: - if isCloudEE: + if isCloudEE(): await deployment_manager.remove_repository(image.tags) else: await deployment_manager.remove_image(image) @@ -395,7 +395,7 @@ async def add_variant_based_on_image( ): raise ValueError("App variant, variant name or docker_id/template_uri is None") - if not isCloudEE: + if not isCloudEE(): if tags in [None, ""]: raise ValueError("OSS: Tags is None") @@ -418,20 +418,20 @@ async def add_variant_based_on_image( db_image = await db_manager.get_orga_image_instance_by_uri( template_uri=docker_id_or_template_uri, organization_id=str(app.organization.id) - if isCloudEE + if isCloudEE() else None, # noqa workspace_id=str(app.workspace.id) - if isCloudEE + if isCloudEE() else None, # noqa ) else: db_image = await db_manager.get_orga_image_instance_by_docker_id( docker_id=docker_id_or_template_uri, organization_id=str(app.organization.id) - if isCloudEE + if isCloudEE() else None, # noqa workspace_id=str(app.workspace.id) - if isCloudEE + if isCloudEE() else None, # noqa ) @@ -445,10 +445,10 @@ async def add_variant_based_on_image( deletable=not (is_template_image), user=user_instance, organization=app.organization - if isCloudEE + if isCloudEE() else None, # noqa workspace=app.workspace - if isCloudEE + if isCloudEE() else None, # noqa ) else: @@ -460,10 +460,10 @@ async def add_variant_based_on_image( deletable=not (is_template_image), user=user_instance, organization=app.organization - if isCloudEE + if isCloudEE() else None, # noqa workspace=app.workspace - if isCloudEE + if isCloudEE() else None, # noqa ) @@ -482,9 +482,9 @@ async def add_variant_based_on_image( db_base = await db_manager.create_new_variant_base( app=app, organization=app.organization - if isCloudEE + if isCloudEE() else None, # noqa - workspace=app.workspace if isCloudEE else None, # noqa + workspace=app.workspace if isCloudEE() else None, # noqa user=user_instance, base_name=base_name, # the first variant always has default base image=db_image, @@ -498,9 +498,9 @@ async def add_variant_based_on_image( image=db_image, user=user_instance, organization=app.organization - if isCloudEE + if isCloudEE() else None, # noqa - workspace=app.workspace if isCloudEE else None, # noqa + workspace=app.workspace if isCloudEE() else None, # noqa parameters={}, base_name=base_name, config_name=config_name, diff --git a/agenta-backend/agenta_backend/services/container_manager.py b/agenta-backend/agenta_backend/services/container_manager.py index ba1dc61d85..2e812df40c 100644 --- a/agenta-backend/agenta_backend/services/container_manager.py +++ b/agenta-backend/agenta_backend/services/container_manager.py @@ -20,7 +20,7 @@ AppDB, ) from agenta_backend.services import docker_utils -from agenta_backend.utils.common import isCloud +from agenta_backend.utils.common import isCloud() client = docker.from_env() @@ -100,7 +100,7 @@ def build_image_job( shutil.unpack_archive(tar_path, temp_dir) try: - if isCloud: + if isCloud(): dockerfile = "Dockerfile.cloud" else: dockerfile = "Dockerfile" diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 1ab472d9ca..bef07ac7b5 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse from agenta_backend.models import converters -from agenta_backend.utils.common import isCloudEE +from agenta_backend.utils.common import isCloudEE() from agenta_backend.services.json_importer_helper import get_json from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum @@ -33,7 +33,7 @@ ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.services import db_manager_ee from agenta_backend.commons.utils.permissions import check_rbac_permission from agenta_backend.commons.services.selectors import get_user_org_and_workspace_id @@ -125,7 +125,7 @@ async def add_testset_to_app_variant( user=user_db, ) - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( org_id is not None and workspace_id is not None @@ -254,7 +254,7 @@ async def create_new_variant_base( image=image, ) - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -326,7 +326,7 @@ async def create_new_app_variant( parameters=parameters, ) - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -389,7 +389,7 @@ async def create_image( image.tags = tags image.docker_id = docker_id - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -435,7 +435,7 @@ async def create_deployment( status=status, ) - if isCloudEE: + if isCloudEE(): deployment.organization = organization deployment.workspace = workspace @@ -479,7 +479,7 @@ async def create_app_and_envs( app = AppDB(app_name=app_name, user=user_instance) - if isCloudEE: + if isCloudEE(): # assert that if organization_id is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None @@ -573,7 +573,7 @@ async def get_user(user_uid: str) -> UserDB: user = await UserDB.find_one(UserDB.uid == user_uid) if user is None: - if not isCloudEE: + if not isCloudEE(): # create user user_db = UserDB(uid="0") user = await user_db.create() @@ -662,7 +662,7 @@ async def get_orga_image_instance_by_docker_id( query_expression = {"docker_id": docker_id} - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None @@ -698,7 +698,7 @@ async def get_orga_image_instance_by_uri( query_expression = {"template_uri": template_uri} - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None @@ -786,7 +786,7 @@ async def add_variant_from_base_and_config( is_deleted=False, ) - if isCloudEE: + if isCloudEE(): db_app_variant.organization = previous_app_variant_db.organization db_app_variant.workspace = previous_app_variant_db.workspace @@ -823,7 +823,7 @@ async def list_apps( return [converters.app_db_to_pydantic(app_db)] elif (org_id is not None) or (workspace_id is not None): - if not isCloudEE: + if not isCloudEE(): return JSONResponse( { "error": "organization and/or workspace is only available in Cloud and EE" @@ -892,7 +892,7 @@ async def check_is_last_variant_for_image(db_app_variant: AppVariantDB) -> bool: query_expression = {"base.id": db_app_variant.base.id} - if isCloudEE: + if isCloudEE(): query_expression.update( { "organization.id": db_app_variant.organization.id, @@ -1032,7 +1032,7 @@ async def create_environment(name: str, app_db: AppDB) -> AppEnvironmentDB: """ environment_db = AppEnvironmentDB(app=app_db, name=name, user=app_db.user) - if isCloudEE: + if isCloudEE(): environment_db.organization = app_db.organization environment_db.workspace = app_db.workspace @@ -1545,7 +1545,7 @@ async def fetch_app_by_name_and_parameters( query_expression = {"app_name": app_name} - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace_id is also provided, and vice versa assert ( organization_id is not None and workspace_id is not None @@ -1595,7 +1595,7 @@ async def create_new_evaluation( updated_at=datetime.now().isoformat(), ) - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -1641,7 +1641,7 @@ async def create_new_evaluation_scenario( updated_at=datetime.utcnow(), ) - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None @@ -1771,7 +1771,7 @@ async def create_evaluator_config( settings_values=settings_values, ) - if isCloudEE: + if isCloudEE(): # assert that if organization is provided, workspace is also provided, and vice versa assert ( organization is not None and workspace is not None diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index 4c3ea9ee39..5d9385782e 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -3,7 +3,7 @@ from typing import Dict from agenta_backend.config import settings -from agenta_backend.utils.common import isCloudEE +from agenta_backend.utils.common import isCloudEE() from agenta_backend.models.api.api_models import Image from agenta_backend.models.db_models import AppVariantDB, DeploymentDB, ImageDB from agenta_backend.services import db_manager, docker_utils @@ -27,7 +27,7 @@ async def start_service( True if successful, False otherwise. """ - if isCloudEE: + if isCloudEE(): uri_path = f"{app_variant_db.organization.id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.organization.id}" else: @@ -62,9 +62,9 @@ async def start_service( uri=uri, status="running", organization=app_variant_db.organization - if isCloudEE + if isCloudEE() else None, - workspace=app_variant_db.workspace if isCloudEE else None, + workspace=app_variant_db.workspace if isCloudEE() else None, ) return deployment @@ -80,7 +80,7 @@ async def remove_image(image: Image): None """ try: - if not isCloudEE and image.deletable: + if not isCloudEE() and image.deletable: docker_utils.delete_image(image.docker_id) logger.info(f"Image {image.docker_id} deleted") except RuntimeError as e: diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index de29e8d557..ea1228394a 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -8,7 +8,7 @@ from agenta_backend.models import converters from agenta_backend.services import db_manager -from agenta_backend.utils.common import isCloudEE +from agenta_backend.utils.common import isCloudEE() from agenta_backend.models.api.evaluation_model import ( Evaluation, @@ -24,7 +24,7 @@ NewHumanEvaluation, ) -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, UserDB_ as UserDB, @@ -163,7 +163,7 @@ async def prepare_csvdata_and_create_evaluation_scenario( outputs=[], ) - if isCloudEE: + if isCloudEE(): eval_scenario_instance.organization = app.organization eval_scenario_instance.workspace = app.workspace @@ -552,7 +552,7 @@ async def create_new_human_evaluation( updated_at=current_time, ) - if isCloudEE: + if isCloudEE(): eval_instance.organization = app.organization eval_instance.workspace = app.workspace @@ -603,8 +603,8 @@ async def create_new_evaluation( status=EvaluationStatusEnum.EVALUATION_STARTED, variant=variant_id, evaluators_configs=evaluator_config_ids, - organization=app.organization if isCloudEE else None, - workspace=app.workspace if isCloudEE else None, + organization=app.organization if isCloudEE() else None, + workspace=app.workspace if isCloudEE() else None, ) return await converters.evaluation_db_to_pydantic(evaluation_db) diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 28abca4486..2c79b4b0ba 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -5,9 +5,9 @@ from fastapi.responses import JSONResponse from agenta_backend.services import db_manager -from agenta_backend.utils.common import isCloudEE +from agenta_backend.utils.common import isCloudEE() -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import ( AppDB_ as AppDB, EvaluatorConfigDB_ as EvaluatorConfigDB, @@ -92,9 +92,9 @@ async def create_evaluator_config( evaluator_config = await db_manager.create_evaluator_config( app=app, organization=app.organization - if isCloudEE + if isCloudEE() else None, # noqa, - workspace=app.workspace if isCloudEE else None, # noqa, + workspace=app.workspace if isCloudEE() else None, # noqa, user=app.user, name=name, evaluator_key=evaluator_key, @@ -161,10 +161,10 @@ async def create_ready_to_use_evaluators(app: AppDB): await db_manager.create_evaluator_config( app=app, organization=app.organization - if isCloudEE + if isCloudEE() else None, # noqa, workspace=app.workspace - if isCloudEE + if isCloudEE() else None, # noqa, user=app.user, name=evaluator["name"], diff --git a/agenta-backend/agenta_backend/services/results_service.py b/agenta-backend/agenta_backend/services/results_service.py index 14fb559de5..fc6f6bb104 100644 --- a/agenta-backend/agenta_backend/services/results_service.py +++ b/agenta-backend/agenta_backend/services/results_service.py @@ -1,10 +1,10 @@ from beanie import PydanticObjectId as ObjectId from agenta_backend.services import db_manager -from agenta_backend.utils.common import isCloudEE +from agenta_backend.utils.common import isCloudEE() from agenta_backend.models.api.evaluation_model import EvaluationType -if isCloudEE: +if isCloudEE(): from agenta_backend.commons.models.db_models import ( HumanEvaluationDB_ as HumanEvaluationDB, EvaluationScenarioDB_ as EvaluationScenarioDB, diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index 46f1dbf84d..a960062920 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -74,14 +74,14 @@ async def check_action_access( return True -def isCloudEE(): +def isCloudEE()(): return os.environ["FEATURE_FLAG"] in ["cloud", "ee"] -def isCloud(): +def isCloud()(): return os.environ["FEATURE_FLAG"] == "cloud" -def isEE(): +def isEE()(): return os.environ["FEATURE_FLAG"] == "ee" -def isOssEE(): +def isOssEE()(): return os.environ["FEATURE_FLAG"] in ["oss", "ee"] From ab23ee3cb92952b71a2fce749b7046d0f059e142 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 16:32:24 +0100 Subject: [PATCH 56/99] remove paranthesis for imports --- agenta-backend/agenta_backend/main.py | 2 +- agenta-backend/agenta_backend/models/converters.py | 2 +- agenta-backend/agenta_backend/models/db_engine.py | 2 +- agenta-backend/agenta_backend/routers/app_router.py | 6 +++--- agenta-backend/agenta_backend/routers/bases_router.py | 2 +- agenta-backend/agenta_backend/routers/configs_router.py | 2 +- agenta-backend/agenta_backend/routers/container_router.py | 2 +- .../agenta_backend/routers/environment_router.py | 2 +- .../agenta_backend/routers/evaluation_router.py | 2 +- .../agenta_backend/routers/evaluators_router.py | 2 +- .../agenta_backend/routers/human_evaluation_router.py | 2 +- agenta-backend/agenta_backend/routers/testset_router.py | 2 +- agenta-backend/agenta_backend/routers/variants_router.py | 2 +- agenta-backend/agenta_backend/services/app_manager.py | 8 ++++---- .../agenta_backend/services/container_manager.py | 2 +- agenta-backend/agenta_backend/services/db_manager.py | 2 +- .../agenta_backend/services/deployment_manager.py | 2 +- .../agenta_backend/services/evaluation_service.py | 2 +- .../agenta_backend/services/evaluator_manager.py | 2 +- agenta-backend/agenta_backend/services/results_service.py | 2 +- agenta-backend/agenta_backend/utils/common.py | 8 ++++---- 21 files changed, 29 insertions(+), 29 deletions(-) diff --git a/agenta-backend/agenta_backend/main.py b/agenta-backend/agenta_backend/main.py index 790940500e..c556824a1e 100644 --- a/agenta-backend/agenta_backend/main.py +++ b/agenta-backend/agenta_backend/main.py @@ -18,7 +18,7 @@ configs_router, health_router, ) -from agenta_backend.utils.common import isCloudEE() +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.db_engine import DBEngine from agenta_backend.open_api import open_api_tags_metadata diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 9621f9ebd3..1ff70846d2 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -4,7 +4,7 @@ import logging from typing import List from agenta_backend.services import db_manager -from agenta_backend.utils.common import isCloudEE() +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.api.user_models import User from agenta_backend.models.api.observability_models import ( diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index 3dd13606c7..9410bd74ac 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -6,7 +6,7 @@ from beanie import init_beanie, Document from motor.motor_asyncio import AsyncIOMotorClient -from agenta_backend.utils.common import isCloudEE() +from agenta_backend.utils.common import isCloudEE if isCloudEE(): from agenta_backend.commons.models.db_models import ( diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index a903b57b72..45de4715e1 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -10,10 +10,10 @@ from agenta_backend.config import settings from agenta_backend.models import converters from agenta_backend.utils.common import ( - isEE(), - isCloud(), + isEE, + isCloud, APIRouter, - isCloudEE(), + isCloudEE, ) from agenta_backend.services import ( diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index e4f47b6ece..cf1eaf7e98 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -6,7 +6,7 @@ from agenta_backend.models import converters from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter, isCloudEE() +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import BaseOutput if isCloudEE(): diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index 98e83c9af2..cdbef4082c 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -3,7 +3,7 @@ from typing import Optional from fastapi.responses import JSONResponse from fastapi import Request, HTTPException -from agenta_backend.utils.common import APIRouter, isCloudEE() +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import ( SaveConfigPayload, diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 2cd27827ae..6a9da6c96b 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -5,7 +5,7 @@ from fastapi import Request, UploadFile, HTTPException from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter, isCloudEE(), isCloud(), isEE() +from agenta_backend.utils.common import APIRouter, isCloudEE, isCloud, isEE if isCloudEE(): from agenta_backend.commons.models.db_models import Permission diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index e9a0bec46a..152a3b98d4 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -4,7 +4,7 @@ from fastapi import Request, HTTPException from agenta_backend.services import db_manager -from agenta_backend.utils.common import APIRouter, isCloudEE() +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import DeployToEnvironmentPayload if isCloudEE(): diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 31b9a6728d..418fcb8473 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -6,7 +6,7 @@ from fastapi import HTTPException, Request, status, Response from agenta_backend.tasks.evaluations import evaluate -from agenta_backend.utils.common import APIRouter, isCloudEE() +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import evaluation_service, db_manager from agenta_backend.models.api.evaluation_model import ( diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 4bfe315846..4b1783fef5 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse from agenta_backend.services import evaluator_manager -from agenta_backend.utils.common import APIRouter, isCloudEE() +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.evaluation_model import ( Evaluator, diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 0f5428d83e..c54089cda7 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -1,6 +1,6 @@ from typing import List, Dict from fastapi.responses import JSONResponse -from agenta_backend.utils.common import APIRouter, isCloudEE() +from agenta_backend.utils.common import APIRouter, isCloudEE from fastapi import HTTPException, Body, Request, status, Response from agenta_backend.models import converters diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index f422e850f7..23ca01b577 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse from agenta_backend.services import db_manager from agenta_backend.services.db_manager import get_user -from agenta_backend.utils.common import APIRouter, isCloudEE() +from agenta_backend.utils.common import APIRouter, isCloudEE from fastapi import HTTPException, UploadFile, File, Form, Request from agenta_backend.models.converters import testset_db_to_pydantic diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 984b76df1e..4856ee8ddb 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -6,7 +6,7 @@ from fastapi.responses import JSONResponse from agenta_backend.models import converters from fastapi import HTTPException, Request, Body -from agenta_backend.utils.common import APIRouter, isCloudEE() +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import ( app_manager, diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 26810e6433..fc6053e414 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -22,10 +22,10 @@ ) from agenta_backend.utils.common import ( - isEE(), - isOssEE(), - isCloud(), - isCloudEE(), + isEE, + isOssEE, + isCloud, + isCloudEE, ) if isCloud(): diff --git a/agenta-backend/agenta_backend/services/container_manager.py b/agenta-backend/agenta_backend/services/container_manager.py index 2e812df40c..c055c83ddd 100644 --- a/agenta-backend/agenta_backend/services/container_manager.py +++ b/agenta-backend/agenta_backend/services/container_manager.py @@ -20,7 +20,7 @@ AppDB, ) from agenta_backend.services import docker_utils -from agenta_backend.utils.common import isCloud() +from agenta_backend.utils.common import isCloud client = docker.from_env() diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index bef07ac7b5..e7f107922c 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse from agenta_backend.models import converters -from agenta_backend.utils.common import isCloudEE() +from agenta_backend.utils.common import isCloudEE from agenta_backend.services.json_importer_helper import get_json from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index 5d9385782e..9af80a1f44 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -3,7 +3,7 @@ from typing import Dict from agenta_backend.config import settings -from agenta_backend.utils.common import isCloudEE() +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.api.api_models import Image from agenta_backend.models.db_models import AppVariantDB, DeploymentDB, ImageDB from agenta_backend.services import db_manager, docker_utils diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index ea1228394a..cd869e9f51 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -8,7 +8,7 @@ from agenta_backend.models import converters from agenta_backend.services import db_manager -from agenta_backend.utils.common import isCloudEE() +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.api.evaluation_model import ( Evaluation, diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 2c79b4b0ba..07300fdbce 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse from agenta_backend.services import db_manager -from agenta_backend.utils.common import isCloudEE() +from agenta_backend.utils.common import isCloudEE if isCloudEE(): from agenta_backend.commons.models.db_models import ( diff --git a/agenta-backend/agenta_backend/services/results_service.py b/agenta-backend/agenta_backend/services/results_service.py index fc6f6bb104..6cf9a826df 100644 --- a/agenta-backend/agenta_backend/services/results_service.py +++ b/agenta-backend/agenta_backend/services/results_service.py @@ -1,7 +1,7 @@ from beanie import PydanticObjectId as ObjectId from agenta_backend.services import db_manager -from agenta_backend.utils.common import isCloudEE() +from agenta_backend.utils.common import isCloudEE from agenta_backend.models.api.evaluation_model import EvaluationType if isCloudEE(): diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index a960062920..46f1dbf84d 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -74,14 +74,14 @@ async def check_action_access( return True -def isCloudEE()(): +def isCloudEE(): return os.environ["FEATURE_FLAG"] in ["cloud", "ee"] -def isCloud()(): +def isCloud(): return os.environ["FEATURE_FLAG"] == "cloud" -def isEE()(): +def isEE(): return os.environ["FEATURE_FLAG"] == "ee" -def isOssEE()(): +def isOssEE(): return os.environ["FEATURE_FLAG"] in ["oss", "ee"] From 251d2a90c87385d0b54cff0d825b5024e6f9ee5b Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 16:50:03 +0100 Subject: [PATCH 57/99] rename rbac migration folder --- .../oss/20240126100524_models_revamp.py | 0 .../oss/20240126144938_drop_organization_model.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename agenta-backend/agenta_backend/migrations/{26_01_24_rbac => v0_9_0_to_v0_11_0}/oss/20240126100524_models_revamp.py (100%) rename agenta-backend/agenta_backend/migrations/{26_01_24_rbac => v0_9_0_to_v0_11_0}/oss/20240126144938_drop_organization_model.py (100%) diff --git a/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126100524_models_revamp.py b/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/oss/20240126100524_models_revamp.py similarity index 100% rename from agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126100524_models_revamp.py rename to agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/oss/20240126100524_models_revamp.py diff --git a/agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py b/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/oss/20240126144938_drop_organization_model.py similarity index 100% rename from agenta-backend/agenta_backend/migrations/26_01_24_rbac/oss/20240126144938_drop_organization_model.py rename to agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/oss/20240126144938_drop_organization_model.py From 9d1a281399f702ead149b9fe6b95702aa1965a90 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 16:52:33 +0100 Subject: [PATCH 58/99] remove oss/cloud folder --- .../v0_9_0_to_v0_11_0/{oss => }/20240126100524_models_revamp.py | 0 .../{oss => }/20240126144938_drop_organization_model.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/{oss => }/20240126100524_models_revamp.py (100%) rename agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/{oss => }/20240126144938_drop_organization_model.py (100%) diff --git a/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/oss/20240126100524_models_revamp.py b/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py similarity index 100% rename from agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/oss/20240126100524_models_revamp.py rename to agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py diff --git a/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/oss/20240126144938_drop_organization_model.py b/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126144938_drop_organization_model.py similarity index 100% rename from agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/oss/20240126144938_drop_organization_model.py rename to agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126144938_drop_organization_model.py From 7408275388e9cfcda430a5423831ebbbc22173dc Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 21:35:41 +0100 Subject: [PATCH 59/99] update migration with current db schema --- .../20240126100524_models_revamp.py | 160 ++++++++---------- 1 file changed, 70 insertions(+), 90 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py b/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py index d87f359662..6830690646 100644 --- a/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py +++ b/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py @@ -9,17 +9,25 @@ from beanie import iterative_migration -# Common Models (BaseModels) -class ConfigVersionDB(BaseModel): - version: int - parameters: Dict[str, Any] - created_at: Optional[datetime] = Field(default=datetime.utcnow()) - updated_at: Optional[datetime] = Field(default=datetime.utcnow()) +# Common Models +class ConfigDB(BaseModel): + config_name: str + parameters: Dict[str, Any] = Field(default_factory=dict) + + +class Error(BaseModel): + message: str + stacktrace: Optional[str] = None class Result(BaseModel): type: str - value: Any + value: Optional[Any] = None + error: Optional[Error] = None + + +class InvokationResult(BaseModel): + result: Result class EvaluationScenarioResult(BaseModel): @@ -39,8 +47,7 @@ class EvaluationScenarioInputDB(BaseModel): class EvaluationScenarioOutputDB(BaseModel): - type: str - value: Any + result: Result class HumanEvaluationScenarioInput(BaseModel): @@ -85,6 +92,22 @@ class Feedback(BaseModel): updated_at: datetime = Field(default=datetime.utcnow()) +class TemplateDB(Document): + type: Optional[str] = Field(default="image") + template_uri: Optional[str] + tag_id: Optional[int] + name: str = Field(unique=True) # tag name of image + repo_name: Optional[str] + title: str + description: str + size: Optional[int] + digest: Optional[str] # sha256 hash of image digest + last_pushed: Optional[datetime] + + class Settings: + name = "templates" + + @@ -180,21 +203,10 @@ class Settings: name = "bases" -class ConfigDB(Document): - config_name: str - current_version: int = Field(default=1) - parameters: Dict[str, Any] = Field(default=dict) - version_history: List[ConfigVersionDB] = Field(default=[]) - created_at: Optional[datetime] = Field(default=datetime.utcnow()) - updated_at: Optional[datetime] = Field(default=datetime.utcnow()) - - class Settings: - name = "configs" - - class OldAppVariantDB(Document): app: Link[OldAppDB] variant_name: str + revision: int image: Link[OldImageDB] user: Link[OldUserDB] organization: Link[OldOrganizationDB] @@ -203,7 +215,7 @@ class OldAppVariantDB(Document): base_name: Optional[str] base: Link[OldVariantBaseDB] config_name: Optional[str] - config: Link[ConfigDB] + config: ConfigDB created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -215,12 +227,26 @@ class Settings: name = "app_variants" +class OldAppVariantRevisionsDB(Document): + variant: Link[OldAppVariantDB] + revision: int + modified_by: Link[OldUserDB] + base: Link[OldVariantBaseDB] + config: ConfigDB + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "app_variant_revisions" + + class OldAppEnvironmentDB(Document): app: Link[OldAppDB] name: str user: Link[OldUserDB] organization: Link[OldOrganizationDB] deployed_app_variant: Optional[PydanticObjectId] + deployed_app_variant_revision: Optional[Link[OldAppVariantRevisionsDB]] deployment: Optional[PydanticObjectId] # reference to deployment created_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -278,6 +304,7 @@ class OldHumanEvaluationDB(Document): status: str evaluation_type: str variants: List[PydanticObjectId] + variants_revisions: List[PydanticObjectId] testset: Link[OldTestSetDB] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -308,9 +335,10 @@ class OldEvaluationDB(Document): app: Link[OldAppDB] organization: Link[OldOrganizationDB] user: Link[OldUserDB] - status: str = Field(default="EVALUATION_INITIALIZED") + status: Result testset: Link[OldTestSetDB] variant: PydanticObjectId + variant_revision: PydanticObjectId evaluators_configs: List[PydanticObjectId] aggregated_results: List[AggregatedResult] created_at: datetime = Field(default=datetime.utcnow()) @@ -339,26 +367,6 @@ class Settings: name = "new_evaluation_scenarios" -class TraceDB(Document): - app_id: Optional[str] - variant_id: str - spans: List[PydanticObjectId] - start_time: datetime - end_time: datetime = Field(default=datetime.utcnow()) - cost: Optional[float] - latency: float - status: str # initiated, completed, stopped, cancelled, failed - token_consumption: Optional[int] - user: Link[OldUserDB] - tags: Optional[List[str]] - feedbacks: Optional[List[Feedback]] - - class Settings: - name = "traces" - - - - @@ -427,21 +435,10 @@ class Settings: name = "bases" -class ConfigDB(Document): - config_name: str - current_version: int = Field(default=1) - parameters: Dict[str, Any] = Field(default=dict) - version_history: List[ConfigVersionDB] = Field(default=[]) - created_at: Optional[datetime] = Field(default=datetime.utcnow()) - updated_at: Optional[datetime] = Field(default=datetime.utcnow()) - - class Settings: - name = "configs" - - class NewAppVariantDB(Document): app: Link[NewAppDB] variant_name: str + revision: int image: Link[NewImageDB] user: Link[NewUserDB] parameters: Dict[str, Any] = Field(default=dict) # TODO: deprecated. remove @@ -449,7 +446,7 @@ class NewAppVariantDB(Document): base_name: Optional[str] base: Link[NewVariantBaseDB] config_name: Optional[str] - config: Link[ConfigDB] + config: ConfigDB created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -461,11 +458,25 @@ class Settings: name = "app_variants" +class NewAppVariantRevisionsDB(Document): + variant: Link[NewAppVariantDB] + revision: int + modified_by: Link[NewUserDB] + base: Link[NewVariantBaseDB] + config: ConfigDB + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Settings: + name = "app_variant_revisions" + + class NewAppEnvironmentDB(Document): app: Link[NewAppDB] name: str user: Link[NewUserDB] deployed_app_variant: Optional[PydanticObjectId] + deployed_app_variant_revision: Optional[Link[NewAppVariantRevisionsDB]] deployment: Optional[PydanticObjectId] # reference to deployment created_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -473,22 +484,6 @@ class Settings: name = "environments" -class TemplateDB(Document): - type: Optional[str] = Field(default="image") - template_uri: Optional[str] - tag_id: Optional[int] - name: str = Field(unique=True) # tag name of image - repo_name: Optional[str] - title: str - description: str - size: Optional[int] - digest: Optional[str] # sha256 hash of image digest - last_pushed: Optional[datetime] - - class Settings: - name = "templates" - - class NewTestSetDB(Document): name: str app: Link[NewAppDB] @@ -520,6 +515,7 @@ class NewHumanEvaluationDB(Document): status: str evaluation_type: str variants: List[PydanticObjectId] + variants_revisions: List[PydanticObjectId] testset: Link[NewTestSetDB] created_at: Optional[datetime] = Field(default=datetime.utcnow()) updated_at: Optional[datetime] = Field(default=datetime.utcnow()) @@ -548,9 +544,10 @@ class Settings: class NewEvaluationDB(Document): app: Link[NewAppDB] user: Link[NewUserDB] - status: str = Field(default="EVALUATION_INITIALIZED") + status: Result testset: Link[NewTestSetDB] variant: PydanticObjectId + variant_revision: PydanticObjectId evaluators_configs: List[PydanticObjectId] aggregated_results: List[AggregatedResult] created_at: datetime = Field(default=datetime.utcnow()) @@ -578,23 +575,6 @@ class Settings: name = "new_evaluation_scenarios" -class TraceDB(Document): - app_id: Optional[str] - variant_id: str - spans: List[PydanticObjectId] - start_time: datetime - end_time: datetime = Field(default=datetime.utcnow()) - cost: Optional[float] - latency: float - status: str # initiated, completed, stopped, cancelled, failed - token_consumption: Optional[int] - user: Link[NewUserDB] - tags: Optional[List[str]] - feedbacks: Optional[List[Feedback]] - - class Settings: - name = "traces" - class Forward: From 4a871647636cd8f1ecb7019d209dc149f7ec094f Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Tue, 30 Jan 2024 21:37:35 +0100 Subject: [PATCH 60/99] refactor from merge: use object in check_action_access and --- .../routers/container_router.py | 48 +++------ .../routers/evaluation_router.py | 40 +++---- .../routers/evaluators_router.py | 10 +- .../routers/human_evaluation_router.py | 94 +++++++++++----- .../agenta_backend/routers/testset_router.py | 63 +++++------ .../agenta_backend/routers/variants_router.py | 62 +++++------ .../services/evaluation_service.py | 100 ++++-------------- .../services/evaluator_manager.py | 7 +- .../agenta_backend/tasks/evaluations.py | 15 +-- agenta-backend/agenta_backend/utils/common.py | 24 ----- 10 files changed, 200 insertions(+), 263 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 6a9da6c96b..afd6ce76d0 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -143,7 +143,7 @@ async def construct_app_container_url( Args: base_id (Optional[str]): The ID of the base to use for the app container. variant_id (Optional[str]): The ID of the variant to use for the app container. - stoken_session (SessionContainer): The session container for the user. + request (Request): The request object. Returns: URI: The URI for the app container. @@ -154,43 +154,27 @@ async def construct_app_container_url( # assert that one of base_id or variant_id is provided assert base_id or variant_id, "Please provide either base_id or variant_id" + if base_id: + object_db = await db_manager.fetch_base_by_id(base_id) + else: + object_db = await db_manager.fetch_app_variant_by_id(variant_id) + + # Check app access if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=base_id if base_id else variant_id, - object_type="base" if base_id else "app_variant", - permission=Permission.VIEW_APPLICATION, + object = object_db, + permission=Permission.READ_APPLICATION, ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, - ) - - if base_id: - base_db = await db_manager.fetch_base_by_id(base_id) - # TODO: Add status check if base_db.status == "running" - if base_db.deployment: - deployment = await db_manager.get_deployment_by_objectid(base_db.deployment) - uri = deployment.uri - else: - raise HTTPException( - status_code=400, - detail=f"Base {base_id} does not have a deployment", - ) - - return URI(uri=uri) - elif variant_id: - variant_db = await db_manager.fetch_app_variant_by_id(variant_id) - deployment = await db_manager.get_deployment_by_objectid( - variant_db.base.deployment - ) + raise HTTPException(status_code=403, detail=error_msg) + + try: + deployment = await db_manager.get_deployment_by_objectid(object_db.deployment) assert deployment and deployment.uri, "Deployment not found" return URI(uri=deployment.uri) - else: - return JSONResponse( - {"detail": "Please provide either base_id or variant_id"}, - status_code=400, - ) + except Exception as e: + return JSONResponse({"message": str(e)}, status_code=500) + diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 0e83cce2d5..da813103ee 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -5,6 +5,7 @@ from fastapi.responses import JSONResponse from fastapi import HTTPException, Request, status, Response +from agenta_backend.models import converters from agenta_backend.tasks.evaluations import evaluate from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import evaluation_service, db_manager @@ -43,11 +44,14 @@ async def create_evaluation( _description_ """ try: + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) + if app is None: + raise HTTPException(status_code=404, detail="App not found") + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=payload.app_id, - object_type="app", + object=app, permission=Permission.CREATE_EVALUATION, ) logger.debug(f"User has permission to create evaluation: {has_permission}") @@ -59,10 +63,6 @@ async def create_evaluation( status_code=403, ) - app = await db_manager.fetch_app_by_id(app_id=payload.app_id) - if app is None: - raise HTTPException(status_code=404, detail="App not found") - success, response = await check_ai_critique_inputs( payload.evaluators_configs, payload.lm_providers_keys ) @@ -117,11 +117,11 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): """ try: + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="evaluation", + object=evaluation, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -135,7 +135,6 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): status_code=403, ) - evaluation = await evaluation_service.fetch_evaluation(evaluation_id) return {"status": evaluation.status} except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -154,11 +153,11 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): """ try: + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="evaluation", + object=evaluation, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -171,8 +170,8 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): {"detail": error_msg}, status_code=403, ) - - results = await evaluation_service.retrieve_evaluation_results(evaluation_id) + + results = await converters.aggregated_result_to_pydantic(evaluation.aggregated_results) return {"results": results, "evaluation_id": evaluation_id} except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -200,11 +199,11 @@ async def fetch_evaluation_scenarios( """ try: + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="evaluation", + object=evaluation, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -220,7 +219,7 @@ async def fetch_evaluation_scenarios( eval_scenarios = ( await evaluation_service.fetch_evaluation_scenarios_for_evaluation( - evaluation_id + evaluation=evaluation ) ) return eval_scenarios @@ -243,11 +242,11 @@ async def fetch_list_evaluations( List[Evaluation]: A list of evaluations. """ try: + app = await db_manager.fetch_app_by_id(app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + object=app, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -261,7 +260,7 @@ async def fetch_list_evaluations( status_code=403, ) - return await evaluation_service.fetch_list_evaluations(app_id) + return await evaluation_service.fetch_list_evaluations(app) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -282,6 +281,7 @@ async def fetch_evaluation( Evaluation: The fetched evaluation. """ try: + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -300,7 +300,7 @@ async def fetch_evaluation( status_code=403, ) - return await evaluation_service.fetch_evaluation(evaluation_id) + return await converters.evaluation_db_to_pydantic(evaluation) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 4b1783fef5..8a2b03177f 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -4,8 +4,8 @@ from fastapi import HTTPException, Request from fastapi.responses import JSONResponse -from agenta_backend.services import evaluator_manager from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.services import evaluator_manager, db_manager from agenta_backend.models.api.evaluation_model import ( Evaluator, @@ -91,11 +91,11 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): """ try: + evaluator_config_db = await db_manager.fetch_evaluator_config(evaluator_config_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluator_config_id, - object_type="evaluator_config", + object_id=evaluator_config_db.app, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -106,9 +106,7 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): status_code=403, ) - evaluators_configs = await evaluator_manager.get_evaluator_config( - evaluator_config_id - ) + evaluators_configs = await evaluator_manager.get_evaluator_config(evaluator_config_db) return evaluators_configs except Exception as e: raise HTTPException( diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index c54089cda7..418fe208ec 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -49,6 +49,10 @@ async def create_evaluation( _description_ """ try: + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) + if app is None: + raise HTTPException(status_code=404, detail="App not found") + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -63,11 +67,6 @@ async def create_evaluation( status_code=403, ) - app = await db_manager.fetch_app_by_id(app_id=payload.app_id) - - if app is None: - raise HTTPException(status_code=404, detail="App not found") - new_human_evaluation_db = await evaluation_service.create_new_human_evaluation( payload, request.state.user_id ) @@ -128,6 +127,10 @@ async def fetch_human_evaluation( HumanEvaluation: The fetched evaluation. """ try: + human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + if not human_evaluation: + raise HTTPException(status_code=404, detail="Evaluation not found") + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -142,7 +145,7 @@ async def fetch_human_evaluation( status_code=403, ) - return await evaluation_service.fetch_human_evaluation(evaluation_id) + return await evaluation_service.fetch_human_evaluation(human_evaluation) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e @@ -170,10 +173,11 @@ async def fetch_evaluation_scenarios( try: if isCloudEE(): + human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, - object_type="human_evaluation_scenario_by_evaluation_id", + object_type="human_evaluation", permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -184,9 +188,7 @@ async def fetch_evaluation_scenarios( ) eval_scenarios = ( - await evaluation_service.fetch_human_evaluation_scenarios_for_evaluation( - evaluation_id - ) + await evaluation_service.fetch_human_evaluation_scenarios_for_evaluation(human_evaluation) ) return eval_scenarios @@ -210,10 +212,13 @@ async def update_human_evaluation( """ try: if isCloudEE(): + human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + if not human_evaluation: + raise HTTPException(status_code=404, detail="Evaluation not found") + has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="human_evaluation", + object=human_evaluation, permission=Permission.EDIT_EVALUATION, ) if not has_permission: @@ -223,7 +228,7 @@ async def update_human_evaluation( status_code=403, ) - await update_human_evaluation_service(evaluation_id, update_data) + await update_human_evaluation_service(human_evaluation, update_data) return Response(status_code=status.HTTP_204_NO_CONTENT) except KeyError: @@ -252,11 +257,23 @@ async def update_evaluation_scenario_router( None: 204 No Content status code upon successful update. """ try: + evaluation_scenario_db = db_manager.fetch_human_evaluation_scenario_by_id(evaluation_scenario_id) + if evaluation_scenario is None: + raise HTTPException( + status_code=404, + detail=f"Evaluation scenario with id {evaluation_scenario_id} not found", + ) + evaluation = evaluation_scenario.evaluation + if evaluation is None: + raise HTTPException( + status_code=404, + detail=f"Evaluation scenario for evaluation scenario with id {evaluation_scenario_id} not found", + ) + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_scenario_id, - object_type="human_evaluation_scenario", + object=evaluation_scenario_db, permission=Permission.EDIT_EVALUATION, ) if not has_permission: @@ -267,7 +284,7 @@ async def update_evaluation_scenario_router( ) await update_human_evaluation_scenario( - evaluation_scenario_id, + evaluation_scenario_db, evaluation_scenario, evaluation_type, ) @@ -292,11 +309,17 @@ async def get_evaluation_scenario_score_router( Dictionary containing the scenario ID and its score. """ try: + evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id(evaluation_scenario_id) + if evaluation_scenario is None: + raise HTTPException( + status_code=404, + detail=f"Evaluation scenario with id {evaluation_scenario_id} not found", + ) + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_scenario_id, - object_type="human_evaluation_scenario", + object=evaluation_scenario, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -306,7 +329,10 @@ async def get_evaluation_scenario_score_router( status_code=403, ) - return await get_evaluation_scenario_score_service(evaluation_scenario_id) + return { + "scenario_id": str(evaluation_scenario.id), + "score": evaluation_scenario.score, + } except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e @@ -326,11 +352,17 @@ async def update_evaluation_scenario_score_router( None: 204 No Content status code upon successful update. """ try: + evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id(evaluation_scenario_id) + if evaluation_scenario is None: + raise HTTPException( + status_code=404, + detail=f"Evaluation scenario with id {evaluation_scenario_id} not found", + ) + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_scenario_id, - object_type="human_evaluation_scenario", + object=evaluation_scenario, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -340,9 +372,9 @@ async def update_evaluation_scenario_score_router( status_code=403, ) - await update_evaluation_scenario_score_service( - evaluation_scenario_id, payload.score - ) + evaluation_scenario.score = payload.score + await evaluation_scenario.save() + return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e @@ -364,10 +396,19 @@ async def fetch_results( try: if isCloudEE(): + evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + if evaluation is None: + raise HTTPException( + status_code=404, + detail=f"Evaluation with id {evaluation_id} not found", + ) + + if not evaluation: + raise HTTPException(status_code=404, detail="Evaluation not found") + has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="evaluation", + object=evaluation, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -377,7 +418,6 @@ async def fetch_results( status_code=403, ) - evaluation = await evaluation_service._fetch_human_evaluation(evaluation_id) if evaluation.evaluation_type == EvaluationType.human_a_b_testing: results = await results_service.fetch_results_for_evaluation(evaluation) return {"votes_data": results} diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 23ca01b577..6a50385e0c 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -62,11 +62,11 @@ async def upload_file( dict: The result of the upload process. """ + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + object=app, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to upload Testset: {has_permission}") @@ -78,7 +78,6 @@ async def upload_file( status_code=403, ) - app = await db_manager.fetch_app_by_id(app_id=app_id) # Create a document document = { "created_at": datetime.now().isoformat(), @@ -146,11 +145,11 @@ async def import_testset( Returns: dict: The result of the import process. """ + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + object=app, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to import Testset: {has_permission}") @@ -162,7 +161,6 @@ async def import_testset( status_code=403, ) - app = await db_manager.fetch_app_by_id(app_id=app_id) try: response = requests.get(endpoint, timeout=10) @@ -233,11 +231,11 @@ async def create_testset( str: The id of the test set created. """ + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + object=app, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to create Testset: {has_permission}") @@ -250,7 +248,6 @@ async def create_testset( ) user = await get_user(request.state.user_id) - app = await db_manager.fetch_app_by_id(app_id=app_id) testset = { "created_at": datetime.now().isoformat(), "name": csvdata.name, @@ -292,11 +289,11 @@ async def update_testset( Returns: str: The id of the test set updated. """ + test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=testset_id, - object_type="testset", + object=test_set, permission=Permission.EDIT_TESTSET, ) logger.debug(f"User has Permission to update Testset: {has_permission}") @@ -314,7 +311,6 @@ async def update_testset( "updated_at": datetime.now().isoformat(), } - test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: raise HTTPException(status_code=404, detail="testset not found") @@ -347,11 +343,11 @@ async def get_testsets( Raises: - `HTTPException` with status code 404 if no testsets are found. """ + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + object=app, permission=Permission.VIEW_TESTSET, ) logger.debug(f"User has Permission to view Testsets: {has_permission}") @@ -363,7 +359,6 @@ async def get_testsets( status_code=403, ) - app = await db_manager.fetch_app_by_id(app_id=app_id) if app is None: raise HTTPException(status_code=404, detail="App not found") @@ -393,11 +388,11 @@ async def get_single_testset( Returns: The requested testset if found, else an HTTPException. """ + test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=testset_id, - object_type="testset", + object=test_set, permission=Permission.VIEW_TESTSET, ) logger.debug(f"User has Permission to view Testset: {has_permission}") @@ -409,7 +404,6 @@ async def get_single_testset( status_code=403, ) - test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: raise HTTPException(status_code=404, detail="testset not found") @@ -430,29 +424,28 @@ async def delete_testsets( Returns: A list of the deleted testsets' IDs. """ - if isCloudEE(): - for testset_id in delete_testsets.testset_ids: - has_permission = await check_action_access( - user_uid=request.state.user_id, - object_id=testset_id, - object_type="testset", - permission=Permission.DELETE_TESTSET, - ) - logger.debug(f"User has Permission to delete Testset: {has_permission}") - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, - ) - deleted_ids = [] for testset_id in delete_testsets.testset_ids: test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if test_set is None: raise HTTPException(status_code=404, detail="testset not found") + if isCloudEE(): + for testset_id in delete_testsets.testset_ids: + has_permission = await check_action_access( + user_uid=request.state.user_id, + object=test_set, + permission=Permission.DELETE_TESTSET, + ) + logger.debug(f"User has Permission to delete Testset: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + await test_set.delete() deleted_ids.append(testset_id) diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 0b2fbc9fd9..23ad6d41cc 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -66,13 +66,14 @@ async def add_variant_from_base_and_config( try: logger.debug("Initiating process to add a variant based on a previous one.") logger.debug(f"Received payload: {payload}") + + base_db = await db_manager.fetch_base_by_id(payload.base_id) # Check user has permission to add variant - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=payload.base_id, - object_type="base", + object=base_db, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -86,8 +87,6 @@ async def add_variant_from_base_and_config( status_code=403, ) - base_db = await db_manager.fetch_base_by_id(payload.base_id) - # Find the previous variant in the database db_app_variant = await db_manager.add_variant_from_base_and_config( base_db=base_db, @@ -124,7 +123,7 @@ async def remove_variant( HTTPException: If there is a problem removing the app variant """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=variant_id, @@ -170,7 +169,7 @@ async def update_variant_parameters( JSONResponse: A JSON response containing the updated app variant parameters. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=variant_id, @@ -220,11 +219,14 @@ async def update_variant_image( JSONResponse: A JSON response indicating whether the update was successful or not. """ try: - if FEATURE_FLAG in ["cloud", "ee"]: + db_app_variant = await db_manager.fetch_app_variant_by_id( + app_variant_id=variant_id + ) + + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=variant_id, - object_type="app_variant", + object=db_app_variant, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -238,10 +240,6 @@ async def update_variant_image( status_code=403, ) - db_app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id - ) - await app_manager.update_variant_image(db_app_variant, image) except ValueError as e: detail = f"Error while trying to update the app variant: {str(e)}" @@ -276,13 +274,13 @@ async def start_variant( Raises: HTTPException: If the app container cannot be started. """ + app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id=variant_id) # Check user has permission to start variant - if FEATURE_FLAG in ["cloud", "ee"]: + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=variant_id, - object_type="app_variant", + object=app_variant_db, permission=Permission.CREATE_APPLICATION, ) logger.debug(f"User has Permission to start variant: {has_permission}") @@ -309,7 +307,6 @@ async def start_variant( else: envvars = {} if env_vars is None else env_vars.env_vars - app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id=variant_id) if action.action == VariantActionEnum.START: url: URI = await app_manager.start_variant(app_variant_db, envvars) return url @@ -325,23 +322,26 @@ async def get_variant( request: Request, ): logger.debug("getting variant " + variant_id) - user_org_data: dict = await get_user_and_org_id(request.state.user_id) try: - user_org_data: dict = await get_user_and_org_id(request.state.user_id) - - access = await check_access_to_variant( - user_org_data=user_org_data, variant_id=variant_id - ) - if not access: - error_msg = f"You do not have access to this variant: {variant_id}" - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=400, - ) app_variant = await db_manager.fetch_app_variant_by_id( app_variant_id=variant_id ) + + if isCloudEE(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + object=app_variant, + permission=Permission.VIEW_APPLICATION, + ) + logger.debug(f"User has Permission to get variant: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + app_variant_revisions = await db_manager.list_app_variant_revisions_by_variant( app_variant=app_variant ) diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 55595f7442..efde431db2 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -212,20 +212,18 @@ async def create_evaluation_scenario( async def update_human_evaluation_service( - evaluation_id: str, update_payload: HumanEvaluationUpdate + evaluation: EvaluationDB, update_payload: HumanEvaluationUpdate ) -> None: """ Update an existing evaluation based on the provided payload. Args: - evaluation_id (str): The existing evaluation ID. + evaluation (EvaluationDB): The evaluation instance. update_payload (EvaluationUpdate): The payload for the update. Raises: HTTPException: If the evaluation is not found or access is denied. """ - # Fetch the evaluation by ID - evaluation = await _fetch_human_evaluation(evaluation_id=evaluation_id) # Prepare updates updates = {} @@ -237,13 +235,15 @@ async def update_human_evaluation_service( async def fetch_evaluation_scenarios_for_evaluation( - evaluation_id: str, + evaluation_id: str = None, + evaluation: EvaluationDB = None ) -> List[EvaluationScenario]: """ Fetch evaluation scenarios for a given evaluation ID. Args: evaluation_id (str): The ID of the evaluation. + evaluation (EvaluationDB): The evaluation instance. Raises: HTTPException: If the evaluation is not found or access is denied. @@ -251,7 +251,11 @@ async def fetch_evaluation_scenarios_for_evaluation( Returns: List[EvaluationScenario]: A list of evaluation scenarios. """ - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) + assert evaluation_id or evaluation, "Please provide either evaluation_id or evaluation" + + if not evaluation: + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) + scenarios = await EvaluationScenarioDB.find( EvaluationScenarioDB.evaluation.id == ObjectId(evaluation.id) ).to_list() @@ -263,7 +267,7 @@ async def fetch_evaluation_scenarios_for_evaluation( async def fetch_human_evaluation_scenarios_for_evaluation( - evaluation_id: str, + human_evaluation: HumanEvaluationDB ) -> List[HumanEvaluationScenario]: """ Fetch evaluation scenarios for a given evaluation ID. @@ -277,15 +281,12 @@ async def fetch_human_evaluation_scenarios_for_evaluation( Returns: List[EvaluationScenario]: A list of evaluation scenarios. """ - evaluation = await _fetch_human_evaluation( - evaluation_id=evaluation_id, - ) scenarios = await HumanEvaluationScenarioDB.find( - HumanEvaluationScenarioDB.evaluation.id == ObjectId(evaluation.id), + HumanEvaluationScenarioDB.evaluation.id == ObjectId(human_evaluation.id), ).to_list() eval_scenarios = [ converters.human_evaluation_scenario_db_to_pydantic( - scenario, str(evaluation.id) + scenario, str(human_evaluation.id) ) for scenario in scenarios ] @@ -293,7 +294,7 @@ async def fetch_human_evaluation_scenarios_for_evaluation( async def update_human_evaluation_scenario( - evaluation_scenario_id: str, + evaluation_scenario_db: HumanEvaluationScenarioDB, evaluation_scenario_data: EvaluationScenarioUpdate, evaluation_type: EvaluationType, ) -> None: @@ -301,14 +302,13 @@ async def update_human_evaluation_scenario( Updates an evaluation scenario. Args: - evaluation_scenario_id (str): The ID of the evaluation scenario. + evaluation_scenario_db (EvaluationScenarioDB): The evaluation scenario instance. evaluation_scenario_data (EvaluationScenarioUpdate): New data for the scenario. evaluation_type (EvaluationType): Type of the evaluation. Raises: HTTPException: If evaluation scenario not found or access denied. """ - eval_scenario = await _fetch_human_evaluation_scenario(evaluation_scenario_id) updated_data = evaluation_scenario_data.dict() updated_data["updated_at"] = datetime.utcnow() @@ -353,46 +353,7 @@ async def update_human_evaluation_scenario( if updated_data["correct_answer"] is not None: new_eval_set["correct_answer"] = updated_data["correct_answer"] - await eval_scenario.update({"$set": new_eval_set}) - - -async def update_evaluation_scenario_score_service( - evaluation_scenario_id: str, score: float -) -> None: - """ - Updates the score of an evaluation scenario. - - Args: - evaluation_scenario_id (str): The ID of the evaluation scenario. - score (float): The new score to set. - - Raises: - HTTPException: If evaluation scenario not found or access denied. - """ - eval_scenario = await _fetch_human_evaluation_scenario(evaluation_scenario_id) - eval_scenario.score = score - - # Save the updated evaluation scenario - await eval_scenario.save() - - -async def get_evaluation_scenario_score_service( - evaluation_scenario_id: str, -) -> Dict[str, str]: - """ - Retrieve the score of a given evaluation scenario. - - Args: - evaluation_scenario_id: The ID of the evaluation scenario. - - Returns: - Dictionary with 'scenario_id' and 'score' keys. - """ - evaluation_scenario = await _fetch_human_evaluation_scenario(evaluation_scenario_id) - return { - "scenario_id": str(evaluation_scenario.id), - "score": evaluation_scenario.score, - } + await evaluation_scenario_db.update({"$set": new_eval_set}) def _extend_with_evaluation(evaluation_type: EvaluationType): @@ -413,20 +374,20 @@ def _extend_with_correct_answer(evaluation_type: EvaluationType, row: dict): async def fetch_list_evaluations( - app_id: str, + app: AppDB, ) -> List[Evaluation]: """ Fetches a list of evaluations based on the provided filtering criteria. Args: - app_id (Optional[str]): An optional app ID to filter the evaluations. + app (AppDB): An app to filter the evaluations. Returns: List[Evaluation]: A list of evaluations. """ evaluations_db = await EvaluationDB.find( - EvaluationDB.app.id == ObjectId(app_id), fetch_links=True + EvaluationDB.app.id == app.id, fetch_links=True ).to_list() return [ await converters.evaluation_db_to_pydantic(evaluation) @@ -434,20 +395,6 @@ async def fetch_list_evaluations( ] -async def fetch_evaluation(evaluation_id: str) -> Evaluation: - """ - Fetches a single evaluation based on its ID. - - Args: - evaluation_id (str): The ID of the evaluation. - - Returns: - Evaluation: The fetched evaluation. - """ - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) - return await converters.evaluation_db_to_pydantic(evaluation) - - async def fetch_list_human_evaluations( app_id: str, ) -> List[HumanEvaluation]: @@ -469,18 +416,17 @@ async def fetch_list_human_evaluations( ] -async def fetch_human_evaluation(evaluation_id: str) -> HumanEvaluation: +async def fetch_human_evaluation(human_evaluation_db) -> HumanEvaluation: """ Fetches a single evaluation based on its ID. Args: - evaluation_id (str): The ID of the evaluation. + human_evaluation_db (HumanEvaluationDB): The evaluation instance. Returns: Evaluation: The fetched evaluation. """ - evaluation = await _fetch_human_evaluation(evaluation_id=evaluation_id) - return await converters.human_evaluation_db_to_pydantic(evaluation) + return await converters.human_evaluation_db_to_pydantic(human_evaluation_db) async def delete_human_evaluations(evaluation_ids: List[str]) -> None: @@ -658,7 +604,7 @@ async def compare_evaluations_scenarios( all_scenarios = [] for evaluation_id in evaluations_ids: - eval_scenarios = await fetch_evaluation_scenarios_for_evaluation(evaluation_id) + eval_scenarios = await fetch_evaluation_scenarios_for_evaluation(evaluation_id=evaluation_id) all_scenarios.append(eval_scenarios) grouped_scenarios_by_inputs = find_scenarios_by_input( diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 538954c3d7..141aab8b18 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -47,18 +47,17 @@ async def get_evaluators_configs(app_id: str) -> List[EvaluatorConfig]: ] -async def get_evaluator_config(evaluator_config_id: str) -> EvaluatorConfig: +async def get_evaluator_config(evaluator_config: EvaluatorConfig) -> EvaluatorConfig: """ Get an evaluator configuration by its ID. Args: - evaluator_config_id (str): The ID of the evaluator configuration. + evaluator_config: The evaluator configuration object. Returns: EvaluatorConfig: The evaluator configuration object. """ - evaluator_config_db = await db_manager.fetch_evaluator_config(evaluator_config_id) - return evaluator_config_db_to_pydantic(evaluator_config_db) + return evaluator_config_db_to_pydantic(evaluator_config) async def create_evaluator_config( diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index bc76c2467c..09568e2590 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -2,13 +2,17 @@ import asyncio import logging import traceback + from typing import Any, Dict, List +from celery import shared_task, states + +from agenta_backend.utils.common import isCloudEE +from agenta_backend.models.db_engine import DBEngine +from agenta_backend.services import evaluators_service, llm_apps_service from agenta_backend.models.api.evaluation_model import ( EvaluationStatusEnum, ) -from agenta_backend.models.db_engine import DBEngine -from agenta_backend.services import evaluators_service, llm_apps_service from agenta_backend.models.db_models import ( AggregatedResult, AppDB, @@ -37,13 +41,10 @@ EvaluationScenarioResult, check_if_evaluation_contains_failed_evaluation_scenarios, ) -from celery import shared_task, states - if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: from agenta_backend.commons.models.db_models import AppDB_ as AppDB else: from agenta_backend.models.db_models import AppDB - from agenta_backend.models.db_models import ( Result, AggregatedResult, @@ -263,10 +264,10 @@ def evaluate( ], results=evaluators_results, organization=app.organization - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE() else None, workspace=app.workspace - if FEATURE_FLAG in ["cloud", "ee"] + if isCloudEE() else None, ) ) diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index 46f1dbf84d..1c8276f8a0 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -50,30 +50,6 @@ def decorator(func: DecoratedCallable) -> DecoratedCallable: return decorator -async def check_action_access( - user_uid: str, - object: dict = None, - object_id: str = None, - object_type: str = None, - permission = None, - role: str = None, -) -> bool: - """ - Validate that a user has access. - - Args: - user_id (str): The user's ID. - object_id (str): The ID of the object to check. - type (str): The type of the object to check. - permission (Permission): The permission to check. - role (str): The role to check. - - Returns: - bool: True. - """ - - return True - def isCloudEE(): return os.environ["FEATURE_FLAG"] in ["cloud", "ee"] From 740a5d265a108dafd3ce55309b0b9f660a596b6e Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 08:28:55 +0100 Subject: [PATCH 61/99] Feat: complete merge and fix merge bugs --- .../agenta_backend/models/converters.py | 5 +- .../agenta_backend/models/db_engine.py | 2 - .../routers/human_evaluation_router.py | 2 - .../agenta_backend/services/db_manager.py | 66 +++---------------- 4 files changed, 14 insertions(+), 61 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 73f6ec56d8..f19b616e6b 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -69,6 +69,7 @@ TraceDB, TemplateDB, AggregatedResult, + AppVariantRevisionsDB, Feedback as FeedbackDB, EvaluationScenarioResult, ) @@ -79,6 +80,8 @@ TestSetOutput, TemplateImageInfo, EnvironmentOutput, + AppVariantRevision, + AppVariantOutputExtended, ) logger = logging.getLogger(__name__) @@ -287,7 +290,7 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantRe async def app_variant_db_and_revision_to_extended_output( app_variant_db: AppVariantDB, app_variant_revisions_db: AppVariantRevisionsDB -) -> AppVariantOutput: +) -> AppVariantResponse: if app_variant_db.base.deployment: deployment = await db_manager.get_deployment_by_objectid( app_variant_db.base.deployment diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index 98eda8ae39..c161d68435 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -47,7 +47,6 @@ from agenta_backend.models.db_models import ( SpanDB, TraceDB, - ConfigDB, TemplateDB, ) @@ -58,7 +57,6 @@ SpanDB, TraceDB, ImageDB, - ConfigDB, TestSetDB, TemplateDB, AppVariantDB, diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 418fe208ec..60feaf4930 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -22,8 +22,6 @@ from agenta_backend.services.evaluation_service import ( UpdateEvaluationScenarioError, - get_evaluation_scenario_score_service, - update_evaluation_scenario_score_service, update_human_evaluation_scenario, update_human_evaluation_service, ) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index d75cf65896..9d79a18584 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -5,73 +5,18 @@ from pathlib import Path from datetime import datetime from urllib.parse import urlparse -from typing import Any, Dict, List, Optional - -from agenta_backend.models.api.api_models import ( - App, - AppVariant, - ImageExtended, - Template, -) -from agenta_backend.models.converters import ( - app_db_to_pydantic, - image_db_to_pydantic, - templates_db_to_pydantic, -) -from agenta_backend.services.json_importer_helper import get_json -from agenta_backend.models.db_models import ( - HumanEvaluationDB, - HumanEvaluationScenarioDB, - Result, - AggregatedResult, - AppDB, - AppVariantDB, - AppVariantRevisionsDB, - ConfigDB, - EvaluationScenarioInputDB, - EvaluationScenarioOutputDB, - EvaluationScenarioResult, - EvaluatorConfigDB, - VariantBaseDB, - AppEnvironmentDB, - EvaluationDB, - EvaluationScenarioDB, - ImageDB, - OrganizationDB, - DeploymentDB, - TemplateDB, - TestSetDB, - UserDB, -) -from agenta_backend.utils.common import check_user_org_access -from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum - from fastapi import HTTPException from fastapi.responses import JSONResponse +from typing import Any, Dict, List, Optional from agenta_backend.models import converters from agenta_backend.utils.common import isCloudEE from agenta_backend.services.json_importer_helper import get_json -from agenta_backend.models.api.evaluation_model import EvaluationStatusEnum from agenta_backend.models.api.api_models import ( App, Template, ) - - -from agenta_backend.models.db_models import ( - Result, - ConfigDB, - TemplateDB, - ConfigVersionDB, - AggregatedResult, - EvaluationScenarioResult, - EvaluationScenarioInputDB, - EvaluationScenarioOutputDB, -) - - if isCloudEE(): from agenta_backend.commons.services import db_manager_ee from agenta_backend.commons.utils.permissions import check_rbac_permission @@ -110,6 +55,15 @@ EvaluationScenarioDB, HumanEvaluationScenarioDB, ) +from agenta_backend.models.db_models import ( + ConfigDB, + TemplateDB, + AggregatedResult, + AppVariantRevisionsDB, + EvaluationScenarioResult, + EvaluationScenarioInputDB, + EvaluationScenarioOutputDB, +) from beanie.operators import In from beanie import PydanticObjectId as ObjectId From 6512a5c89bf122ece4e9144bea43f23ab795d382 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 11:32:34 +0100 Subject: [PATCH 62/99] rename migration folder --- .../20240126100524_models_revamp.py | 60 +++++++------------ .../20240126144938_drop_organization_model.py | 4 +- 2 files changed, 25 insertions(+), 39 deletions(-) rename agenta-backend/agenta_backend/migrations/{v0_9_0_to_v0_11_0 => v0_10_0_to_v0_11_0}/20240126100524_models_revamp.py (97%) rename agenta-backend/agenta_backend/migrations/{v0_9_0_to_v0_11_0 => v0_10_0_to_v0_11_0}/20240126144938_drop_organization_model.py (100%) diff --git a/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py similarity index 97% rename from agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py rename to agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py index 6830690646..fb94fa52d1 100644 --- a/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126100524_models_revamp.py +++ b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py @@ -108,10 +108,6 @@ class Settings: name = "templates" - - - - # Old DB Models class InvitationDB(BaseModel): token: str = Field(unique=True) @@ -367,9 +363,6 @@ class Settings: name = "new_evaluation_scenarios" - - - # New DB Models class NewUserDB(Document): uid: str = Field(default="0", unique=True, index=True) @@ -575,10 +568,7 @@ class Settings: name = "new_evaluation_scenarios" - - class Forward: - @iterative_migration( document_models=[ OldUserDB, @@ -590,8 +580,7 @@ async def remove_organization_from_user_model( ): data = input_document.dict(exclude={"organizations"}) new_document = NewUserDB(**data) - - + @iterative_migration( document_models=[ OldAppDB, @@ -603,8 +592,7 @@ async def remove_organization_from_app_model( ): data = input_document.dict(exclude={"organization"}) new_document = NewAppDB(**data) - - + @iterative_migration( document_models=[ OldImageDB, @@ -616,8 +604,7 @@ async def remove_organization_from_image_model( ): data = input_document.dict(exclude={"organization"}) new_document = NewImageDB(**data) - - + @iterative_migration( document_models=[ OldTestSetDB, @@ -629,7 +616,6 @@ async def remove_organization_from_testset_model( ): data = input_document.dict(exclude={"organization"}) new_document = NewTestSetDB(**data) - @iterative_migration( document_models=[ @@ -642,8 +628,7 @@ async def remove_organization_from_variant_base_model( ): data = input_document.dict(exclude={"organization"}) new_document = NewVariantBaseDB(**data) - - + @iterative_migration( document_models=[ OldAppVariantDB, @@ -655,8 +640,7 @@ async def remove_organization_from_app_variant_model( ): data = input_document.dict(exclude={"organization"}) new_document = NewAppVariantDB(**data) - - + @iterative_migration( document_models=[ OldEvaluationDB, @@ -668,8 +652,7 @@ async def remove_organization_from_evaluation_model( ): data = input_document.dict(exclude={"organization"}) new_document = NewEvaluationDB(**data) - - + @iterative_migration( document_models=[ OldDeploymentDB, @@ -681,8 +664,7 @@ async def remove_organization_from_deployment_model( ): data = input_document.dict(exclude={"organization"}) new_document = NewDeploymentDB(**data) - - + @iterative_migration( document_models=[ OldAppEnvironmentDB, @@ -694,8 +676,7 @@ async def remove_organization_from_app_environment_model( ): data = input_document.dict(exclude={"organization"}) new_document = NewAppEnvironmentDB(**data) - - + @iterative_migration( document_models=[ OldEvaluatorConfigDB, @@ -703,12 +684,13 @@ async def remove_organization_from_app_environment_model( ] ) async def remove_organization_from_evaluator_config_model( - self, input_document: OldEvaluatorConfigDB, output_document: NewEvaluatorConfigDB + self, + input_document: OldEvaluatorConfigDB, + output_document: NewEvaluatorConfigDB, ): data = input_document.dict(exclude={"organization"}) new_document = NewEvaluatorConfigDB(**data) - - + @iterative_migration( document_models=[ OldHumanEvaluationDB, @@ -716,12 +698,13 @@ async def remove_organization_from_evaluator_config_model( ] ) async def remove_organization_from_human_evaluation_model( - self, input_document: OldHumanEvaluationDB, output_document: NewHumanEvaluationDB + self, + input_document: OldHumanEvaluationDB, + output_document: NewHumanEvaluationDB, ): data = input_document.dict(exclude={"organization"}) new_document = NewHumanEvaluationDB(**data) - - + @iterative_migration( document_models=[ OldEvaluationScenarioDB, @@ -729,11 +712,13 @@ async def remove_organization_from_human_evaluation_model( ] ) async def remove_organization_from_evaluation_scenario_model( - self, input_document: OldEvaluationScenarioDB, output_document: NewEvaluationScenarioDB + self, + input_document: OldEvaluationScenarioDB, + output_document: NewEvaluationScenarioDB, ): data = input_document.dict(exclude={"organization"}) new_document = NewEvaluationScenarioDB(**data) - + @iterative_migration( document_models=[ OldHumanEvaluationScenarioDB, @@ -741,11 +726,12 @@ async def remove_organization_from_evaluation_scenario_model( ] ) async def remove_organization_from_app_environment_model( - self, input_document: OldHumanEvaluationScenarioDB, output_document: NewHumanEvaluationScenarioDB + self, + input_document: OldHumanEvaluationScenarioDB, + output_document: NewHumanEvaluationScenarioDB, ): data = input_document.dict(exclude={"organization"}) new_document = NewHumanEvaluationScenarioDB(**data) - class Backward: diff --git a/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126144938_drop_organization_model.py b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py similarity index 100% rename from agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126144938_drop_organization_model.py rename to agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py index 43687d4894..85dd054c49 100644 --- a/agenta-backend/agenta_backend/migrations/v0_9_0_to_v0_11_0/20240126144938_drop_organization_model.py +++ b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py @@ -4,12 +4,14 @@ from beanie import Document, PydanticObjectId, free_fall_migration + class InvitationDB(BaseModel): token: str = Field(unique=True) email: str expiration_date: datetime = Field(default="0") used: bool = False + class OldOrganizationDB(Document): name: str = Field(default="agenta") description: str = Field(default="") @@ -25,7 +27,6 @@ class Settings: class Forward: - @free_fall_migration(document_models=[OldOrganizationDB]) async def drop_old_organization_db(self, session): # Wrap deletion loop in a with_transaction context for potential rollback @@ -37,6 +38,5 @@ async def drop_old_organization_db(self, session): await session.commit() - class Backward: pass From 713bd1cf31c823286e61533468baa1e0b8bb9021 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 11:33:08 +0100 Subject: [PATCH 63/99] fix bugs and format --- .../agenta_backend/models/api/api_models.py | 1 - .../agenta_backend/models/converters.py | 12 +++- .../agenta_backend/models/db_engine.py | 2 + .../agenta_backend/models/db_models.py | 1 + .../agenta_backend/routers/app_router.py | 18 +++--- .../agenta_backend/routers/configs_router.py | 9 +-- .../routers/container_router.py | 12 ++-- .../routers/evaluation_router.py | 8 ++- .../routers/evaluators_router.py | 19 +++--- .../routers/human_evaluation_router.py | 48 +++++++++----- .../routers/observability_router.py | 2 +- .../agenta_backend/routers/testset_router.py | 24 ++++--- .../agenta_backend/routers/variants_router.py | 14 +++-- .../agenta_backend/services/app_manager.py | 62 ++++++------------- .../agenta_backend/services/db_manager.py | 45 ++++++++++---- .../services/deployment_manager.py | 4 +- .../services/evaluation_service.py | 25 ++++---- .../services/evaluator_manager.py | 12 +--- .../services/llm_apps_service.py | 6 +- .../services/results_service.py | 1 - .../agenta_backend/tasks/evaluations.py | 9 +-- agenta-backend/agenta_backend/utils/common.py | 3 + 22 files changed, 182 insertions(+), 155 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index f4e1236b40..4443409bfc 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -99,7 +99,6 @@ class AppVariantOutputExtended(BaseModel): variant_name: str parameters: Optional[Dict[str, Any]] previous_variant_name: Optional[str] - organization_id: str user_id: str base_name: str base_id: str diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index f19b616e6b..a81cb7439e 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -42,6 +42,7 @@ AppVariant_ as AppVariant, ImageExtended_ as ImageExtended, AppVariantResponse_ as AppVariantResponse, + AppVariantOutputExtended_ as AppVariantOutputExtended, ) else: from agenta_backend.models.db_models import ( @@ -62,6 +63,7 @@ AppVariant, ImageExtended, AppVariantResponse, + AppVariantOutputExtended, ) from agenta_backend.models.db_models import ( @@ -81,7 +83,6 @@ TemplateImageInfo, EnvironmentOutput, AppVariantRevision, - AppVariantOutputExtended, ) logger = logging.getLogger(__name__) @@ -311,13 +312,12 @@ async def app_variant_db_and_revision_to_extended_output( created_at=app_variant_revision_db.created_at, ) ) - return AppVariantOutputExtended( + variant_extended = AppVariantOutputExtended( app_id=str(app_variant_db.app.id), app_name=str(app_variant_db.app.app_name), variant_name=app_variant_db.variant_name, variant_id=str(app_variant_db.id), user_id=str(app_variant_db.user.id), - organization_id=str(app_variant_db.organization.id), parameters=app_variant_db.config.parameters, previous_variant_name=app_variant_db.previous_variant_name, base_name=app_variant_db.base_name, @@ -327,6 +327,12 @@ async def app_variant_db_and_revision_to_extended_output( revision=app_variant_db.revision, revisions=app_variant_revisions, ) + + if isCloudEE(): + variant_extended.organization_id = str(app_variant_db.organization.id) + variant_extended.workspace_id = str(app_variant_db.workspace.id) + + return variant_extended async def environment_db_to_output( diff --git a/agenta-backend/agenta_backend/models/db_engine.py b/agenta-backend/agenta_backend/models/db_engine.py index c161d68435..6da6aadb62 100644 --- a/agenta-backend/agenta_backend/models/db_engine.py +++ b/agenta-backend/agenta_backend/models/db_engine.py @@ -48,6 +48,7 @@ SpanDB, TraceDB, TemplateDB, + AppVariantRevisionsDB, ) # Define Document Models @@ -67,6 +68,7 @@ EvaluatorConfigDB, HumanEvaluationDB, EvaluationScenarioDB, + AppVariantRevisionsDB, HumanEvaluationScenarioDB, ] diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 98694acd96..973d3c6776 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -82,6 +82,7 @@ class AppVariantDB(Document): revision: int image: Link[ImageDB] user: Link[UserDB] + modified_by: Link[UserDB] parameters: Dict[str, Any] = Field(default=dict) # TODO: deprecated. remove previous_variant_name: Optional[str] # TODO: deprecated. remove base_name: Optional[str] diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index ba1c8a704b..3a6c092581 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -11,9 +11,9 @@ from agenta_backend.models import converters from agenta_backend.utils.common import ( isEE, - isCloud, - APIRouter, - isCloudEE, + isCloud, + APIRouter, + isCloudEE, ) from agenta_backend.services import ( @@ -337,11 +337,11 @@ async def add_variant_from_image( try: app = await db_manager.fetch_app_by_id(app_id) - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object = app, + object=app, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -384,11 +384,11 @@ async def remove_app(app_id: str, request: Request): """ try: app = await db_manager.fetch_app_by_id(app_id) - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object = app, + object=app, permission=Permission.DELETE_APPLICATION, ) logger.debug(f"User has Permission to delete app: {has_permission}") @@ -442,7 +442,9 @@ async def create_app_and_variant_from_template( request.state.user_id ) - logger.debug("Step 2: Checking that workspace ID and organization ID are provided") + logger.debug( + "Step 2: Checking that workspace ID and organization ID are provided" + ) if payload.organization_id is None or payload.workspace_id is None: raise Exception( "Organization ID and Workspace ID must be provided to create app from template", diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index 248ea3b31a..3d8193337e 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -31,11 +31,11 @@ async def save_config( ): try: base_db = await db_manager.fetch_base_by_id(payload.base_id) - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object = base_db, + object=base_db, permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: @@ -58,6 +58,7 @@ async def save_config( await app_manager.update_variant_parameters( app_variant_id=str(variant_to_overwrite.id), parameters=payload.parameters, + user_uid=request.state.user_id, ) else: raise HTTPException( @@ -91,12 +92,12 @@ async def get_config( ): try: base_db = await db_manager.fetch_base_by_id(base_id) - + # detemine whether the user has access to the base if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object = base_db, + object=base_db, permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index afd6ce76d0..2490647cb4 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -57,12 +57,12 @@ async def build_image( """ try: app_db = await db_manager.fetch_app_by_id(app_id) - + # Check app access if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object = app_db, + object=app_db, permission=Permission.CREATE_APPLICATION, ) if not has_permission: @@ -73,7 +73,6 @@ async def build_image( status_code=403, ) - image_result = await container_manager.build_image( app_db=app_db, base_name=base_name, @@ -158,23 +157,22 @@ async def construct_app_container_url( object_db = await db_manager.fetch_base_by_id(base_id) else: object_db = await db_manager.fetch_app_variant_by_id(variant_id) - + # Check app access if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object = object_db, + object=object_db, permission=Permission.READ_APPLICATION, ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." logger.error(error_msg) raise HTTPException(status_code=403, detail=error_msg) - + try: deployment = await db_manager.get_deployment_by_objectid(object_db.deployment) assert deployment and deployment.uri, "Deployment not found" return URI(uri=deployment.uri) except Exception as e: return JSONResponse({"message": str(e)}, status_code=500) - diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index da813103ee..8983442636 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -47,7 +47,7 @@ async def create_evaluation( app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: raise HTTPException(status_code=404, detail="App not found") - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -170,8 +170,10 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): {"detail": error_msg}, status_code=403, ) - - results = await converters.aggregated_result_to_pydantic(evaluation.aggregated_results) + + results = await converters.aggregated_result_to_pydantic( + evaluation.aggregated_results + ) return {"results": results, "evaluation_id": evaluation_id} except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 8a2b03177f..3c1ddfc870 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -73,7 +73,7 @@ async def get_evaluator_configs(app_id: str, request: Request): {"detail": error_msg}, status_code=403, ) - + evaluators_configs = await evaluator_manager.get_evaluators_configs(app_id) return evaluators_configs except Exception as e: @@ -91,7 +91,9 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): """ try: - evaluator_config_db = await db_manager.fetch_evaluator_config(evaluator_config_id) + evaluator_config_db = await db_manager.fetch_evaluator_config( + evaluator_config_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -106,7 +108,9 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): status_code=403, ) - evaluators_configs = await evaluator_manager.get_evaluator_config(evaluator_config_db) + evaluators_configs = await evaluator_manager.get_evaluator_config( + evaluator_config_db + ) return evaluators_configs except Exception as e: raise HTTPException( @@ -115,10 +119,7 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): @router.post("/configs/", response_model=EvaluatorConfig) -async def create_new_evaluator_config( - payload: NewEvaluatorConfig, - request: Request -): +async def create_new_evaluator_config(payload: NewEvaluatorConfig, request: Request): """Endpoint to fetch evaluator configurations for a specific app. Args: @@ -181,7 +182,7 @@ async def update_evaluator_config( {"detail": error_msg}, status_code=403, ) - + evaluators_configs = await evaluator_manager.update_evaluator_config( evaluator_config_id=evaluator_config_id, updates=payload ) @@ -217,7 +218,7 @@ async def delete_evaluator_config(evaluator_config_id: str, request: Request): {"detail": error_msg}, status_code=403, ) - + success = await evaluator_manager.delete_evaluator_config(evaluator_config_id) return success except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 60feaf4930..d8bca92e97 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -27,8 +27,12 @@ ) if isCloudEE(): - from agenta_backend.commons.models.db_models import Permission # noqa pylint: disable-all - from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all + from agenta_backend.commons.models.db_models import ( + Permission, + ) # noqa pylint: disable-all + from agenta_backend.commons.utils.permissions import ( + check_action_access, + ) # noqa pylint: disable-all router = APIRouter() @@ -50,7 +54,7 @@ async def create_evaluation( app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: raise HTTPException(status_code=404, detail="App not found") - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -128,7 +132,7 @@ async def fetch_human_evaluation( human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -171,7 +175,9 @@ async def fetch_evaluation_scenarios( try: if isCloudEE(): - human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + human_evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation_id + ) has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -186,7 +192,9 @@ async def fetch_evaluation_scenarios( ) eval_scenarios = ( - await evaluation_service.fetch_human_evaluation_scenarios_for_evaluation(human_evaluation) + await evaluation_service.fetch_human_evaluation_scenarios_for_evaluation( + human_evaluation + ) ) return eval_scenarios @@ -210,10 +218,12 @@ async def update_human_evaluation( """ try: if isCloudEE(): - human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + human_evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation_id + ) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") - + has_permission = await check_action_access( user_uid=request.state.user_id, object=human_evaluation, @@ -255,7 +265,9 @@ async def update_evaluation_scenario_router( None: 204 No Content status code upon successful update. """ try: - evaluation_scenario_db = db_manager.fetch_human_evaluation_scenario_by_id(evaluation_scenario_id) + evaluation_scenario_db = db_manager.fetch_human_evaluation_scenario_by_id( + evaluation_scenario_id + ) if evaluation_scenario is None: raise HTTPException( status_code=404, @@ -307,13 +319,15 @@ async def get_evaluation_scenario_score_router( Dictionary containing the scenario ID and its score. """ try: - evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id(evaluation_scenario_id) + evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id( + evaluation_scenario_id + ) if evaluation_scenario is None: raise HTTPException( status_code=404, detail=f"Evaluation scenario with id {evaluation_scenario_id} not found", ) - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -350,13 +364,15 @@ async def update_evaluation_scenario_score_router( None: 204 No Content status code upon successful update. """ try: - evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id(evaluation_scenario_id) + evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id( + evaluation_scenario_id + ) if evaluation_scenario is None: raise HTTPException( status_code=404, detail=f"Evaluation scenario with id {evaluation_scenario_id} not found", ) - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -372,7 +388,7 @@ async def update_evaluation_scenario_score_router( evaluation_scenario.score = payload.score await evaluation_scenario.save() - + return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e @@ -400,10 +416,10 @@ async def fetch_results( status_code=404, detail=f"Evaluation with id {evaluation_id} not found", ) - + if not evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") - + has_permission = await check_action_access( user_uid=request.state.user_id, object=evaluation, diff --git a/agenta-backend/agenta_backend/routers/observability_router.py b/agenta-backend/agenta_backend/routers/observability_router.py index 7e66f9edca..45282038a4 100644 --- a/agenta-backend/agenta_backend/routers/observability_router.py +++ b/agenta-backend/agenta_backend/routers/observability_router.py @@ -70,7 +70,7 @@ async def create_span( payload: CreateSpan, request: Request, ): - spans_id = await create_trace_span(payload, request.state.user_id) + spans_id = await create_trace_span(payload) return spans_id diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 6a50385e0c..d79cbde880 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -24,7 +24,9 @@ ) if isCloudEE(): - from agenta_backend.commons.utils.permissions import check_action_access # noqa pylint: disable-all + from agenta_backend.commons.utils.permissions import ( + check_action_access, + ) # noqa pylint: disable-all from agenta_backend.commons.models.db_models import ( Permission, TestSetDB_ as TestSetDB, @@ -83,10 +85,12 @@ async def upload_file( "created_at": datetime.now().isoformat(), "name": testset_name if testset_name else file.filename, "app": app, - "organization": app.organization, - "workspace": app.workspace, "csvdata": [], } + + if isCloudEE(): + document["organization"] = app.organization + document["workspace"] = app.workspace if upload_type == "JSON": # Read and parse the JSON file @@ -161,7 +165,6 @@ async def import_testset( status_code=403, ) - try: response = requests.get(endpoint, timeout=10) @@ -175,10 +178,12 @@ async def import_testset( "created_at": datetime.now().isoformat(), "name": testset_name, "app": app, - "organization": app.organization, - "workspace": app.workspace, "csvdata": [], } + + if isCloudEE(): + document["organization"] = app.organization + document["workspace"] = app.workspace # Populate the document with column names and values json_response = response.json() @@ -252,11 +257,13 @@ async def create_testset( "created_at": datetime.now().isoformat(), "name": csvdata.name, "app": app, - "organization": app.organization, - "workspace": app.workspace, "csvdata": csvdata.csvdata, "user": user, } + + if isCloudEE(): + testset["organization"] = app.organization + testset["workspace"] = app.workspace try: testset_instance = TestSetDB(**testset) @@ -359,7 +366,6 @@ async def get_testsets( status_code=403, ) - if app is None: raise HTTPException(status_code=404, detail="App not found") diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 23ad6d41cc..a679416ed9 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -23,11 +23,13 @@ from agenta_backend.commons.models.api.api_models import ( Image_ as Image, AppVariantResponse_ as AppVariantResponse, + AppVariantOutputExtended_ as AppVariantOutputExtended, ) else: from agenta_backend.models.api.api_models import ( Image, AppVariantResponse, + AppVariantOutputExtended, ) from agenta_backend.models.api.api_models import ( @@ -37,7 +39,6 @@ VariantActionEnum, AddVariantFromBasePayload, UpdateVariantParameterPayload, - AppVariantOutputExtended, ) router = APIRouter() @@ -66,7 +67,7 @@ async def add_variant_from_base_and_config( try: logger.debug("Initiating process to add a variant based on a previous one.") logger.debug(f"Received payload: {payload}") - + base_db = await db_manager.fetch_base_by_id(payload.base_id) # Check user has permission to add variant @@ -190,6 +191,7 @@ async def update_variant_parameters( await app_manager.update_variant_parameters( app_variant_id=variant_id, parameters=payload.parameters, + user_uid=request.state.user_id, ) except ValueError as e: detail = f"Error while trying to update the app variant: {str(e)}" @@ -222,7 +224,7 @@ async def update_variant_image( db_app_variant = await db_manager.fetch_app_variant_by_id( app_variant_id=variant_id ) - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -240,7 +242,7 @@ async def update_variant_image( status_code=403, ) - await app_manager.update_variant_image(db_app_variant, image) + await app_manager.update_variant_image(db_app_variant, image, request.state.user_id) except ValueError as e: detail = f"Error while trying to update the app variant: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -326,7 +328,7 @@ async def get_variant( app_variant = await db_manager.fetch_app_variant_by_id( app_variant_id=variant_id ) - + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -341,7 +343,7 @@ async def get_variant( {"detail": error_msg}, status_code=403, ) - + app_variant_revisions = await db_manager.list_app_variant_revisions_by_variant( app_variant=app_variant ) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index b1c9959291..094cc2d0a7 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -70,8 +70,8 @@ async def start_variant( RuntimeError: If there is an error starting the Docker container. """ try: - print( - "Starting variant %s with image name %s and tags %s and app_name %s and organization %s", + logger.debug( + "Starting variant %s with image name %s and tags %s and app_name %s and organization %s and workspace %s", db_app_variant.variant_name, db_app_variant.image.docker_id, db_app_variant.image.tags, @@ -118,7 +118,7 @@ async def start_variant( return URI(uri=deployment.uri) -async def update_variant_image(app_variant_db: AppVariantDB, image: Image): +async def update_variant_image(app_variant_db: AppVariantDB, image: Image, user_uid: str): """Updates the image for app variant in the database. Arguments: @@ -147,18 +147,14 @@ async def update_variant_image(app_variant_db: AppVariantDB, image: Image): docker_id=image.docker_id, user=app_variant_db.user, deletable=True, - organization=app_variant_db.organization - if isCloudEE() - else None, # noqa - workspace=app_variant_db.workspace - if isCloudEE() - else None, # noqa + organization=app_variant_db.organization if isCloudEE() else None, # noqa + workspace=app_variant_db.workspace if isCloudEE() else None, # noqa ) # Update base with new image await db_manager.update_base(app_variant_db.base, image=db_image) # Update variant to remove configuration await db_manager.update_variant_parameters( - app_variant_db=app_variant_db, parameters={}, **user_org_data + app_variant_db=app_variant_db, parameters={}, user_uid=user_uid ) # Update variant with new image app_variant_db = await db_manager.update_app_variant(app_variant_db, image=db_image) @@ -331,11 +327,13 @@ async def remove_app(app: AppDB): raise e from None -async def update_variant_parameters(app_variant_id: str, parameters: Dict[str, Any]): +async def update_variant_parameters(app_variant_id: str, parameters: Dict[str, Any], user_uid: str): """Updates the parameters for app variant in the database. Arguments: app_variant -- the app variant to update + parameters -- the parameters to update + user_uid -- the user uid """ assert app_variant_id is not None, "app_variant_id must be provided" assert parameters is not None, "parameters must be provided" @@ -346,7 +344,7 @@ async def update_variant_parameters(app_variant_id: str, parameters: Dict[str, A raise ValueError(error_msg) try: await db_manager.update_variant_parameters( - app_variant_db=app_variant_db, parameters=parameters + app_variant_db=app_variant_db, parameters=parameters, user_uid=user_uid ) except Exception as e: logger.error( @@ -418,22 +416,14 @@ async def add_variant_based_on_image( if parsed_url.scheme and parsed_url.netloc: db_image = await db_manager.get_orga_image_instance_by_uri( template_uri=docker_id_or_template_uri, - organization_id=str(app.organization.id) - if isCloudEE() - else None, # noqa - workspace_id=str(app.workspace.id) - if isCloudEE() - else None, # noqa + organization_id=str(app.organization.id) if isCloudEE() else None, # noqa + workspace_id=str(app.workspace.id) if isCloudEE() else None, # noqa ) else: db_image = await db_manager.get_orga_image_instance_by_docker_id( docker_id=docker_id_or_template_uri, - organization_id=str(app.organization.id) - if isCloudEE() - else None, # noqa - workspace_id=str(app.workspace.id) - if isCloudEE() - else None, # noqa + organization_id=str(app.organization.id) if isCloudEE() else None, # noqa + workspace_id=str(app.workspace.id) if isCloudEE() else None, # noqa ) # Create new image if not exists @@ -445,12 +435,8 @@ async def add_variant_based_on_image( template_uri=docker_id_or_template_uri, deletable=not (is_template_image), user=user_instance, - organization=app.organization - if isCloudEE() - else None, # noqa - workspace=app.workspace - if isCloudEE() - else None, # noqa + organization=app.organization if isCloudEE() else None, # noqa + workspace=app.workspace if isCloudEE() else None, # noqa ) else: docker_id = docker_id_or_template_uri @@ -460,12 +446,8 @@ async def add_variant_based_on_image( tags=tags, deletable=not (is_template_image), user=user_instance, - organization=app.organization - if isCloudEE() - else None, # noqa - workspace=app.workspace - if isCloudEE() - else None, # noqa + organization=app.organization if isCloudEE() else None, # noqa + workspace=app.workspace if isCloudEE() else None, # noqa ) # Create config @@ -482,9 +464,7 @@ async def add_variant_based_on_image( ] # TODO: Change this in SDK2 to directly use base_name db_base = await db_manager.create_new_variant_base( app=app, - organization=app.organization - if isCloudEE() - else None, # noqa + organization=app.organization if isCloudEE() else None, # noqa workspace=app.workspace if isCloudEE() else None, # noqa user=user_instance, base_name=base_name, # the first variant always has default base @@ -498,9 +478,7 @@ async def add_variant_based_on_image( variant_name=variant_name, image=db_image, user=user_instance, - organization=app.organization - if isCloudEE() - else None, # noqa + organization=app.organization if isCloudEE() else None, # noqa workspace=app.workspace if isCloudEE() else None, # noqa parameters={}, base_name=base_name, diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 9d79a18584..7f0566c0de 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -180,6 +180,32 @@ async def fetch_app_variant_by_id( return app_variant +async def fetch_app_variant_revision_by_variant( + app_variant_id: str, revision: int +) -> AppVariantRevisionsDB: + """Fetches app variant revision by variant id and revision + + Args: + app_variant_id: str + revision: str + + Returns: + AppVariantRevisionDB + """ + assert app_variant_id is not None, "app_variant_id cannot be None" + assert revision is not None, "revision cannot be None" + app_variant_revision = await AppVariantRevisionsDB.find_one( + AppVariantRevisionsDB.variant.id == ObjectId(app_variant_id), + AppVariantRevisionsDB.revision == revision, + ) + + if app_variant_revision is None: + raise Exception( + f"app variant revision for app_variant {app_variant_id} and revision {revision} not found" + ) + return app_variant_revision + + async def fetch_base_by_id(base_id: str) -> Optional[VariantBaseDB]: """ Fetches a base by its ID. @@ -771,6 +797,8 @@ async def add_variant_from_base_and_config( variant_name=new_variant_name, image=base_db.image, user=user_db, + modified_by=user_db, + revision=1, parameters=parameters, previous_variant_name=previous_app_variant_db.variant_name, # TODO: Remove in future base_name=base_db.base_name, @@ -907,7 +935,7 @@ async def check_is_last_variant_for_image(db_app_variant: AppVariantDB) -> bool: return count_variants == 1 -async def remove_deployment(deployment_db: DeploymentDB, **kwargs: dict): +async def remove_deployment(deployment_db: DeploymentDB): """Remove a deployment from the db Arguments: @@ -937,7 +965,7 @@ async def remove_app_variant_from_db(app_variant_db: AppVariantDB): await environment.save() app_variant_revisions = await list_app_variant_revisions_by_variant( - app_variant_db, **kwargs + app_variant_db ) for app_variant_revision in app_variant_revisions: await app_variant_revision.delete() @@ -952,7 +980,6 @@ async def deploy_to_environment(environment_name: str, variant_id: str): Args: environment_name (str): The name of the environment to deploy the app variant to. variant_id (str): The ID of the app variant to deploy. - **kwargs (dict): Additional keyword arguments. Raises: ValueError: If the app variant is not found or if the environment is not found or if the app variant is already @@ -994,7 +1021,6 @@ async def list_environments(app_id: str) -> List[AppEnvironmentDB]: Args: app_id (str): The ID of the app to list environments for. - **kwargs (dict): Additional keyword arguments. Returns: List[AppEnvironmentDB]: A list of AppEnvironmentDB objects representing the environments for the given app ID. @@ -1017,7 +1043,6 @@ async def initialize_environments(app_db: AppDB) -> List[AppEnvironmentDB]: Args: app_db (AppDB): The database for the app. - **kwargs (dict): Additional keyword arguments. Returns: List[AppEnvironmentDB]: A list of the initialized environments. @@ -1036,7 +1061,6 @@ async def create_environment(name: str, app_db: AppDB) -> AppEnvironmentDB: Args: name (str): The name of the environment. app_db (AppDB): The AppDB object representing the app that the environment belongs to. - **kwargs (dict): Additional keyword arguments. Returns: AppEnvironmentDB: The newly created AppEnvironmentDB object. @@ -1052,13 +1076,12 @@ async def create_environment(name: str, app_db: AppDB) -> AppEnvironmentDB: async def list_app_variant_revisions_by_variant( - app_variant: AppVariantDB, **kwargs: dict + app_variant: AppVariantDB ) -> List[AppVariantRevisionsDB]: """Returns list of app variant revision for the given app variant Args: app_variant (AppVariantDB): The app variant to retrieve environments for. - **kwargs (dict): Additional keyword arguments. Returns: List[AppVariantRevisionsDB]: A list of AppVariantRevisionsDB objects. @@ -1077,7 +1100,6 @@ async def list_environments_by_variant( Args: app_variant (AppVariantDB): The app variant to retrieve environments for. - **kwargs (dict): Additional keyword arguments. Returns: List[AppEnvironmentDB]: A list of AppEnvironmentDB objects. @@ -1095,7 +1117,6 @@ async def remove_image(image: ImageDB): Args: image (ImageDB): The image to remove from the database. - **kwargs (dict): Additional keyword arguments. Raises: ValueError: If the image is None. @@ -1114,7 +1135,6 @@ async def remove_environment(environment_db: AppEnvironmentDB): Args: environment_db (AppEnvironmentDB): The environment to remove from the database. - **kwargs (dict): Additional keyword arguments. Raises: AssertionError: If environment_db is None. @@ -1163,7 +1183,6 @@ async def remove_base_from_db(base: VariantBaseDB): Args: base (VariantBaseDB): The base to be removed from the database. - **kwargs: Additional keyword arguments. Raises: ValueError: If the base is None. @@ -1204,7 +1223,7 @@ async def update_variant_parameters( Args: app_variant_db (AppVariantDB): The app variant to update. parameters (Dict[str, Any]): The new parameters to set for the app variant. - **kwargs (dict): Additional keyword arguments. + user_uid (str): The UID of the user that is updating the app variant. Raises: ValueError: If there is an issue updating the variant parameters. diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index 3139772485..6511d319b7 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -61,9 +61,7 @@ async def start_service( container_id=container_id, uri=uri, status="running", - organization=app_variant_db.organization - if isCloudEE() - else None, + organization=app_variant_db.organization if isCloudEE() else None, workspace=app_variant_db.workspace if isCloudEE() else None, ) return deployment diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index efde431db2..220708474b 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -163,11 +163,11 @@ async def prepare_csvdata_and_create_evaluation_scenario( inputs=list_of_scenario_input, outputs=[], ) - + if isCloudEE(): eval_scenario_instance.organization = app.organization eval_scenario_instance.workspace = app.workspace - + await eval_scenario_instance.create() @@ -235,8 +235,7 @@ async def update_human_evaluation_service( async def fetch_evaluation_scenarios_for_evaluation( - evaluation_id: str = None, - evaluation: EvaluationDB = None + evaluation_id: str = None, evaluation: EvaluationDB = None ) -> List[EvaluationScenario]: """ Fetch evaluation scenarios for a given evaluation ID. @@ -251,11 +250,13 @@ async def fetch_evaluation_scenarios_for_evaluation( Returns: List[EvaluationScenario]: A list of evaluation scenarios. """ - assert evaluation_id or evaluation, "Please provide either evaluation_id or evaluation" - + assert ( + evaluation_id or evaluation + ), "Please provide either evaluation_id or evaluation" + if not evaluation: evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) - + scenarios = await EvaluationScenarioDB.find( EvaluationScenarioDB.evaluation.id == ObjectId(evaluation.id) ).to_list() @@ -267,7 +268,7 @@ async def fetch_evaluation_scenarios_for_evaluation( async def fetch_human_evaluation_scenarios_for_evaluation( - human_evaluation: HumanEvaluationDB + human_evaluation: HumanEvaluationDB, ) -> List[HumanEvaluationScenario]: """ Fetch evaluation scenarios for a given evaluation ID. @@ -512,11 +513,11 @@ async def create_new_human_evaluation( created_at=current_time, updated_at=current_time, ) - + if isCloudEE(): eval_instance.organization = app.organization eval_instance.workspace = app.workspace - + newEvaluation = await eval_instance.create() if newEvaluation is None: raise HTTPException( @@ -604,7 +605,9 @@ async def compare_evaluations_scenarios( all_scenarios = [] for evaluation_id in evaluations_ids: - eval_scenarios = await fetch_evaluation_scenarios_for_evaluation(evaluation_id=evaluation_id) + eval_scenarios = await fetch_evaluation_scenarios_for_evaluation( + evaluation_id=evaluation_id + ) all_scenarios.append(eval_scenarios) grouped_scenarios_by_inputs = find_scenarios_by_input( diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 141aab8b18..c86a4ef946 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -81,9 +81,7 @@ async def create_evaluator_config( app = await db_manager.fetch_app_by_id(app_id) evaluator_config = await db_manager.create_evaluator_config( app=app, - organization=app.organization - if isCloudEE() - else None, # noqa, + organization=app.organization if isCloudEE() else None, # noqa, workspace=app.workspace if isCloudEE() else None, # noqa, user=app.user, name=name, @@ -150,12 +148,8 @@ async def create_ready_to_use_evaluators(app: AppDB): for evaluator in direct_use_evaluators: await db_manager.create_evaluator_config( app=app, - organization=app.organization - if isCloudEE() - else None, # noqa, - workspace=app.workspace - if isCloudEE() - else None, # noqa, + organization=app.organization if isCloudEE() else None, # noqa, + workspace=app.workspace if isCloudEE() else None, # noqa, user=app.user, name=evaluator["name"], evaluator_key=evaluator["key"], diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index 964d1bff35..e9c597e0a3 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -178,9 +178,9 @@ async def batch_invoke( "delay_between_batches" ] # Delay between batches (in seconds) - list_of_app_outputs: List[InvokationResult] = ( - [] - ) # Outputs after running all batches + list_of_app_outputs: List[ + InvokationResult + ] = [] # Outputs after running all batches openapi_parameters = await get_parameters_from_openapi(uri + "/openapi.json") async def run_batch(start_idx: int): diff --git a/agenta-backend/agenta_backend/services/results_service.py b/agenta-backend/agenta_backend/services/results_service.py index 06777af6b7..86425f9fa5 100644 --- a/agenta-backend/agenta_backend/services/results_service.py +++ b/agenta-backend/agenta_backend/services/results_service.py @@ -18,7 +18,6 @@ ) - async def fetch_results_for_evaluation(evaluation: HumanEvaluationDB): evaluation_scenarios = await HumanEvaluationScenarioDB.find( HumanEvaluationScenarioDB.evaluation.id == evaluation.id, diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index 09568e2590..942b55b75c 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -41,6 +41,7 @@ EvaluationScenarioResult, check_if_evaluation_contains_failed_evaluation_scenarios, ) + if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]: from agenta_backend.commons.models.db_models import AppDB_ as AppDB else: @@ -263,12 +264,8 @@ def evaluate( ) ], results=evaluators_results, - organization=app.organization - if isCloudEE() - else None, - workspace=app.workspace - if isCloudEE() - else None, + organization=app.organization if isCloudEE() else None, + workspace=app.workspace if isCloudEE() else None, ) ) diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index 1c8276f8a0..02af4e73b8 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -53,11 +53,14 @@ def decorator(func: DecoratedCallable) -> DecoratedCallable: def isCloudEE(): return os.environ["FEATURE_FLAG"] in ["cloud", "ee"] + def isCloud(): return os.environ["FEATURE_FLAG"] == "cloud" + def isEE(): return os.environ["FEATURE_FLAG"] == "ee" + def isOssEE(): return os.environ["FEATURE_FLAG"] in ["oss", "ee"] From f7b23c4c221e211e0b14eeab666cd40aa55bce0e Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 11:33:32 +0100 Subject: [PATCH 64/99] [draft]: fix tests failing --- .../tests/variants_main_router/conftest.py | 5 +---- .../test_app_variant_router.py | 17 ----------------- .../test_variant_evaluators_router.py | 6 ++---- 3 files changed, 3 insertions(+), 25 deletions(-) diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py b/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py index 9642311181..fdefa1852e 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py @@ -75,9 +75,7 @@ async def get_first_user_app(get_first_user_object): parameters={}, ) - db_base = VariantBaseDB( - base_name="app", image=db_image, user=user, app=app - ) + db_base = VariantBaseDB(base_name="app", image=db_image, user=user, app=app) await db_base.create() appvariant = AppVariantDB( @@ -223,7 +221,6 @@ def app_from_template(): return { "app_name": "string", "env_vars": {"OPENAI_API_KEY": OPEN_AI_KEY}, - "organization_id": "string", "template_id": "string", } diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py index a260976859..880ecdab84 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py @@ -112,23 +112,6 @@ async def test_list_app_variants(): assert len(response.json()) == 1 -@pytest.mark.asyncio -async def test_delete_app_without_permission(get_second_user_object): - user2 = await get_second_user_object - - user2_app = AppDB( - app_name="test_app_by_user2", - user=user2, - ) - await user2_app.create() - - response = await test_client.delete( - f"{BACKEND_API_HOST}/apps/{str(user2_app.id)}/", - timeout=timeout, - ) - assert response.status_code == 400 - - @pytest.mark.asyncio async def test_list_environments(): app = await AppDB.find_one(AppDB.app_name == "app_variant_test") diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py index 89b8e69544..bf5c2e584c 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py @@ -30,12 +30,10 @@ @pytest.mark.asyncio async def test_create_app_from_template( - app_from_template, fetch_user, fetch_single_prompt_template + app_from_template, fetch_single_prompt_template ): - user = await fetch_user payload = app_from_template payload["app_name"] = APP_NAME - payload["organization_id"] = str(user.organizations[0]) payload["template_id"] = fetch_single_prompt_template["id"] response = httpx.post( @@ -51,7 +49,7 @@ async def test_get_evaluators_endpoint(): timeout=timeout, ) assert response.status_code == 200 - assert len(response.json()) == 8 # currently we have 8 evaluators + assert len(response.json()) == 9 # currently we have 9 evaluators @pytest.mark.asyncio From 24f5408a0e9631b982b0df437a3dcf9fdc338841 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 11:36:15 +0100 Subject: [PATCH 65/99] add new field from prompt_versioning --- .../v0_10_0_to_v0_11_0/20240126100524_models_revamp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py index fb94fa52d1..410b64ea58 100644 --- a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py +++ b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py @@ -205,6 +205,7 @@ class OldAppVariantDB(Document): revision: int image: Link[OldImageDB] user: Link[OldUserDB] + modified_by: Link[OldUserDB] organization: Link[OldOrganizationDB] parameters: Dict[str, Any] = Field(default=dict) # TODO: deprecated. remove previous_variant_name: Optional[str] # TODO: deprecated. remove @@ -434,6 +435,7 @@ class NewAppVariantDB(Document): revision: int image: Link[NewImageDB] user: Link[NewUserDB] + modified_by: Link[NewUserDB] parameters: Dict[str, Any] = Field(default=dict) # TODO: deprecated. remove previous_variant_name: Optional[str] # TODO: deprecated. remove base_name: Optional[str] From dd37d9816d3847b68d818a4d73715d4a1e46d367 Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Wed, 31 Jan 2024 15:45:31 +0500 Subject: [PATCH 66/99] fixed import issues for dynamicComponent --- agenta-web/src/components/Evaluations/Evaluations.tsx | 3 ++- agenta-web/src/components/Playground/Views/TestView.tsx | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/agenta-web/src/components/Evaluations/Evaluations.tsx b/agenta-web/src/components/Evaluations/Evaluations.tsx index b41b85c57e..a009e946ea 100644 --- a/agenta-web/src/components/Evaluations/Evaluations.tsx +++ b/agenta-web/src/components/Evaluations/Evaluations.tsx @@ -13,7 +13,7 @@ import { } from "antd" import {DownOutlined} from "@ant-design/icons" import {createNewEvaluation, fetchVariants, useLoadTestsetsList} from "@/lib/services/api" -import {dynamicComponent, getAllProviderLlmKeys, getApikeys, isDemo} from "@/lib/helpers/utils" +import {getAllProviderLlmKeys, getApikeys, isDemo} from "@/lib/helpers/utils" import {useRouter} from "next/router" import {Variant, Parameter, GenericObject, JSSTheme} from "@/lib/Types" import {EvaluationType} from "@/lib/enums" @@ -29,6 +29,7 @@ import {createUseStyles} from "react-jss" import HumanEvaluationResult from "./HumanEvaluationResult" import {getErrorMessage} from "@/lib/helpers/errorHandler" import AutomaticEvaluationResult from "./AutomaticEvaluationResult" +import {dynamicComponent} from "@/lib/helpers/dynamic" type StyleProps = { themeMode: "dark" | "light" diff --git a/agenta-web/src/components/Playground/Views/TestView.tsx b/agenta-web/src/components/Playground/Views/TestView.tsx index 5b4e1292f5..38e3deefd6 100644 --- a/agenta-web/src/components/Playground/Views/TestView.tsx +++ b/agenta-web/src/components/Playground/Views/TestView.tsx @@ -10,7 +10,7 @@ import { Parameter, Variant, } from "@/lib/Types" -import {batchExecute, dynamicComponent, randString, removeKeys} from "@/lib/helpers/utils" +import {batchExecute, randString, removeKeys} from "@/lib/helpers/utils" import LoadTestsModal from "../LoadTestsModal" import AddToTestSetDrawer from "../AddToTestSetDrawer/AddToTestSetDrawer" import {DeleteOutlined} from "@ant-design/icons" @@ -29,6 +29,7 @@ import dayjs from "dayjs" import relativeTime from "dayjs/plugin/relativeTime" import duration from "dayjs/plugin/duration" import {useQueryParam} from "@/hooks/useQuery" +import {dynamicComponent} from "@/lib/helpers/dynamic" const PromptVersioningDrawer: any = dynamicComponent( `PromptVersioningDrawer/PromptVersioningDrawer`, From 70266b65f6d1b04a3b337aeaf699b37c726e4717 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 15:25:48 +0100 Subject: [PATCH 67/99] Add mongo_express to docker test compose file --- docker-compose.test.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index fe2525335a..65058140fb 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -61,6 +61,21 @@ services: networks: - agenta-network + mongo_express: + image: mongo-express:0.54.0 + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: username + ME_CONFIG_MONGODB_ADMINPASSWORD: password + ME_CONFIG_MONGODB_SERVER: mongo + ports: + - "8081:8081" + networks: + - agenta-network + depends_on: + mongo: + condition: service_healthy + restart: always + mongo: image: mongo:5.0 container_name: agenta-mongo-test From 0c05e7b766ca3ae792cc827ae7ade44f5298c35f Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 16:37:42 +0100 Subject: [PATCH 68/99] import re to fix failing tests --- agenta-backend/agenta_backend/tasks/evaluations.py | 1 + .../variants_main_router/test_variant_evaluators_router.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index 37385596c5..9889ebb87f 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -1,3 +1,4 @@ +import re import os import asyncio import logging diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py index 413fb4a969..b0ef877caf 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py @@ -190,7 +190,7 @@ async def test_create_evaluation(): assert response.status_code == 200 assert response_data["app_id"] == payload["app_id"] - assert response_data["status"]["value"] == EvaluationStatusEnum.EVALUATION_STARTED + assert response_data["status"]["value"] == EvaluationStatusEnum.EVALUATION_STARTED.value assert response_data is not None @@ -203,7 +203,7 @@ async def test_fetch_evaluation_status(): # Prepare and start short-polling request max_attempts = 10 - intervals = 3 # seconds + intervals = 6 # seconds for _ in range(max_attempts): response = await test_client.get( f"{BACKEND_API_HOST}/evaluations/{str(evaluation.id)}/status/", From e4d9e303db368699597c1e46f6a158af25cf7928 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 16:42:57 +0100 Subject: [PATCH 69/99] format black --- agenta-backend/agenta_backend/models/converters.py | 4 ++-- agenta-backend/agenta_backend/routers/testset_router.py | 6 +++--- agenta-backend/agenta_backend/routers/variants_router.py | 4 +++- agenta-backend/agenta_backend/services/app_manager.py | 8 ++++++-- agenta-backend/agenta_backend/services/db_manager.py | 7 +++---- .../test_variant_evaluators_router.py | 5 ++++- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index a81cb7439e..84de08f51a 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -327,11 +327,11 @@ async def app_variant_db_and_revision_to_extended_output( revision=app_variant_db.revision, revisions=app_variant_revisions, ) - + if isCloudEE(): variant_extended.organization_id = str(app_variant_db.organization.id) variant_extended.workspace_id = str(app_variant_db.workspace.id) - + return variant_extended diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index d79cbde880..649723d4fc 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -87,7 +87,7 @@ async def upload_file( "app": app, "csvdata": [], } - + if isCloudEE(): document["organization"] = app.organization document["workspace"] = app.workspace @@ -180,7 +180,7 @@ async def import_testset( "app": app, "csvdata": [], } - + if isCloudEE(): document["organization"] = app.organization document["workspace"] = app.workspace @@ -260,7 +260,7 @@ async def create_testset( "csvdata": csvdata.csvdata, "user": user, } - + if isCloudEE(): testset["organization"] = app.organization testset["workspace"] = app.workspace diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index a679416ed9..af650162c0 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -242,7 +242,9 @@ async def update_variant_image( status_code=403, ) - await app_manager.update_variant_image(db_app_variant, image, request.state.user_id) + await app_manager.update_variant_image( + db_app_variant, image, request.state.user_id + ) except ValueError as e: detail = f"Error while trying to update the app variant: {str(e)}" raise HTTPException(status_code=500, detail=detail) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 094cc2d0a7..11a50b077b 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -118,7 +118,9 @@ async def start_variant( return URI(uri=deployment.uri) -async def update_variant_image(app_variant_db: AppVariantDB, image: Image, user_uid: str): +async def update_variant_image( + app_variant_db: AppVariantDB, image: Image, user_uid: str +): """Updates the image for app variant in the database. Arguments: @@ -327,7 +329,9 @@ async def remove_app(app: AppDB): raise e from None -async def update_variant_parameters(app_variant_id: str, parameters: Dict[str, Any], user_uid: str): +async def update_variant_parameters( + app_variant_id: str, parameters: Dict[str, Any], user_uid: str +): """Updates the parameters for app variant in the database. Arguments: diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 7f0566c0de..3f06366349 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -17,6 +17,7 @@ App, Template, ) + if isCloudEE(): from agenta_backend.commons.services import db_manager_ee from agenta_backend.commons.utils.permissions import check_rbac_permission @@ -964,9 +965,7 @@ async def remove_app_variant_from_db(app_variant_db: AppVariantDB): environment.deployed_app_variant = None await environment.save() - app_variant_revisions = await list_app_variant_revisions_by_variant( - app_variant_db - ) + app_variant_revisions = await list_app_variant_revisions_by_variant(app_variant_db) for app_variant_revision in app_variant_revisions: await app_variant_revision.delete() @@ -1076,7 +1075,7 @@ async def create_environment(name: str, app_db: AppDB) -> AppEnvironmentDB: async def list_app_variant_revisions_by_variant( - app_variant: AppVariantDB + app_variant: AppVariantDB, ) -> List[AppVariantRevisionsDB]: """Returns list of app variant revision for the given app variant diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py index b0ef877caf..acb88a35aa 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py @@ -190,7 +190,10 @@ async def test_create_evaluation(): assert response.status_code == 200 assert response_data["app_id"] == payload["app_id"] - assert response_data["status"]["value"] == EvaluationStatusEnum.EVALUATION_STARTED.value + assert ( + response_data["status"]["value"] + == EvaluationStatusEnum.EVALUATION_STARTED.value + ) assert response_data is not None From 6ea28061d0cdba08666efcc5da45056db0d6716f Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 17:09:23 +0100 Subject: [PATCH 70/99] format after upgrading pip --- agenta-backend/agenta_backend/routers/app_router.py | 6 +++--- agenta-backend/agenta_backend/services/llm_apps_service.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 3a6c092581..4bfac73940 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -515,9 +515,9 @@ async def create_app_and_variant_from_template( app_variant_db = await app_manager.add_variant_based_on_image( app=app, variant_name="app.default", - docker_id_or_template_uri=template_db.template_uri - if isCloudEE() - else template_db.digest, + docker_id_or_template_uri=( + template_db.template_uri if isCloudEE() else template_db.digest + ), tags=f"{image_name}" if not isCloudEE() else None, base_name="app", config_name="default", diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index e9c597e0a3..964d1bff35 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -178,9 +178,9 @@ async def batch_invoke( "delay_between_batches" ] # Delay between batches (in seconds) - list_of_app_outputs: List[ - InvokationResult - ] = [] # Outputs after running all batches + list_of_app_outputs: List[InvokationResult] = ( + [] + ) # Outputs after running all batches openapi_parameters = await get_parameters_from_openapi(uri + "/openapi.json") async def run_batch(start_idx: int): From f0ce13dab03a5bed79ef36126b760233983740ff Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 17:39:30 +0100 Subject: [PATCH 71/99] Allow none for variant revision --- .../v0_10_0_to_v0_11_0/20240126100524_models_revamp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py index 410b64ea58..98f1ba49f4 100644 --- a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py +++ b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py @@ -335,7 +335,7 @@ class OldEvaluationDB(Document): status: Result testset: Link[OldTestSetDB] variant: PydanticObjectId - variant_revision: PydanticObjectId + variant_revision: Optional[PydanticObjectId] = None evaluators_configs: List[PydanticObjectId] aggregated_results: List[AggregatedResult] created_at: datetime = Field(default=datetime.utcnow()) @@ -542,7 +542,7 @@ class NewEvaluationDB(Document): status: Result testset: Link[NewTestSetDB] variant: PydanticObjectId - variant_revision: PydanticObjectId + variant_revision: Optional[PydanticObjectId] = None evaluators_configs: List[PydanticObjectId] aggregated_results: List[AggregatedResult] created_at: datetime = Field(default=datetime.utcnow()) From aa8f8dc1d88c9b166d60aa916ec64c301596f021 Mon Sep 17 00:00:00 2001 From: Nehemiah Onyekachukwu Emmanuel Date: Wed, 31 Jan 2024 18:23:33 +0100 Subject: [PATCH 72/99] removed session commit --- .../20240126144938_drop_organization_model.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py index 85dd054c49..784c76c68e 100644 --- a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py +++ b/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py @@ -34,9 +34,6 @@ async def drop_old_organization_db(self, session): async for old_organization in OldOrganizationDB.find_all(): await old_organization.delete() - # Commit the transaction if everything succeeds - await session.commit() - class Backward: pass From 261a121cb4baf9715975c6a7acf4d7eda154727c Mon Sep 17 00:00:00 2001 From: Abram Date: Sat, 3 Feb 2024 01:29:47 +0100 Subject: [PATCH 73/99] Refactor - remove redundant code in modes_revamp in oss rbac migration and renamed folder to next version release --- .../20240126100524_models_revamp.py | 39 +++++++------------ .../20240126144938_drop_organization_model.py | 0 2 files changed, 13 insertions(+), 26 deletions(-) rename agenta-backend/agenta_backend/migrations/{v0_10_0_to_v0_11_0 => v0_11_0_to_v0_12_0}/20240126100524_models_revamp.py (93%) rename agenta-backend/agenta_backend/migrations/{v0_10_0_to_v0_11_0 => v0_11_0_to_v0_12_0}/20240126144938_drop_organization_model.py (100%) diff --git a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py b/agenta-backend/agenta_backend/migrations/v0_11_0_to_v0_12_0/20240126100524_models_revamp.py similarity index 93% rename from agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py rename to agenta-backend/agenta_backend/migrations/v0_11_0_to_v0_12_0/20240126100524_models_revamp.py index 98f1ba49f4..2305e27323 100644 --- a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126100524_models_revamp.py +++ b/agenta-backend/agenta_backend/migrations/v0_11_0_to_v0_12_0/20240126100524_models_revamp.py @@ -580,8 +580,7 @@ class Forward: async def remove_organization_from_user_model( self, input_document: OldUserDB, output_document: NewUserDB ): - data = input_document.dict(exclude={"organizations"}) - new_document = NewUserDB(**data) + input_document.dict(exclude={"organizations"}) @iterative_migration( document_models=[ @@ -592,8 +591,7 @@ async def remove_organization_from_user_model( async def remove_organization_from_app_model( self, input_document: OldAppDB, output_document: NewAppDB ): - data = input_document.dict(exclude={"organization"}) - new_document = NewAppDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -604,8 +602,7 @@ async def remove_organization_from_app_model( async def remove_organization_from_image_model( self, input_document: OldImageDB, output_document: NewImageDB ): - data = input_document.dict(exclude={"organization"}) - new_document = NewImageDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -616,8 +613,7 @@ async def remove_organization_from_image_model( async def remove_organization_from_testset_model( self, input_document: OldTestSetDB, output_document: NewTestSetDB ): - data = input_document.dict(exclude={"organization"}) - new_document = NewTestSetDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -628,8 +624,7 @@ async def remove_organization_from_testset_model( async def remove_organization_from_variant_base_model( self, input_document: OldVariantBaseDB, output_document: NewVariantBaseDB ): - data = input_document.dict(exclude={"organization"}) - new_document = NewVariantBaseDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -640,8 +635,7 @@ async def remove_organization_from_variant_base_model( async def remove_organization_from_app_variant_model( self, input_document: OldAppVariantDB, output_document: NewAppVariantDB ): - data = input_document.dict(exclude={"organization"}) - new_document = NewAppVariantDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -652,8 +646,7 @@ async def remove_organization_from_app_variant_model( async def remove_organization_from_evaluation_model( self, input_document: OldEvaluationDB, output_document: NewEvaluationDB ): - data = input_document.dict(exclude={"organization"}) - new_document = NewEvaluationDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -664,8 +657,7 @@ async def remove_organization_from_evaluation_model( async def remove_organization_from_deployment_model( self, input_document: OldDeploymentDB, output_document: NewDeploymentDB ): - data = input_document.dict(exclude={"organization"}) - new_document = NewDeploymentDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -676,8 +668,7 @@ async def remove_organization_from_deployment_model( async def remove_organization_from_app_environment_model( self, input_document: OldAppEnvironmentDB, output_document: NewAppEnvironmentDB ): - data = input_document.dict(exclude={"organization"}) - new_document = NewAppEnvironmentDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -690,8 +681,7 @@ async def remove_organization_from_evaluator_config_model( input_document: OldEvaluatorConfigDB, output_document: NewEvaluatorConfigDB, ): - data = input_document.dict(exclude={"organization"}) - new_document = NewEvaluatorConfigDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -704,8 +694,7 @@ async def remove_organization_from_human_evaluation_model( input_document: OldHumanEvaluationDB, output_document: NewHumanEvaluationDB, ): - data = input_document.dict(exclude={"organization"}) - new_document = NewHumanEvaluationDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -718,8 +707,7 @@ async def remove_organization_from_evaluation_scenario_model( input_document: OldEvaluationScenarioDB, output_document: NewEvaluationScenarioDB, ): - data = input_document.dict(exclude={"organization"}) - new_document = NewEvaluationScenarioDB(**data) + input_document.dict(exclude={"organization"}) @iterative_migration( document_models=[ @@ -732,8 +720,7 @@ async def remove_organization_from_app_environment_model( input_document: OldHumanEvaluationScenarioDB, output_document: NewHumanEvaluationScenarioDB, ): - data = input_document.dict(exclude={"organization"}) - new_document = NewHumanEvaluationScenarioDB(**data) + input_document.dict(exclude={"organization"}) class Backward: diff --git a/agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py b/agenta-backend/agenta_backend/migrations/v0_11_0_to_v0_12_0/20240126144938_drop_organization_model.py similarity index 100% rename from agenta-backend/agenta_backend/migrations/v0_10_0_to_v0_11_0/20240126144938_drop_organization_model.py rename to agenta-backend/agenta_backend/migrations/v0_11_0_to_v0_12_0/20240126144938_drop_organization_model.py From b9f9f12665f1b10b6bd7a70fe87279dbc566fa72 Mon Sep 17 00:00:00 2001 From: Akrem Abayed Date: Tue, 6 Feb 2024 15:56:04 +0100 Subject: [PATCH 74/99] fix removing app --- agenta-backend/agenta_backend/routers/app_router.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 4bfac73940..63f26ac21c 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -399,8 +399,7 @@ async def remove_app(app_id: str, request: Request): status_code=403, ) - else: - await app_manager.remove_app(app) + await app_manager.remove_app(app) except DockerException as e: detail = f"Docker error while trying to remove the app: {str(e)}" logger.exception(f"Docker error while trying to remove the app: {str(e)}") From 380ca574a6a387096367e503bb32b179e659307b Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 17:05:49 +0100 Subject: [PATCH 75/99] Update - resolve 500 error in getting app container_url --- agenta-backend/agenta_backend/routers/container_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 2490647cb4..8597b52cc4 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -163,7 +163,7 @@ async def construct_app_container_url( has_permission = await check_action_access( user_uid=request.state.user_id, object=object_db, - permission=Permission.READ_APPLICATION, + permission=Permission.VIEW_APPLICATION, ) if not has_permission: error_msg = f"You do not have permission to perform this action. Please contact your organization admin." From 8bd37678400311169dcfae5e112f38daeb9cd9aa Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 21:06:42 +0100 Subject: [PATCH 76/99] Update - resolve breaking endpoint page in agenta web --- agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx b/agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx index 263325b5a7..4d8f425a5a 100644 --- a/agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx +++ b/agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx @@ -5,7 +5,8 @@ import DynamicCodeBlock from "@/components/DynamicCodeBlock/DynamicCodeBlock" import ResultComponent from "@/components/ResultComponent/ResultComponent" import {useQueryParam} from "@/hooks/useQuery" import {Environment, GenericObject, Parameter, Variant} from "@/lib/Types" -import {dynamicComponent, isDemo} from "@/lib/helpers/utils" +import {isDemo} from "@/lib/helpers/utils" +import {dynamicComponent} from "@/lib/helpers/dynamic" import {useVariant} from "@/lib/hooks/useVariant" import {fetchEnvironments, fetchVariants, getAppContainerURL} from "@/lib/services/api" import {ApiOutlined, AppstoreOutlined, DownOutlined, HistoryOutlined} from "@ant-design/icons" @@ -16,6 +17,7 @@ import {createUseStyles} from "react-jss" const DeploymentHistory: any = dynamicComponent("DeploymentHistory/DeploymentHistory") + const {Text, Title} = Typography const useStyles = createUseStyles({ From 26dba6a6999c9184fda1bf7bb06658d6665574f0 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 21:17:28 +0100 Subject: [PATCH 77/99] Update - fix failing deploy endpoint and update deploy_to_environment db function --- agenta-backend/agenta_backend/routers/environment_router.py | 1 + agenta-backend/agenta_backend/services/db_manager.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 98e305d4c0..a21efaa581 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -51,6 +51,7 @@ async def deploy_to_environment( await db_manager.deploy_to_environment( environment_name=payload.environment_name, variant_id=payload.variant_id, + user_uid=request.state.user_id, ) except Exception as e: logger.exception(f"An error occurred: {str(e)}") diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index d451447610..7ede18e674 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -1050,7 +1050,7 @@ async def deploy_to_environment( environment_db.deployment = deployment.id # Create revision for app environment - user = await get_user(user_uid=user_org_data["uid"]) + user = await get_user(user_uid=user_org_data["user_uid"]) await create_environment_revision( environment_db, user, From 0b4f8b889ec5fb113e679c3d3293c222f349898d Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 21:17:55 +0100 Subject: [PATCH 78/99] Refactor - modified construct_app_container_url endpoint --- .../agenta_backend/routers/container_router.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 8597b52cc4..6cd839ad9b 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -155,7 +155,7 @@ async def construct_app_container_url( if base_id: object_db = await db_manager.fetch_base_by_id(base_id) - else: + elif variant_id: object_db = await db_manager.fetch_app_variant_by_id(variant_id) # Check app access @@ -171,8 +171,17 @@ async def construct_app_container_url( raise HTTPException(status_code=403, detail=error_msg) try: - deployment = await db_manager.get_deployment_by_objectid(object_db.deployment) - assert deployment and deployment.uri, "Deployment not found" + if getattr(object_db, "deployment", None): # this is a base + deployment = await db_manager.get_deployment_by_objectid(object_db.deployment) + elif getattr(object_db.base, "deployment", None): # this is a variant + deployment = await db_manager.get_deployment_by_objectid( + object_db.base.deployment + ) + else: + raise HTTPException( + status_code=400, + detail="Deployment not found", + ) return URI(uri=deployment.uri) except Exception as e: return JSONResponse({"message": str(e)}, status_code=500) From 281dccca350c8eca90b9e3729e732aecfc0a85d2 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 23:31:30 +0100 Subject: [PATCH 79/99] Update - resolve failing endpoints in human evaluation router --- .../routers/human_evaluation_router.py | 71 +++++++++---------- .../services/evaluation_service.py | 24 ------- 2 files changed, 34 insertions(+), 61 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index d8bca92e97..ccb37c5eb8 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -63,7 +63,7 @@ async def create_evaluation( permission=Permission.CREATE_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -104,7 +104,7 @@ async def fetch_list_human_evaluations( permission=Permission.VIEW_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -141,7 +141,7 @@ async def fetch_human_evaluation( permission=Permission.VIEW_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -174,10 +174,15 @@ async def fetch_evaluation_scenarios( """ try: - if isCloudEE(): - human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id + human_evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation_id + ) + if human_evaluation is None: + raise HTTPException( + status_code=404, + detail=f"Evaluation with id {evaluation_id} not found", ) + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object_id=evaluation_id, @@ -185,7 +190,7 @@ async def fetch_evaluation_scenarios( permission=Permission.VIEW_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -217,20 +222,20 @@ async def update_human_evaluation( None: A 204 No Content status code, indicating that the update was successful. """ try: - if isCloudEE(): - human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id - ) - if not human_evaluation: - raise HTTPException(status_code=404, detail="Evaluation not found") + human_evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation_id + ) + if not human_evaluation: + raise HTTPException(status_code=404, detail="Evaluation not found") + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, object=human_evaluation, permission=Permission.EDIT_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -265,20 +270,14 @@ async def update_evaluation_scenario_router( None: 204 No Content status code upon successful update. """ try: - evaluation_scenario_db = db_manager.fetch_human_evaluation_scenario_by_id( + evaluation_scenario_db = await db_manager.fetch_human_evaluation_scenario_by_id( evaluation_scenario_id ) - if evaluation_scenario is None: + if evaluation_scenario_db is None: raise HTTPException( status_code=404, detail=f"Evaluation scenario with id {evaluation_scenario_id} not found", ) - evaluation = evaluation_scenario.evaluation - if evaluation is None: - raise HTTPException( - status_code=404, - detail=f"Evaluation scenario for evaluation scenario with id {evaluation_scenario_id} not found", - ) if isCloudEE(): has_permission = await check_action_access( @@ -287,7 +286,7 @@ async def update_evaluation_scenario_router( permission=Permission.EDIT_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -300,6 +299,8 @@ async def update_evaluation_scenario_router( ) return Response(status_code=status.HTTP_204_NO_CONTENT) except UpdateEvaluationScenarioError as e: + import traceback + traceback.print_exc() raise HTTPException(status_code=500, detail=str(e)) from e @@ -335,7 +336,7 @@ async def get_evaluation_scenario_score_router( permission=Permission.VIEW_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -380,7 +381,7 @@ async def update_evaluation_scenario_score_router( permission=Permission.VIEW_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -409,24 +410,20 @@ async def fetch_results( """ try: + evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + if evaluation is None: + raise HTTPException( + status_code=404, + detail=f"Evaluation with id {evaluation_id} not found", + ) if isCloudEE(): - evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) - if evaluation is None: - raise HTTPException( - status_code=404, - detail=f"Evaluation with id {evaluation_id} not found", - ) - - if not evaluation: - raise HTTPException(status_code=404, detail="Evaluation not found") - has_permission = await check_action_access( user_uid=request.state.user_id, object=evaluation, permission=Permission.VIEW_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, @@ -470,7 +467,7 @@ async def delete_evaluations( permission=Permission.DELETE_EVALUATION, ) if not has_permission: - error_msg = f"You do not have permissiom to perform this action. Please contact your Organization Admin." + error_msg = f"You do not have permission to perform this action. Please contact your Organization Admin." return JSONResponse( {"detail": error_msg}, status_code=403, diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 3df92709fc..b1b85fe4a2 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -75,30 +75,6 @@ async def _fetch_human_evaluation(evaluation_id: str) -> HumanEvaluationDB: return evaluation -async def _fetch_human_evaluation_scenario( - evaluation_scenario_id: str, -) -> HumanEvaluationDB: - # Fetch the evaluation by ID - evaluation_scenario = await db_manager.fetch_human_evaluation_scenario_by_id( - evaluation_scenario_id=evaluation_scenario_id - ) - if evaluation_scenario is None: - raise HTTPException( - status_code=404, - detail=f"Evaluation scenario with id {evaluation_scenario_id} not found", - ) - evaluation = evaluation_scenario.evaluation - - # Check if the evaluation exists - if evaluation is None: - raise HTTPException( - status_code=404, - detail=f"Evaluation scenario for evaluation scenario with id {evaluation_scenario_id} not found", - ) - - return evaluation_scenario - - async def prepare_csvdata_and_create_evaluation_scenario( csvdata: List[Dict[str, str]], payload_inputs: List[str], From 79f62b2adaa31961c0564ddc3bcb3bbe594567ce Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 23:33:05 +0100 Subject: [PATCH 80/99] :art: Format - ran black --- agenta-backend/agenta_backend/models/converters.py | 4 +++- .../agenta_backend/routers/container_router.py | 8 +++++--- .../agenta_backend/routers/human_evaluation_router.py | 9 +++------ agenta-backend/agenta_backend/services/db_manager.py | 4 +++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index ac57490464..110b8d74bb 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -421,7 +421,9 @@ async def environment_db_and_revision_to_extended_output( ) if isCloudEE(): - environment_output_extended.organization_id = str(environment_db.organization.id) + environment_output_extended.organization_id = str( + environment_db.organization.id + ) environment_output_extended.workspace_id = str(environment_db.workspace.id) return environment_output_extended diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 6cd839ad9b..1e0b5bc3d1 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -171,9 +171,11 @@ async def construct_app_container_url( raise HTTPException(status_code=403, detail=error_msg) try: - if getattr(object_db, "deployment", None): # this is a base - deployment = await db_manager.get_deployment_by_objectid(object_db.deployment) - elif getattr(object_db.base, "deployment", None): # this is a variant + if getattr(object_db, "deployment", None): # this is a base + deployment = await db_manager.get_deployment_by_objectid( + object_db.deployment + ) + elif getattr(object_db.base, "deployment", None): # this is a variant deployment = await db_manager.get_deployment_by_objectid( object_db.base.deployment ) diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index ccb37c5eb8..6cfadb4dc2 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -174,9 +174,7 @@ async def fetch_evaluation_scenarios( """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id - ) + human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) if human_evaluation is None: raise HTTPException( status_code=404, @@ -222,9 +220,7 @@ async def update_human_evaluation( None: A 204 No Content status code, indicating that the update was successful. """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id - ) + human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") @@ -300,6 +296,7 @@ async def update_evaluation_scenario_router( return Response(status_code=status.HTTP_204_NO_CONTENT) except UpdateEvaluationScenarioError as e: import traceback + traceback.print_exc() raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 7ede18e674..5bf7b86e18 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -1214,7 +1214,9 @@ async def create_environment(name: str, app_db: AppDB) -> AppEnvironmentDB: Returns: AppEnvironmentDB: The newly created AppEnvironmentDB object. """ - environment_db = AppEnvironmentDB(app=app_db, name=name, user=app_db.user, revision=0) + environment_db = AppEnvironmentDB( + app=app_db, name=name, user=app_db.user, revision=0 + ) if isCloudEE(): environment_db.organization = app_db.organization From a086de432649728951f8e79b1cf55a40353e78b9 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 23:35:48 +0100 Subject: [PATCH 81/99] :art: Format - ran prettier --- agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx | 1 - .../src/pages/apps/[app_id]/testsets/new/upload/index.tsx | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx b/agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx index 4d8f425a5a..f99a2963a4 100644 --- a/agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx +++ b/agenta-web/src/pages/apps/[app_id]/endpoints/index.tsx @@ -17,7 +17,6 @@ import {createUseStyles} from "react-jss" const DeploymentHistory: any = dynamicComponent("DeploymentHistory/DeploymentHistory") - const {Text, Title} = Typography const useStyles = createUseStyles({ diff --git a/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx b/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx index 139c45ffdc..29f7c47337 100644 --- a/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx +++ b/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx @@ -70,8 +70,8 @@ export default function AddANewTestset() { router.push(`/apps/${appId}/testsets`) } catch (e: any) { if ( - e?.response?.data?.detail?.find((item: GenericObject) => - item?.loc?.includes("csvdata"), + e?.response?.data?.detail?.find( + (item: GenericObject) => item?.loc?.includes("csvdata"), ) ) message.error(malformedFileError) From 6fb5b782f7fe562328c5a0a0457f98c6f5e90324 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 23:52:48 +0100 Subject: [PATCH 82/99] Update - remove redundant testcases for deployment versioning in oss --- .../test_variant_versioning_deployment.py | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_versioning_deployment.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_versioning_deployment.py index ae1375cbe7..405801405c 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_versioning_deployment.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_versioning_deployment.py @@ -65,55 +65,3 @@ async def test_deploy_to_environment(deploy_to_environment_payload): assert ( list_of_response_status_codes.count(200) == 3 ), "The list does not contain 3 occurrences of 200 status code" - - -@pytest.mark.asyncio -async def test_list_app_environment_revisions(): - app = await AppDB.find_one(AppDB.app_name == APP_NAME) - list_of_response_data = [] - list_of_response_status_codes = [] - for environment in VARIANT_DEPLOY_ENVIRONMENTS: - response = await test_client.get( - f"{BACKEND_API_HOST}/apps/{str(app.id)}/revisions/{environment}" - ) - list_of_response_data.append(response.json()) - list_of_response_status_codes.append(response.status_code) - assert ( - list_of_response_status_codes.count(200) == 3 - ), "The list does not container 3 occurrences of 200 status code" - assert len(list_of_response_data) == 3, "The list does not contain 3 response data" - - -@pytest.mark.asyncio -async def test_get_config_deployment_revision(): - app = await AppDB.find_one(AppDB.app_name == APP_NAME) - app_environment_revisions_response = await test_client.get( - f"{BACKEND_API_HOST}/apps/{str(app.id)}/revisions/{VARIANT_DEPLOY_ENVIRONMENTS[0]}" - ) - - if app_environment_revisions_response.status_code == 200: - revisions = app_environment_revisions_response.json()["revisions"] - config_deployment_revision_response = await test_client.get( - f"{BACKEND_API_HOST}/configs/deployment/{revisions[0]['id']}" - ) - assert config_deployment_revision_response.status_code == 200 - assert config_deployment_revision_response.json() is not None - else: - assert False, "App environment revisions response is not 200" - - -@pytest.mark.asyncio -async def test_revert_deployment_revision(): - app = await AppDB.find_one(AppDB.app_name == APP_NAME) - app_environment_revisions_response = await test_client.get( - f"{BACKEND_API_HOST}/apps/{str(app.id)}/revisions/{VARIANT_DEPLOY_ENVIRONMENTS[0]}" - ) - - if app_environment_revisions_response.status_code == 200: - revisions = app_environment_revisions_response.json()["revisions"] - revert_deployment_revision_response = await test_client.post( - f"{BACKEND_API_HOST}/configs/deployment/{revisions[0]['id']}/revert/" - ) - assert revert_deployment_revision_response.status_code == 200 - else: - assert False, "App environment revisions response is not 200" From 488f62cd3d1c2afa8bca96cbe7361ae3e9671975 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 6 Feb 2024 23:53:04 +0100 Subject: [PATCH 83/99] Update - change database mode to test --- docker-compose.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 65058140fb..8be8cfcd99 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -22,7 +22,7 @@ services: - DOMAIN_NAME=http://localhost - CELERY_BROKER_URL=amqp://guest@rabbitmq// - CELERY_RESULT_BACKEND=redis://redis:6379/0 - - DATABASE_MODE=v2 + - DATABASE_MODE=test - FEATURE_FLAG=oss - OPENAI_API_KEY=${OPENAI_API_KEY} - AGENTA_TEMPLATE_REPO=agentaai/templates_v2 From 41bd10a31f9a8b1159db47dbfc94842ca04bf4cc Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 7 Feb 2024 03:35:08 +0100 Subject: [PATCH 84/99] Update - added environment output to app router and converters base on if-else condition --- agenta-backend/agenta_backend/models/converters.py | 3 ++- agenta-backend/agenta_backend/routers/app_router.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 110b8d74bb..2eed354e0a 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -46,6 +46,7 @@ AppVariantResponse_ as AppVariantResponse, AppVariantOutputExtended_ as AppVariantOutputExtended, EnvironmentRevision_ as EnvironmentRevision, + EnvironmentOutput_ as EnvironmentOutput, EnvironmentOutputExtended_ as EnvironmentOutputExtended, ) else: @@ -70,6 +71,7 @@ AppVariantResponse, AppVariantOutputExtended, EnvironmentRevision, + EnvironmentOutput, EnvironmentOutputExtended, ) @@ -88,7 +90,6 @@ BaseOutput, TestSetOutput, TemplateImageInfo, - EnvironmentOutput, AppVariantRevision, ) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 300c235660..9bfb97f3bc 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -23,12 +23,8 @@ ) from agenta_backend.models.api.api_models import ( App, - Image, - CreateApp, CreateAppOutput, - EnvironmentOutput, AddVariantFromImagePayload, - EnvironmentOutputExtended, ) if isCloudEE(): @@ -37,6 +33,8 @@ CreateApp_ as CreateApp, AppVariantResponse_ as AppVariantResponse, CreateAppVariant_ as CreateAppVariant, + EnvironmentOutput_ as EnvironmentOutput, + EnvironmentOutputExtended_ as EnvironmentOutputExtended, ) else: from agenta_backend.models.api.api_models import ( @@ -44,6 +42,8 @@ CreateApp, AppVariantResponse, CreateAppVariant, + EnvironmentOutput, + EnvironmentOutputExtended, ) if isCloudEE(): from agenta_backend.commons.services import db_manager_ee From 6782e309ab20a8bb8e194b2f0e3a1738e7dc6671 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 8 Feb 2024 17:10:54 +0100 Subject: [PATCH 85/99] Update - allow workspace members with DELETE_EVALUATION permission to delete evaluations --- agenta-backend/agenta_backend/routers/evaluation_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 8983442636..94886d5001 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -329,7 +329,7 @@ async def delete_evaluations( user_uid=request.state.user_id, object_id=evaluation_id, object_type="evaluation", - permission=Permission.VIEW_EVALUATION, + permission=Permission.DELETE_EVALUATION, ) logger.debug( f"User has permission to delete evaluation: {has_permission}" From 10fb341e1b388cadb7ca083845cfa1e0ff44324e Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 9 Feb 2024 14:36:42 +0100 Subject: [PATCH 86/99] Update - added permission check to ensure only users with the right role can revert a deployment revision --- .../agenta_backend/routers/configs_router.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index bf35d7f5a9..acf86f119f 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -204,6 +204,20 @@ async def revert_deployment_revision(request: Request, deployment_revision_id: s f"No environment revision found for deployment revision {deployment_revision_id}", ) + if isCloudEE(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + object=environment_revision, + permission=Permission.EDIT_APP_ENVIRONMENT_DEPLOYMENT, + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + if environment_revision.deployed_app_variant_revision is None: raise HTTPException( 404, From 02194683c9f1d942947fd7b376bf8250cb0ba271 Mon Sep 17 00:00:00 2001 From: Abram Date: Sun, 11 Feb 2024 12:13:50 +0100 Subject: [PATCH 87/99] Update - added logic to check apikey action access for rbac --- agenta-backend/agenta_backend/routers/app_router.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 9bfb97f3bc..72bd263d7c 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -55,6 +55,7 @@ from agenta_backend.commons.utils.permissions import ( check_action_access, check_rbac_permission, + check_apikey_action_access, ) from agenta_backend.commons.models.db_models import Permission @@ -202,6 +203,13 @@ async def create_app( """ try: if isCloudEE(): + api_key_from_headers = request.headers.get("Authorization") + if api_key_from_headers is not None: + await check_apikey_action_access( + api_key_from_headers, + request.state.user_id, + Permission.CREATE_APPLICATION, + ) try: user_org_workspace_data = await get_user_org_and_workspace_id( request.state.user_id From 73825801f642b48111d22ccf01bd51f3e01859ab Mon Sep 17 00:00:00 2001 From: Abram Date: Sun, 11 Feb 2024 13:49:23 +0100 Subject: [PATCH 88/99] Update - fix unbound error --- agenta-backend/agenta_backend/routers/app_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 72bd263d7c..268865d7d4 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -249,7 +249,7 @@ async def create_app( has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, - workspace_id=ObjectId(workspace_id), + workspace_id=workspace.id, organization=organization, permission=Permission.CREATE_APPLICATION, ) @@ -269,7 +269,7 @@ async def create_app( payload.app_name, request.state.user_id, organization_id if isCloudEE() else None, - workspace_id if isCloudEE() else None, + workspace.id if isCloudEE() else None, ) return CreateAppOutput(app_id=str(app_db.id), app_name=str(app_db.app_name)) except Exception as e: From b0f06a6d897e404612a248290de6010c36de77a5 Mon Sep 17 00:00:00 2001 From: Abram Date: Sun, 11 Feb 2024 14:01:11 +0100 Subject: [PATCH 89/99] Update - added logic to allow organization selection for cloud --- agenta-cli/agenta/cli/main.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/agenta-cli/agenta/cli/main.py b/agenta-cli/agenta/cli/main.py index d2ee1fef00..315a4e5295 100644 --- a/agenta-cli/agenta/cli/main.py +++ b/agenta-cli/agenta/cli/main.py @@ -141,11 +141,19 @@ def init(app_name: str, backend_host: str): api_key=api_key if where_question == "On agenta cloud" else "", ) + # list of user organizations + user_organizations = [] + # validate the api key if it is provided if where_question == "On agenta cloud": try: key_prefix = api_key.split(".")[0] client.validate_api_key(key_prefix=key_prefix) + + # Make request to fetch user organizations after api key validation + organizations = client.list_organizations() + if len(organizations) >= 1: + user_organizations = organizations except Exception as ex: if ex.status_code == 401: click.echo(click.style("Error: Invalid API key", fg="red")) @@ -154,9 +162,28 @@ def init(app_name: str, backend_host: str): click.echo(click.style(f"Error: {ex}", fg="red")) sys.exit(1) + if where_question == "On agenta cloud": + which_organization = questionary.select( + "Which organization do you want to create the app for?", + choices=[ + f"{org.name}: {org.description}" for org in user_organizations + ], + ).ask() + filtered_org = next( + ( + org + for org in user_organizations + if org.name == which_organization.split(":")[0] + ), + None, + ) + # Get app_id after creating new app in the backend server try: - app_id = client.create_app(app_name=app_name).app_id + app_id = client.create_app( + app_name=app_name, + organization_id=filtered_org.id if filtered_org else None, + ).app_id except Exception as ex: click.echo(click.style(f"Error: {ex}", fg="red")) sys.exit(1) From 886be346035f9fe79dc6eada9d351a085b1098fb Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Mon, 12 Feb 2024 12:32:41 +0500 Subject: [PATCH 90/99] fixed double loading issue on apps page | fixed missing workspace_id while ccreating api key --- agenta-backend/agenta_backend/services/app_manager.py | 5 ++++- agenta-web/src/contexts/app.context.tsx | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 11a50b077b..04c7d747bb 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -94,7 +94,10 @@ async def start_variant( ) if isCloudEE(): api_key = await api_key_service.create_api_key( - str(db_app_variant.user.uid), expiration_date=None, hidden=True + str(db_app_variant.user.uid), + workspace_id=str(db_app_variant.workspace), + expiration_date=None, + hidden=True, ) env_vars.update({"AGENTA_API_KEY": api_key}) deployment = await deployment_manager.start_service( diff --git a/agenta-web/src/contexts/app.context.tsx b/agenta-web/src/contexts/app.context.tsx index acf44a0215..f45be363d2 100644 --- a/agenta-web/src/contexts/app.context.tsx +++ b/agenta-web/src/contexts/app.context.tsx @@ -31,7 +31,7 @@ const useApps = () => { }) }, []) - const {selectedOrg} = useOrgData() + const {selectedOrg, loading} = useOrgData() const {data, error, isLoading, mutate} = useSWR( `${getAgentaApiUrl()}/api/apps/` + (isDemo() @@ -42,7 +42,7 @@ const useApps = () => { return { data: (data || []) as ListAppsItem[], error, - isLoading, + isLoading: isLoading || loading, mutate, } } From 754168f5e7865097ab40bd8d427363a7f5d39e27 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 12 Feb 2024 10:19:01 +0100 Subject: [PATCH 91/99] Refactor - update list_apps router --- .../agenta_backend/routers/app_router.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 268865d7d4..e1fd3322f7 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -299,6 +299,25 @@ async def list_apps( HTTPException: If there was an error retrieving the list of apps. """ try: + if isCloudEE(): + user_org_workspace_data = await get_user_org_and_workspace_id( + request.state.user_id + ) + has_permission = await check_rbac_permission( + user_org_workspace_data=user_org_workspace_data, + workspace_id=workspace_id, + organization_id=org_id, + permission=Permission.VIEW_APPLICATION, + ) + logger.debug( + f"User has Permission to Create Application: {has_permission}" + ) + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) apps = await db_manager.list_apps( app_name=app_name, user_uid=request.state.user_id, From e9dedcab5124ff768cca6babc7650f7c37b98871 Mon Sep 17 00:00:00 2001 From: MohammedMaaz Date: Mon, 12 Feb 2024 17:09:51 +0500 Subject: [PATCH 92/99] fixes | single role selection --- .../components/Evaluations/Evaluations.tsx | 13 ++++++++----- agenta-web/src/contexts/app.context.tsx | 3 +++ agenta-web/src/lib/helpers/axiosConfig.ts | 15 +++++++++++++++ agenta-web/src/lib/helpers/errorHandler.ts | 12 ------------ agenta-web/src/lib/hooks/useVariant.ts | 19 ++++++++++++------- agenta-web/src/lib/services/api.ts | 2 +- 6 files changed, 39 insertions(+), 25 deletions(-) diff --git a/agenta-web/src/components/Evaluations/Evaluations.tsx b/agenta-web/src/components/Evaluations/Evaluations.tsx index a009e946ea..29230f21ec 100644 --- a/agenta-web/src/components/Evaluations/Evaluations.tsx +++ b/agenta-web/src/components/Evaluations/Evaluations.tsx @@ -30,6 +30,7 @@ import HumanEvaluationResult from "./HumanEvaluationResult" import {getErrorMessage} from "@/lib/helpers/errorHandler" import AutomaticEvaluationResult from "./AutomaticEvaluationResult" import {dynamicComponent} from "@/lib/helpers/dynamic" +import {PERMISSION_ERR_MSG} from "@/lib/helpers/axiosConfig" type StyleProps = { themeMode: "dark" | "light" @@ -355,11 +356,13 @@ export default function Evaluations() { selectedCustomEvaluationID, testsetId: selectedTestset._id!, }).catch((err) => { - setError({ - message: getErrorMessage(err), - btnText: "Go to Test sets", - endpoint: `/apps/${appId}/testsets`, - }) + if (err.message !== PERMISSION_ERR_MSG) { + setError({ + message: getErrorMessage(err), + btnText: "Go to Test sets", + endpoint: `/apps/${appId}/testsets`, + }) + } }) if (!evaluationTableId) { diff --git a/agenta-web/src/contexts/app.context.tsx b/agenta-web/src/contexts/app.context.tsx index f45be363d2..84c84df25a 100644 --- a/agenta-web/src/contexts/app.context.tsx +++ b/agenta-web/src/contexts/app.context.tsx @@ -38,6 +38,9 @@ const useApps = () => { ? `?org_id=${selectedOrg?.id}&workspace_id=${selectedOrg?.default_workspace.id}` : ""), isDemo() ? (selectedOrg?.id ? axiosFetcher : () => {}) : axiosFetcher, + { + shouldRetryOnError: false, + }, ) return { data: (data || []) as ListAppsItem[], diff --git a/agenta-web/src/lib/helpers/axiosConfig.ts b/agenta-web/src/lib/helpers/axiosConfig.ts index cbdb76ef95..13a5844d2b 100644 --- a/agenta-web/src/lib/helpers/axiosConfig.ts +++ b/agenta-web/src/lib/helpers/axiosConfig.ts @@ -4,6 +4,10 @@ import {signOut} from "supertokens-auth-react/recipe/thirdpartypasswordless" import router from "next/router" import {getAgentaApiUrl} from "./utils" import {isObject} from "lodash" +import AlertPopup from "@/components/AlertPopup/AlertPopup" + +export const PERMISSION_ERR_MSG = + "You don't have permission to perform this action. Please contact your organization admin." const axios = axiosApi.create({ baseURL: getAgentaApiUrl(), @@ -27,6 +31,17 @@ axios.interceptors.response.use( return response }, (error) => { + if (error.response?.status === 403 && error.config.method !== "get") { + AlertPopup({ + title: "Permission Denied", + message: PERMISSION_ERR_MSG, + cancelText: null, + okText: "Ok", + }) + error.message = PERMISSION_ERR_MSG + throw error + } + // if axios config has _ignoreError set to true, then don't handle error if (error.config?._ignoreError) throw error diff --git a/agenta-web/src/lib/helpers/errorHandler.ts b/agenta-web/src/lib/helpers/errorHandler.ts index f2271205ee..8389005f04 100644 --- a/agenta-web/src/lib/helpers/errorHandler.ts +++ b/agenta-web/src/lib/helpers/errorHandler.ts @@ -1,4 +1,3 @@ -import AlertPopup from "@/components/AlertPopup/AlertPopup" import {message} from "antd" export const getErrorMessage = (error: any, fallback = "An unknown error occurred!") => { @@ -16,17 +15,6 @@ export const getErrorMessage = (error: any, fallback = "An unknown error occurre } export const globalErrorHandler = (error: any) => { - if (error.response?.status === 403) { - AlertPopup({ - title: "Permission Denied", - message: - "You don't have permission to perform this action. Please contact your organization admin.", - cancelText: null, - okText: "Ok", - }) - return - } - const errorMsg = getErrorMessage(error) console.error(errorMsg, error) message.error(errorMsg) diff --git a/agenta-web/src/lib/hooks/useVariant.ts b/agenta-web/src/lib/hooks/useVariant.ts index abc659c4a4..1262c656ba 100644 --- a/agenta-web/src/lib/hooks/useVariant.ts +++ b/agenta-web/src/lib/hooks/useVariant.ts @@ -1,8 +1,9 @@ -import {useState, useEffect, useContext} from "react" +import {useState, useEffect} from "react" import {promptVersioning, saveNewVariant, updateVariantParams} from "@/lib/services/api" import {Variant, Parameter, IPromptVersioning} from "@/lib/Types" import {getAllVariantParameters, updateInputParams} from "@/lib/helpers/variantHelper" import {isDemo} from "../helpers/utils" +import {PERMISSION_ERR_MSG} from "../helpers/axiosConfig" /** * Hook for using the variant. @@ -42,10 +43,12 @@ export function useVariant(appId: string, variant: Variant) { setIsChatVariant(isChatVariant) setHistoryStatus({loading: false, error: true}) } catch (error: any) { - console.log(error) - setIsError(true) - setError(error) - setHistoryStatus({loading: false, error: true}) + if (error.message !== PERMISSION_ERR_MSG) { + console.log(error) + setIsError(true) + setError(error) + setHistoryStatus({loading: false, error: true}) + } } finally { setIsLoading(false) setHistoryStatus({loading: false, error: false}) @@ -92,8 +95,10 @@ export function useVariant(appId: string, variant: Variant) { }, {}) } setPromptOptParams(updatedOptParams) - } catch (error) { - setIsError(true) + } catch (error: any) { + if (error.message !== PERMISSION_ERR_MSG) { + setIsError(true) + } } finally { setIsParamSaveLoading(false) } diff --git a/agenta-web/src/lib/services/api.ts b/agenta-web/src/lib/services/api.ts index 22af8eaf43..7469822218 100644 --- a/agenta-web/src/lib/services/api.ts +++ b/agenta-web/src/lib/services/api.ts @@ -251,7 +251,7 @@ export const useLoadTestsetsList = (appId: string) => { const {data, error, mutate, isLoading} = useSWR( `${getAgentaApiUrl()}/api/testsets/?app_id=${appId}`, axiosFetcher, - {revalidateOnFocus: false}, + {revalidateOnFocus: false, shouldRetryOnError: false}, ) return { From df19dc72995e5f18b54bf111891394e000bc71cb Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 12 Feb 2024 18:29:10 +0100 Subject: [PATCH 93/99] Update - change permission from view_application to read_system --- agenta-backend/agenta_backend/routers/app_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index e1fd3322f7..c51ae6f0e0 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -307,7 +307,7 @@ async def list_apps( user_org_workspace_data=user_org_workspace_data, workspace_id=workspace_id, organization_id=org_id, - permission=Permission.VIEW_APPLICATION, + permission=Permission.READ_SYSTEM, ) logger.debug( f"User has Permission to Create Application: {has_permission}" From 25e10f6de5d75d27aa7c795935b32d4cbae62db2 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 12 Feb 2024 19:28:39 +0100 Subject: [PATCH 94/99] Refactor - remove redundant rbac check --- .../agenta_backend/routers/app_router.py | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index c51ae6f0e0..730cba1f15 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -299,28 +299,9 @@ async def list_apps( HTTPException: If there was an error retrieving the list of apps. """ try: - if isCloudEE(): - user_org_workspace_data = await get_user_org_and_workspace_id( - request.state.user_id - ) - has_permission = await check_rbac_permission( - user_org_workspace_data=user_org_workspace_data, - workspace_id=workspace_id, - organization_id=org_id, - permission=Permission.READ_SYSTEM, - ) - logger.debug( - f"User has Permission to Create Application: {has_permission}" - ) - if not has_permission: - error_msg = f"You do not have access to perform this action. Please contact your organization admin." - return JSONResponse( - {"detail": error_msg}, - status_code=403, - ) apps = await db_manager.list_apps( - app_name=app_name, user_uid=request.state.user_id, + app_name=app_name, org_id=org_id, workspace_id=workspace_id, ) From efc22ff8318187026250ded015dc650e09951c05 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 12 Feb 2024 20:12:56 +0100 Subject: [PATCH 95/99] Update - make use of right status code (403) --- agenta-backend/agenta_backend/services/db_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 5bf7b86e18..d8dfbb77a2 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -910,7 +910,7 @@ async def list_apps( error_msg = f"You do not have access to perform this action. Please contact your organization admin." return JSONResponse( {"detail": error_msg}, - status_code=400, + status_code=403, ) apps: List[AppDB] = await AppDB.find( From 5331ddcb133c68744019fe65afb257bde5479084 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 12 Feb 2024 21:34:30 +0100 Subject: [PATCH 96/99] :art: Format - ran black --- agenta-backend/agenta_backend/services/llm_apps_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index 964d1bff35..e9c597e0a3 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -178,9 +178,9 @@ async def batch_invoke( "delay_between_batches" ] # Delay between batches (in seconds) - list_of_app_outputs: List[InvokationResult] = ( - [] - ) # Outputs after running all batches + list_of_app_outputs: List[ + InvokationResult + ] = [] # Outputs after running all batches openapi_parameters = await get_parameters_from_openapi(uri + "/openapi.json") async def run_batch(start_idx: int): From b7eabcbd538b3183270a35276e6e6843223610a3 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 12 Feb 2024 21:45:24 +0100 Subject: [PATCH 97/99] Refactor - remove redundant selectedOrg --- agenta-web/src/components/AppSelector/AppSelector.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/agenta-web/src/components/AppSelector/AppSelector.tsx b/agenta-web/src/components/AppSelector/AppSelector.tsx index 94c18dd551..ef158df89b 100644 --- a/agenta-web/src/components/AppSelector/AppSelector.tsx +++ b/agenta-web/src/components/AppSelector/AppSelector.tsx @@ -286,7 +286,6 @@ const AppSelector: React.FC = () => { onClick={() => { if ( isDemo() && - selectedOrg?.is_paying == false && apps.length > 2 ) { showMaxAppError() From 3fbcbf0f5da48d165b26d8b1e5d8ffda43d9bcec Mon Sep 17 00:00:00 2001 From: Akrem Abayed Date: Tue, 13 Feb 2024 07:26:07 +0100 Subject: [PATCH 98/99] hardcode black version --- .github/workflows/check-python-code-black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-python-code-black.yml b/.github/workflows/check-python-code-black.yml index 8179105f17..4ae9298e2b 100644 --- a/.github/workflows/check-python-code-black.yml +++ b/.github/workflows/check-python-code-black.yml @@ -12,4 +12,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: psf/black@stable + - uses: psf/black@23.12.0 From 494769bf424318ca4974beaf434e680e4f94d7e1 Mon Sep 17 00:00:00 2001 From: Akrem Abayed Date: Tue, 13 Feb 2024 07:28:51 +0100 Subject: [PATCH 99/99] format web --- agenta-web/src/components/AppSelector/AppSelector.tsx | 5 +---- .../src/pages/apps/[app_id]/testsets/new/upload/index.tsx | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/agenta-web/src/components/AppSelector/AppSelector.tsx b/agenta-web/src/components/AppSelector/AppSelector.tsx index ef158df89b..3c35672c45 100644 --- a/agenta-web/src/components/AppSelector/AppSelector.tsx +++ b/agenta-web/src/components/AppSelector/AppSelector.tsx @@ -284,10 +284,7 @@ const AppSelector: React.FC = () => { { - if ( - isDemo() && - apps.length > 2 - ) { + if (isDemo() && apps.length > 2) { showMaxAppError() } else { showCreateAppModal() diff --git a/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx b/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx index 29f7c47337..139c45ffdc 100644 --- a/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx +++ b/agenta-web/src/pages/apps/[app_id]/testsets/new/upload/index.tsx @@ -70,8 +70,8 @@ export default function AddANewTestset() { router.push(`/apps/${appId}/testsets`) } catch (e: any) { if ( - e?.response?.data?.detail?.find( - (item: GenericObject) => item?.loc?.includes("csvdata"), + e?.response?.data?.detail?.find((item: GenericObject) => + item?.loc?.includes("csvdata"), ) ) message.error(malformedFileError)