From 4800b5b12d6371c44d599f54d4228dc495ab1681 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 20 Dec 2024 18:22:07 +0100 Subject: [PATCH] initial setup, no CLI/URL --- .../agenta_backend/models/api/api_models.py | 7 + .../agenta_backend/routers/app_router.py | 88 +++++- .../agenta_backend/routers/variants_router.py | 94 +++++- .../agenta_backend/services/app_manager.py | 276 ++++++++++++++---- .../agenta_backend/services/db_manager.py | 13 +- 5 files changed, 408 insertions(+), 70 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index bbf13b41d6..610bf4116c 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -198,6 +198,13 @@ class AddVariantFromImagePayload(BaseModel): config_name: Optional[str] +class AddVariantFromURLPayload(BaseModel): + variant_name: str + url: str + base_name: Optional[str] + config_name: Optional[str] + + class ImageExtended(Image): # includes the mongodb image id id: str diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index d4a54f16b5..c379323c7b 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -27,6 +27,7 @@ UpdateAppOutput, CreateAppOutput, AddVariantFromImagePayload, + AddVariantFromURLPayload, ) if isCloudEE(): @@ -342,7 +343,6 @@ async def add_variant_from_image( Args: app_id (str): The ID of the app to add the variant to. payload (AddVariantFromImagePayload): The payload containing information about the variant to add. - stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()). Raises: HTTPException: If the feature flag is set to "demo" or if the image does not have a tag starting with the registry name (agenta-server) or if the image is not found or if the user does not have access to the app. @@ -367,6 +367,7 @@ async def add_variant_from_image( try: app = await db_manager.fetch_app_by_id(app_id) + if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -394,14 +395,91 @@ async def add_variant_from_image( is_template_image=False, user_uid=request.state.user_id, ) - app_variant_db = await db_manager.fetch_app_variant_by_id(str(variant_db.id)) - logger.debug("Step 8: We create ready-to use evaluators") + app_variant_db = await db_manager.fetch_app_variant_by_id( + str(variant_db.id), + ) + await evaluator_manager.create_ready_to_use_evaluators( - app_name=app.app_name, project_id=str(app.project_id) + app_name=app.app_name, + project_id=str(app.project_id), ) - return await converters.app_variant_db_to_output(app_variant_db) + app_variant_dto = await converters.app_variant_db_to_output( + app_variant_db, + ) + + return app_variant_dto + + except Exception as e: + logger.exception(f"An error occurred: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/{app_id}/variant/from-url/", operation_id="add_variant_from_url") +async def add_variant_from_url( + app_id: str, + payload: AddVariantFromURLPayload, + request: Request, +): + """ + Add a new variant to an app based on a URL. + + Args: + app_id (str): The ID of the app to add the variant to. + payload (AddVariantFromURLPayload): The payload containing information about the variant to add. + + Raises: + HTTPException: If the user does not have access to the app or if there is an error adding the variant. + + Returns: + dict: The newly added variant. + """ + + try: + app = await db_manager.fetch_app_by_id(app_id) + + if isCloudEE(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + object=app, + permission=Permission.CREATE_APPLICATION, + ) + logger.debug( + f"User has Permission to create app from url: {has_permission}" + ) + if not has_permission: + error_msg = f"You do not have access to perform this action. Please contact your organization admin." + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + variant_db = await app_manager.add_variant_based_on_url( + app=app, + project_id=str(app.project_id), + variant_name=payload.variant_name, + url=payload.url, + base_name=payload.base_name, + config_name=payload.config_name, + user_uid=request.state.user_id, + ) + + app_variant_db = await db_manager.fetch_app_variant_by_id( + str(variant_db.id), + ) + + await evaluator_manager.create_ready_to_use_evaluators( + app_name=app.app_name, + project_id=str(app.project_id), + ) + + app_variant_dto = await converters.app_variant_db_to_output( + app_variant_db, + ) + + return app_variant_dto + except Exception as e: logger.exception(f"An error occurred: {str(e)}") raise HTTPException(status_code=500, detail=str(e)) diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 74130f2903..2a0c854b81 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -279,7 +279,88 @@ async def update_variant_image( ) await app_manager.update_variant_image( - db_app_variant, str(db_app_variant.project_id), image, request.state.user_id + db_app_variant, + str(db_app_variant.project_id), + image, + request.state.user_id, + ) + + # Update last_modified_by app information + await app_manager.update_last_modified_by( + user_uid=request.state.user_id, + object_id=str(db_app_variant.app_id), + object_type="app", + project_id=str(db_app_variant.project_id), + ) + logger.debug("Successfully updated last_modified_by app information") + + except ValueError as e: + import traceback + + traceback.print_exc() + detail = f"Error while trying to update the app variant: {str(e)}" + raise HTTPException(status_code=500, detail=detail) + except DockerException as e: + import traceback + + traceback.print_exc() + detail = f"Docker error while trying to update the app variant: {str(e)}" + raise HTTPException(status_code=500, detail=detail) + except Exception as e: + import traceback + + traceback.print_exc() + detail = f"Unexpected error while trying to update the app variant: {str(e)}" + raise HTTPException(status_code=500, detail=detail) + + +@router.put("/{variant_id}/url/", operation_id="update_variant_url") +async def update_variant_url( + variant_id: str, + url: str, + request: Request, +): + """ + Updates the URL used in an app variant. + + Args: + variant_id (str): The ID of the app variant to update. + url (str): The URL to update. + + Raises: + HTTPException: If an error occurs while trying to update the app variant. + + Returns: + JSONResponse: A JSON response indicating whether the update was successful or not. + """ + + try: + db_app_variant = await db_manager.fetch_app_variant_by_id( + app_variant_id=variant_id + ) + + if isCloudEE(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + project_id=str(db_app_variant.project_id), + permission=Permission.CREATE_APPLICATION, + ) + logger.debug( + f"User has Permission to update variant image: {has_permission}" + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) + + await app_manager.update_variant_url( + db_app_variant, + str(db_app_variant.project_id), + url, + request.state.user_id, ) # Update last_modified_by app information @@ -290,6 +371,7 @@ async def update_variant_image( project_id=str(db_app_variant.project_id), ) logger.debug("Successfully updated last_modified_by app information") + except ValueError as e: import traceback @@ -382,8 +464,14 @@ async def retrieve_variant_logs( try: app_variant = await db_manager.fetch_app_variant_by_id(variant_id) deployment = await db_manager.get_deployment_by_appid(str(app_variant.app.id)) - logs_result = await logs_manager.retrieve_logs(deployment.container_id) - return logs_result + if deployment.container_id is not None: + logs_result = await logs_manager.retrieve_logs(deployment.container_id) + return logs_result + else: + raise HTTPException( + 404, + detail="No logs available for this variant.", + ) except Exception as exc: logger.exception(f"An error occurred: {str(exc)}") raise HTTPException(500, {"message": str(exc)}) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index ed7eacc63f..24f0956fcc 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -136,6 +136,57 @@ async def start_variant( return URI(uri=deployment.uri) # type: ignore +async def update_last_modified_by( + user_uid: str, object_id: str, object_type: str, project_id: str +) -> None: + """Updates the last_modified_by field in the app variant table. + + Args: + object_id (str): The object ID to update. + object_type (str): The type of object to update. + user_uid (str): The user UID to update. + project_id (str): The project ID. + """ + + async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: + if object_type == "app": + return object_id + elif object_type == "variant": + app_variant_db = await db_manager.fetch_app_variant_by_id(object_id) + if app_variant_db is None: + raise db_manager.NoResultFound(f"Variant with id {object_id} not found") + return str(app_variant_db.app_id) + elif object_type == "deployment": + deployment_db = await db_manager.get_deployment_by_id(object_id) + if deployment_db is None: + raise db_manager.NoResultFound( + f"Deployment with id {object_id} not found" + ) + return str(deployment_db.app_id) + elif object_type == "evaluation": + evaluation_db = await db_manager.fetch_evaluation_by_id(object_id) + if evaluation_db is None: + raise db_manager.NoResultFound( + f"Evaluation with id {object_id} not found" + ) + return str(evaluation_db.app_id) + else: + raise ValueError( + f"Could not update last_modified_by application information. Unsupported type: {object_type}" + ) + + user = await db_manager.get_user(user_uid=user_uid) + app_id = await get_appdb_str_by_id(object_id=object_id, object_type=object_type) + assert app_id is not None, f"app_id in {object_type} cannot be None" + await db_manager.update_app( + app_id=app_id, + values_to_update={ + "modified_by_id": user.id, + "updated_at": datetime.now(timezone.utc), + }, + ) + + async def update_variant_image( app_variant_db: AppVariantDB, project_id: str, image: Image, user_uid: str ): @@ -155,13 +206,16 @@ async def update_variant_image( base = await db_manager.fetch_base_by_id(str(app_variant_db.base_id)) deployment = await db_manager.get_deployment_by_id(str(base.deployment_id)) - await deployment_manager.stop_and_delete_service(deployment) + if deployment.container_id: + await deployment_manager.stop_and_delete_service(deployment) + await db_manager.remove_deployment(str(deployment.id)) - if isOss(): - await deployment_manager.remove_image(base.image) + if base.image: + if isOss(): + await deployment_manager.remove_image(base.image) - await db_manager.remove_image(base.image, project_id) + await db_manager.remove_image(base.image, project_id) # Create a new image instance db_image = await db_manager.create_image( @@ -186,56 +240,56 @@ async def update_variant_image( await start_variant(app_variant_db, project_id, user_uid=user_uid) -async def update_last_modified_by( - user_uid: str, object_id: str, object_type: str, project_id: str -) -> None: - """Updates the last_modified_by field in the app variant table. +async def update_variant_url( + app_variant_db: AppVariantDB, project_id: str, url: str, user_uid: str +): + """Updates the URL for app variant in the database. - Args: - object_id (str): The object ID to update. - object_type (str): The type of object to update. - user_uid (str): The user UID to update. - project_id (str): The project ID. + Arguments: + app_variant (AppVariantDB): the app variant to update + project_id (str): The ID of the project + url (str): the URL to update + user_uid (str): The ID of the user updating the URL """ - async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: - if object_type == "app": - return object_id - elif object_type == "variant": - app_variant_db = await db_manager.fetch_app_variant_by_id(object_id) - if app_variant_db is None: - raise db_manager.NoResultFound(f"Variant with id {object_id} not found") - return str(app_variant_db.app_id) - elif object_type == "deployment": - deployment_db = await db_manager.get_deployment_by_id(object_id) - if deployment_db is None: - raise db_manager.NoResultFound( - f"Deployment with id {object_id} not found" - ) - return str(deployment_db.app_id) - elif object_type == "evaluation": - evaluation_db = await db_manager.fetch_evaluation_by_id(object_id) - if evaluation_db is None: - raise db_manager.NoResultFound( - f"Evaluation with id {object_id} not found" - ) - return str(evaluation_db.app_id) - else: - raise ValueError( - f"Could not update last_modified_by application information. Unsupported type: {object_type}" - ) + parsed_url = urlparse(url).geturl() - user = await db_manager.get_user(user_uid=user_uid) - app_id = await get_appdb_str_by_id(object_id=object_id, object_type=object_type) - assert app_id is not None, f"app_id in {object_type} cannot be None" - await db_manager.update_app( - app_id=app_id, - values_to_update={ - "modified_by_id": user.id, - "updated_at": datetime.now(timezone.utc), - }, + base = await db_manager.fetch_base_by_id(str(app_variant_db.base_id)) + deployment = await db_manager.get_deployment_by_id(str(base.deployment_id)) + + if deployment.container_id: + await deployment_manager.stop_and_delete_service(deployment) + + await db_manager.remove_deployment(str(deployment.id)) + + if base.image: + if isOss(): + await deployment_manager.remove_image(base.image) + + await db_manager.remove_image(base.image, project_id) + + await db_manager.update_variant_parameters( + str(app_variant_db.id), parameters={}, project_id=project_id, user_uid=user_uid ) + app_variant_db = await db_manager.update_app_variant( + app_variant_id=str(app_variant_db.id), url=parsed_url + ) + + deployment = await db_manager.create_deployment( + app_id=str(app_variant_db.app.id), + project_id=project_id, + uri=parsed_url, + status="running", + ) + + await db_manager.update_base( + str(app_variant_db.base_id), + deployment_id=deployment.id, + ) + + return URI(uri=deployment.uri) + async def terminate_and_remove_app_variant( project_id: str, @@ -468,7 +522,8 @@ async def add_variant_based_on_image( """ logger.debug("Start: Creating app variant based on image") - logger.debug("Step 1: Validating input parameters") + + logger.debug("Validating input parameters") if ( app in [None, ""] or variant_name in [None, ""] @@ -481,22 +536,25 @@ async def add_variant_based_on_image( raise ValueError("OSS: Tags is None") db_image = None - # Check if docker_id_or_template_uri is a URL or not + logger.debug("Parsing URL") parsed_url = urlparse(docker_id_or_template_uri) # Check if app variant already exists - logger.debug("Step 2: Checking if app variant already exists") + logger.debug("Checking if app variant already exists") variants = await db_manager.list_app_variants_for_app_id( - app_id=str(app.id), project_id=project_id + app_id=str(app.id), + project_id=project_id, ) + already_exists = any(av for av in variants if av.variant_name == variant_name) # type: ignore if already_exists: logger.error("App variant with the same name already exists") raise ValueError("App variant with the same name already exists") # Retrieve user and image objects - logger.debug("Step 3: Retrieving user and image objects") + logger.debug("Retrieving user and image objects") user_instance = await db_manager.get_user(user_uid) + if parsed_url.scheme and parsed_url.netloc: db_image = await db_manager.get_orga_image_instance_by_uri( template_uri=docker_id_or_template_uri, @@ -527,13 +585,13 @@ async def add_variant_based_on_image( ) # Create config - logger.debug("Step 5: Creating config") + logger.debug("Creating config") config_db = await db_manager.create_new_config( config_name=config_name, parameters={} ) # Create base - logger.debug("Step 6: Creating base") + logger.debug("Creating base") if not base_name: base_name = variant_name.split(".")[ 0 @@ -546,7 +604,7 @@ async def add_variant_based_on_image( ) # Create app variant - logger.debug("Step 7: Creating app variant") + logger.debug("Creating app variant") db_app_variant = await db_manager.create_new_app_variant( app=app, user=user_instance, @@ -558,4 +616,110 @@ async def add_variant_based_on_image( base_name=base_name, ) logger.debug("End: Successfully created db_app_variant: %s", db_app_variant) + + return db_app_variant + + +async def add_variant_based_on_url( + app: AppDB, + project_id: str, + variant_name: str, + url: str, + user_uid: str, + base_name: Optional[str] = None, + config_name: str = "default", +) -> AppVariantDB: + """ + Adds a new variant to the app based on the specified URL. + + Args: + app (AppDB): The app to add the variant to. + project_id (str): The ID of the project. + variant_name (str): The name of the new variant. + url (str): The URL to use for the new variant. + base_name (str, optional): The name of the base to use for the new variant. Defaults to None. + config_name (str, optional): The name of the configuration to use for the new variant. Defaults to "default". + user_uid (str): The UID of the user. + + Returns: + AppVariantDB: The newly created app variant. + + Raises: + ValueError: If the app variant or URL is None, or if an app variant with the same name already exists. + HTTPException: If an error occurs while creating the app variant. + """ + + logger.debug("Start: Creating app variant based on url") + + logger.debug("Validating input parameters") + if app in [None, ""] or variant_name in [None, ""] or url in [None, ""]: + raise ValueError("App variant, variant name, or URL is None") + + logger.debug("Parsing URL") + parsed_url = urlparse(url).geturl() + + logger.debug("Checking if app variant already exists") + variants = await db_manager.list_app_variants_for_app_id( + app_id=str(app.id), + project_id=project_id, + ) + + already_exists = any(av for av in variants if av.variant_name == variant_name) # type: ignore + if already_exists: + logger.error("App variant with the same name already exists") + raise ValueError("App variant with the same name already exists") + + logger.debug("Retrieving user and image objects") + user_instance = await db_manager.get_user(user_uid) + + # Create config + logger.debug("Creating config") + config_db = await db_manager.create_new_config( + config_name=config_name, parameters={} + ) + + # Create base + logger.debug("Creating base") + if not base_name: + base_name = variant_name.split(".")[ + 0 + ] # TODO: Change this in SDK2 to directly use base_name + db_base = await db_manager.create_new_variant_base( + app=app, + project_id=project_id, + base_name=base_name, # the first variant always has default base + ) + + # Create app variant + logger.debug("Creating app variant") + db_app_variant = await db_manager.create_new_app_variant( + app=app, + user=user_instance, + variant_name=variant_name, + project_id=project_id, + base=db_base, + config=config_db, + base_name=base_name, + ) + + deployment = await db_manager.create_deployment( + app_id=str(db_app_variant.app.id), + project_id=project_id, + uri=parsed_url, + status="running", + ) + + await db_manager.update_base( + str(db_app_variant.base_id), + deployment_id=deployment.id, + ) + + await db_manager.deploy_to_environment( + environment_name="production", + variant_id=str(db_app_variant.id), + user_uid=user_uid, + ) + + logger.debug("End: Successfully created variant: %s", db_app_variant) + return db_app_variant diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 8f221ad47d..b60ff45b31 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -483,7 +483,7 @@ async def create_new_variant_base( app: AppDB, project_id: str, base_name: str, - image: ImageDB, + image: Optional[ImageDB] = None, ) -> VariantBaseDB: """Create a new base. Args: @@ -501,7 +501,7 @@ async def create_new_variant_base( app_id=app.id, project_id=uuid.UUID(project_id), base_name=base_name, - image_id=image.id, + image_id=image.id if image is not None else None, ) session.add(base) @@ -536,10 +536,10 @@ async def create_new_app_variant( user: UserDB, variant_name: str, project_id: str, - image: ImageDB, base: VariantBaseDB, config: ConfigDB, base_name: str, + image: Optional[ImageDB] = None, ) -> AppVariantDB: """Create a new variant. @@ -566,7 +566,7 @@ async def create_new_app_variant( modified_by_id=user.id, revision=0, variant_name=variant_name, - image_id=image.id, + image_id=image.id if image is not None else None, base_id=base.id, base_name=base_name, config_name=config.config_name, @@ -666,10 +666,10 @@ async def create_image( async def create_deployment( app_id: str, project_id: str, - container_name: str, - container_id: str, uri: str, status: str, + container_name: Optional[str] = "", + container_id: Optional[str] = "", ) -> DeploymentDB: """Create a new deployment. @@ -701,6 +701,7 @@ async def create_deployment( await session.refresh(deployment) return deployment + except Exception as e: raise Exception(f"Error while creating deployment: {e}")