From 7d2ad402af5436b9d7cdff73e78214d8aab3f901 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 18 Dec 2024 18:29:44 +0100 Subject: [PATCH 01/27] feat (sdk:tests): create pytest configuration --- agenta-cli/tests/pytest.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 agenta-cli/tests/pytest.ini diff --git a/agenta-cli/tests/pytest.ini b/agenta-cli/tests/pytest.ini new file mode 100644 index 0000000000..4c70b8502f --- /dev/null +++ b/agenta-cli/tests/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +asyncio_default_fixture_loop_scope = session +markers = + asyncio: mark a test as an async test + variant_management: mark test as an SDK variant management test + + +[tool.pytest.ini_options] +asyncio_mode = "auto" From f0441db55fa07d566200ea273a2c7cddc1b7ab1b Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 18 Dec 2024 18:32:00 +0100 Subject: [PATCH 02/27] feat (sdk:tests): add fixtures for HTTP client and app setup, ensure all tests run in the same event loop - Add fixtures to prepare HTTP client - Ensure all tests are marked to run inside the same event loop - Create app from template for consistent test setup --- agenta-cli/tests/management/conftest.py | 129 ++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 agenta-cli/tests/management/conftest.py diff --git a/agenta-cli/tests/management/conftest.py b/agenta-cli/tests/management/conftest.py new file mode 100644 index 0000000000..6698b57f96 --- /dev/null +++ b/agenta-cli/tests/management/conftest.py @@ -0,0 +1,129 @@ +import os +import asyncio + +import httpx +import pytest +import pytest_asyncio +from pytest_asyncio import is_async_test + + +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", None) +AGENTA_API_KEY = os.environ.get("AGENTA_API_KEY", None) +AGENTA_HOST = os.environ.get("AGENTA_HOST", "http://localhost") +API_BASE_URL = f"{AGENTA_HOST}/api/" + + +def pytest_collection_modifyitems(items): + """ + Mark all tests to run inside the same event loop. + + NOTE: remove as soon as a solution for https://github.com/pytest-dev/pytest-asyncio/issues/793 is proposed and the issue closes + """ + + pytest_asyncio_tests = (item for item in items if is_async_test(item)) + session_scope_marker = pytest.mark.asyncio(loop_scope="session") + for async_test in pytest_asyncio_tests: + async_test.add_marker(session_scope_marker, append=False) + + +@pytest_asyncio.fixture(scope="session") +async def http_client(): + """ + Create an HTTP client for API testing. + """ + + async with httpx.AsyncClient( + base_url=API_BASE_URL, + timeout=httpx.Timeout(timeout=6, read=None, write=5), + headers={ + "Authorization": f"ApiKey {AGENTA_API_KEY}", + "Content-Type": "application/json", + }, + ) as client: + yield client + + +@pytest_asyncio.fixture(scope="session") +async def fetch_templates(http_client): + """ + Fetch available templates. + """ + + response = await http_client.get("containers/templates/") + return response.json() + + +@pytest_asyncio.fixture(scope="session") +async def fetch_completion_template(fetch_templates): + """ + Find the chat_openai template. + """ + + return next( + (temp for temp in fetch_templates if temp["image"]["name"] == "chat_openai"), + None, + ) + + +@pytest_asyncio.fixture(scope="session") +async def app_from_template_payload(fetch_completion_template): + """ + Prepare payload for creating an app from a template. + """ + + return { + "app_name": "completion_app", + "env_vars": {"OPENAI_API_KEY": OPENAI_API_KEY}, + "template_id": fetch_completion_template.get("id", None), + } + + +@pytest_asyncio.fixture(scope="session") +async def get_completion_app_from_list(http_client): + """ + Retrieve the first available application. + """ + + list_app_response = await http_client.get("apps/") + list_app_response.raise_for_status() + + apps_response = list_app_response.json() + if not apps_response: + raise ValueError("No applications found") + + return apps_response[0] + + +@pytest_asyncio.fixture(scope="session") +async def create_app_from_template(app_from_template_payload, http_client): + # Create app + create_app_response = await http_client.post( + "apps/app_and_variant_from_template", json=app_from_template_payload + ) + create_app_response.raise_for_status() + + # Fetch created app + list_app_response = await http_client.get("apps/") + list_app_response.raise_for_status() + + apps_response = list_app_response.json() + if not apps_response: + raise ValueError("No applications found after creation") + + # Small delay to ensure app is ready + await asyncio.sleep(3) + + app_response = apps_response[0] + + try: + # Yield the app for tests to use + yield app_response + finally: + # Cleanup: Delete the app after all tests in the class are complete + try: + delete_response = await http_client.delete( + f"apps/{app_response.get('app_id', None)}" + ) + delete_response.raise_for_status() + except Exception as e: + print(f"Error during app cleanup: {e}") From 5637d9af8daba5c439e872bb41acc7cae9cdcdec Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 18 Dec 2024 18:33:54 +0100 Subject: [PATCH 03/27] feat (sdk:tests): create variant manager test suite (in progress) - Set up initial structure for variant manager tests - Mark testcases as variant_manager - Add preliminary test cases (work in progress) --- .../tests/management/test_variant_manager.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 agenta-cli/tests/management/test_variant_manager.py diff --git a/agenta-cli/tests/management/test_variant_manager.py b/agenta-cli/tests/management/test_variant_manager.py new file mode 100644 index 0000000000..2bd3cfb0f1 --- /dev/null +++ b/agenta-cli/tests/management/test_variant_manager.py @@ -0,0 +1,106 @@ +import uuid + +import pytest + + +@pytest.mark.usefixtures("create_app_from_template") +class TestVariantManager: + @pytest.mark.asyncio + @pytest.mark.variant_management + async def test_configs_add_success(self, http_client, get_completion_app_from_list): + # ARRANGE: Prepare test data + test_variant_slug = "from_pytest" + app_id = get_completion_app_from_list.get("app_id", None) + + # ACT: Add configuration + response = await http_client.post( + "/api/variants/configs/add", + json={ + "variant_ref": {"slug": test_variant_slug, "version": None, "id": None}, + "application_ref": {"slug": None, "version": None, "id": app_id}, + }, + ) + + # ASSERT: Verify response + assert ( + response.status_code == 200 + ), f"Failed to add config for variant {test_variant_slug}" + response_data = response.json() + assert "params" in response_data, "Response missing 'params'" + assert "url" in response_data, "Response missing 'url'" + + @pytest.mark.asyncio + @pytest.mark.variant_management + async def test_configs_add_duplicate( + self, http_client, get_completion_app_from_list + ): + # ARRANGE: Prepare test data for an existing configuration + existing_variant_slug = "default" + app_id = get_completion_app_from_list.get("app_id", None) + + # ACT: Attempt to add duplicate configuration + response = await http_client.post( + "/api/variants/configs/add", + json={ + "variant_ref": { + "slug": existing_variant_slug, + "version": None, + "id": None, + }, + "application_ref": {"slug": None, "version": None, "id": app_id}, + }, + ) + + # ASSERT: Verify error response for duplicate + assert response.status_code == 400, "Expected 400 error for duplicate config" + assert ( + response.json()["detail"] == "Config already exists." + ), "Incorrect error message for duplicate config" + + @pytest.mark.asyncio + @pytest.mark.variant_management + async def test_configs_nonexistent_app(self, http_client): + # ARRANGE: Prepare test data with non-existent application + non_existent_app_id = str(uuid.uuid4()) + + # ACT: Attempt to add config for non-existent application + response = await http_client.post( + "/api/variants/configs/add", + json={ + "variant_ref": {"slug": "default", "version": None, "id": None}, + "application_ref": { + "slug": None, + "version": None, + "id": non_existent_app_id, + }, + }, + ) + + # ASSERT: Verify error response for non-existent application + assert ( + response.status_code == 404 + ), "Expected 404 error for non-existent application" + assert ( + response.json()["detail"] == "Config not found." + ), "Incorrect error message for non-existent application" + + @pytest.mark.asyncio + @pytest.mark.variant_management + async def test_configs_add_invalid_data(self, http_client): + # ARRANGE: Prepare invalid test data + invalid_variant_data = { + "variant_ref": { + "slug": "non-existent", + "version": 3, + "id": "non-existent-id", + }, + "application_ref": {"slug": None, "version": None, "id": None}, + } + + # ACT: Attempt to add configuration with invalid data + response = await http_client.post( + "/api/variants/configs/add", json=invalid_variant_data + ) + + # ASSERT: Verify validation error + assert response.status_code == 422, "Expected 422 validation error" From 56dd7572fd911a61a1a0e7273049744d6735c984 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 18 Dec 2024 18:36:12 +0100 Subject: [PATCH 04/27] chore (sdk:tests): add bash script to run tests in sdk --- agenta-cli/tests/run_tests.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 agenta-cli/tests/run_tests.sh diff --git a/agenta-cli/tests/run_tests.sh b/agenta-cli/tests/run_tests.sh new file mode 100755 index 0000000000..57ccf07b30 --- /dev/null +++ b/agenta-cli/tests/run_tests.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +OPENAI_API_KEY=sk-xxxxxxxx AGENTA_HOST=http://localhost AGENTA_API_KEY=xxxxx.xxxxxxxxxxxxx pytest management/ -v -m variant_management # run variant_management tests + From 645a276dedcc5055a64ecb15c8ecb742ee989bb8 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 14:21:59 +0100 Subject: [PATCH 05/27] chore (deps): add mypy and pytest-xdist dev dependencies to agenta-cli --- agenta-cli/poetry.lock | 102 +++++++++++++++++++++++++++++++++++++- agenta-cli/pyproject.toml | 2 + 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/agenta-cli/poetry.lock b/agenta-cli/poetry.lock index a40ffb82b9..df3b43f8a6 100644 --- a/agenta-cli/poetry.lock +++ b/agenta-cli/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -422,6 +422,20 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "execnet" +version = "2.1.1" +description = "execnet: rapid multi-Python deployment" +optional = false +python-versions = ">=3.8" +files = [ + {file = "execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc"}, + {file = "execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3"}, +] + +[package.extras] +testing = ["hatch", "pre-commit", "pytest", "tox"] + [[package]] name = "fastapi" version = "0.115.5" @@ -1168,6 +1182,70 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +[[package]] +name = "mypy" +version = "1.13.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, + {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, + {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, + {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, + {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, + {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, + {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, + {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, + {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, + {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, + {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, + {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, + {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, + {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, + {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, + {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, + {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, + {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, + {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, + {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, + {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, + {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "openai" version = "1.54.4" @@ -1675,6 +1753,26 @@ pytest = ">=8.2,<9" docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] +[[package]] +name = "pytest-xdist" +version = "3.6.1" +description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_xdist-3.6.1-py3-none-any.whl", hash = "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7"}, + {file = "pytest_xdist-3.6.1.tar.gz", hash = "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d"}, +] + +[package.dependencies] +execnet = ">=2.1" +pytest = ">=7.0.0" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2555,4 +2653,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "a0343fecda7a4a2cb0fb7ae961154ece9aa1237991dbc18718ca383add362c47" +content-hash = "522bb5aa006e2ab6dd86d1df90e8ef6ce3c0d6a4e9e179e024fece1167a6e451" diff --git a/agenta-cli/pyproject.toml b/agenta-cli/pyproject.toml index fc823fc410..9b332546b2 100644 --- a/agenta-cli/pyproject.toml +++ b/agenta-cli/pyproject.toml @@ -45,6 +45,8 @@ setuptools = "^71.1.0" [tool.poetry.group.dev.dependencies] pytest-asyncio = "^0.24.0" +mypy = "^1.13.0" +pytest-xdist = "^3.6.1" [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" From 1ab5da6800256a71ad65a1222176d5d33362e313 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 14:22:47 +0100 Subject: [PATCH 06/27] refactor (backend): add optional attribute field to AppVariantRevision api model --- agenta-backend/agenta_backend/models/api/api_models.py | 1 + agenta-backend/agenta_backend/models/converters.py | 1 + 2 files changed, 2 insertions(+) diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index bbf13b41d6..87f887729f 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -116,6 +116,7 @@ class AppVariantResponse(BaseModel): class AppVariantRevision(BaseModel): + id: Optional[str] = None revision: int modified_by: str config: ConfigDB diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index d79b12c0f9..8f7a329fa1 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -334,6 +334,7 @@ async def app_variant_db_revision_to_output( app_variant_revision_db: AppVariantRevisionsDB, ) -> AppVariantRevision: return AppVariantRevision( + id=str(app_variant_revision_db.id) or None, revision=app_variant_revision_db.revision, modified_by=app_variant_revision_db.modified_by.username, config=ConfigDB( From 72a876e7172735441a918cb22a5365a88dabceb1 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 14:23:32 +0100 Subject: [PATCH 07/27] chore (tests): update pytest configuration --- agenta-cli/tests/pytest.ini | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/agenta-cli/tests/pytest.ini b/agenta-cli/tests/pytest.ini index 4c70b8502f..6f1bcf538a 100644 --- a/agenta-cli/tests/pytest.ini +++ b/agenta-cli/tests/pytest.ini @@ -1,9 +1,13 @@ [pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* asyncio_default_fixture_loop_scope = session markers = asyncio: mark a test as an async test - variant_management: mark test as an SDK variant management test - + variant_manager: mark testcase as part of the SDK Variant Manager testsuite + deployment_manager: mark testcase as part of the SDK Deployment Manager testsuite [tool.pytest.ini_options] asyncio_mode = "auto" From 8d43c129eca8da6ffd13bf496051cbc2759756ff Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 14:24:41 +0100 Subject: [PATCH 08/27] refactor (sdk:tests): add saltiness to name generation for app from template creation and update `create_app_from_template` fixture --- agenta-cli/tests/management/conftest.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/agenta-cli/tests/management/conftest.py b/agenta-cli/tests/management/conftest.py index 6698b57f96..2403258801 100644 --- a/agenta-cli/tests/management/conftest.py +++ b/agenta-cli/tests/management/conftest.py @@ -1,4 +1,5 @@ import os +import uuid import asyncio import httpx @@ -65,6 +66,10 @@ async def fetch_completion_template(fetch_templates): ) +def get_random_name(): + return f"completion_app_{uuid.uuid4().hex[:8]}" + + @pytest_asyncio.fixture(scope="session") async def app_from_template_payload(fetch_completion_template): """ @@ -72,7 +77,7 @@ async def app_from_template_payload(fetch_completion_template): """ return { - "app_name": "completion_app", + "app_name": get_random_name(), "env_vars": {"OPENAI_API_KEY": OPENAI_API_KEY}, "template_id": fetch_completion_template.get("id", None), } @@ -102,19 +107,11 @@ async def create_app_from_template(app_from_template_payload, http_client): ) create_app_response.raise_for_status() - # Fetch created app - list_app_response = await http_client.get("apps/") - list_app_response.raise_for_status() - - apps_response = list_app_response.json() - if not apps_response: - raise ValueError("No applications found after creation") - # Small delay to ensure app is ready await asyncio.sleep(3) - app_response = apps_response[0] - + # Get response data + app_response = create_app_response.json() try: # Yield the app for tests to use yield app_response From 68e5924fef73eda1a81d864ed275442016e168ae Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 14:26:15 +0100 Subject: [PATCH 09/27] refactor (sdk:tests): update marker to `variant_manager` and remove `/api/` from http_client in each test case --- agenta-cli/tests/management/variant/__init__.py | 0 .../{ => variant}/test_variant_manager.py | 16 ++++++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 agenta-cli/tests/management/variant/__init__.py rename agenta-cli/tests/management/{ => variant}/test_variant_manager.py (91%) diff --git a/agenta-cli/tests/management/variant/__init__.py b/agenta-cli/tests/management/variant/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/management/test_variant_manager.py b/agenta-cli/tests/management/variant/test_variant_manager.py similarity index 91% rename from agenta-cli/tests/management/test_variant_manager.py rename to agenta-cli/tests/management/variant/test_variant_manager.py index 2bd3cfb0f1..eb4e2bac8a 100644 --- a/agenta-cli/tests/management/test_variant_manager.py +++ b/agenta-cli/tests/management/variant/test_variant_manager.py @@ -6,7 +6,7 @@ @pytest.mark.usefixtures("create_app_from_template") class TestVariantManager: @pytest.mark.asyncio - @pytest.mark.variant_management + @pytest.mark.variant_manager async def test_configs_add_success(self, http_client, get_completion_app_from_list): # ARRANGE: Prepare test data test_variant_slug = "from_pytest" @@ -14,7 +14,7 @@ async def test_configs_add_success(self, http_client, get_completion_app_from_li # ACT: Add configuration response = await http_client.post( - "/api/variants/configs/add", + "variants/configs/add", json={ "variant_ref": {"slug": test_variant_slug, "version": None, "id": None}, "application_ref": {"slug": None, "version": None, "id": app_id}, @@ -30,7 +30,7 @@ async def test_configs_add_success(self, http_client, get_completion_app_from_li assert "url" in response_data, "Response missing 'url'" @pytest.mark.asyncio - @pytest.mark.variant_management + @pytest.mark.variant_manager async def test_configs_add_duplicate( self, http_client, get_completion_app_from_list ): @@ -40,7 +40,7 @@ async def test_configs_add_duplicate( # ACT: Attempt to add duplicate configuration response = await http_client.post( - "/api/variants/configs/add", + "variants/configs/add", json={ "variant_ref": { "slug": existing_variant_slug, @@ -58,14 +58,14 @@ async def test_configs_add_duplicate( ), "Incorrect error message for duplicate config" @pytest.mark.asyncio - @pytest.mark.variant_management + @pytest.mark.variant_manager async def test_configs_nonexistent_app(self, http_client): # ARRANGE: Prepare test data with non-existent application non_existent_app_id = str(uuid.uuid4()) # ACT: Attempt to add config for non-existent application response = await http_client.post( - "/api/variants/configs/add", + "variants/configs/add", json={ "variant_ref": {"slug": "default", "version": None, "id": None}, "application_ref": { @@ -85,7 +85,7 @@ async def test_configs_nonexistent_app(self, http_client): ), "Incorrect error message for non-existent application" @pytest.mark.asyncio - @pytest.mark.variant_management + @pytest.mark.variant_manager async def test_configs_add_invalid_data(self, http_client): # ARRANGE: Prepare invalid test data invalid_variant_data = { @@ -99,7 +99,7 @@ async def test_configs_add_invalid_data(self, http_client): # ACT: Attempt to add configuration with invalid data response = await http_client.post( - "/api/variants/configs/add", json=invalid_variant_data + "variants/configs/add", json=invalid_variant_data ) # ASSERT: Verify validation error From c125d03056e2b3aa9be1f2db22c7f53561dd0e8b Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 14:28:00 +0100 Subject: [PATCH 10/27] feat (sdk:tests): add fixture for Deployment Manager test suites and create test cases for it --- agenta-cli/tests/__init__.py | 0 .../tests/management/deployment/__init__.py | 0 .../tests/management/deployment/fixtures.py | 21 ++++++ .../deployment/test_deployment_manager.py | 65 +++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 agenta-cli/tests/__init__.py create mode 100644 agenta-cli/tests/management/deployment/__init__.py create mode 100644 agenta-cli/tests/management/deployment/fixtures.py create mode 100644 agenta-cli/tests/management/deployment/test_deployment_manager.py diff --git a/agenta-cli/tests/__init__.py b/agenta-cli/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/management/deployment/__init__.py b/agenta-cli/tests/management/deployment/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/management/deployment/fixtures.py b/agenta-cli/tests/management/deployment/fixtures.py new file mode 100644 index 0000000000..943dfbd548 --- /dev/null +++ b/agenta-cli/tests/management/deployment/fixtures.py @@ -0,0 +1,21 @@ +import pytest_asyncio + + +@pytest_asyncio.fixture(scope="session") +async def list_app_variants(http_client, get_completion_app_from_list): + app_id = get_completion_app_from_list.get("app_id", None) + response = await http_client.get(f"apps/{app_id}/variants") + response.raise_for_status() + response_data = response.json() + return response_data + + +@pytest_asyncio.fixture(scope="session") +async def get_variant_revisions(http_client, list_app_variants): + app_variant = list_app_variants[0] + response = await http_client.get( + f"variants/{app_variant.get('variant_id', '')}/revisions" + ) + response.raise_for_status() + response_data = response.json() + return response_data diff --git a/agenta-cli/tests/management/deployment/test_deployment_manager.py b/agenta-cli/tests/management/deployment/test_deployment_manager.py new file mode 100644 index 0000000000..2bd1fb5b6b --- /dev/null +++ b/agenta-cli/tests/management/deployment/test_deployment_manager.py @@ -0,0 +1,65 @@ +import uuid + +import pytest + +from tests.management.deployment.fixtures import * + + +@pytest.mark.usefixtures("create_app_from_template") +class TestDeploymentManager: + @pytest.mark.asyncio + @pytest.mark.deployment_manager + async def test_configs_deploy_success( + self, http_client, get_completion_app_from_list, get_variant_revisions + ): + # ARRANGE: Prepare test data + app_id = get_completion_app_from_list.get("app_id", None) + variant_revision = get_variant_revisions[0] + variant_revision_id = variant_revision.get("id", None) + variant_revision_config_name = variant_revision.get("config", {}).get( + "config_name", None + ) + variant_revision_version = variant_revision.get("revision") + + # ACT: Add configuration + response = await http_client.post( + "variants/configs/deploy", + json={ + "variant_ref": { + "slug": variant_revision_config_name, + "version": variant_revision_version, + "id": variant_revision_id, + }, + "environment_ref": {"slug": "production", "version": None, "id": None}, + "application_ref": { + "slug": None, + "version": None, + "id": app_id, + }, + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert "params" and "url" in response.json() + assert "environment_lifecycle" in response.json() + + @pytest.mark.asyncio + @pytest.mark.deployment_manager + async def test_configs_deploy_not_found(self, http_client): + # ACT: Add configuration + response = await http_client.post( + "variants/configs/deploy", + json={ + "variant_ref": { + "slug": "default.appvariant", + "version": 3, + "id": str(uuid.uuid4()), # non-existent config + }, + "environment_ref": {"slug": "production", "version": None, "id": None}, + }, + ) + + # ASSERT: Verify response + assert response.status_code == 404 + assert response.json()["detail"] == "Config not found." From dc5db1b94a6c072ed3c6140c807a15c9e4cea233 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 15:16:27 +0100 Subject: [PATCH 11/27] feat (sdk:tests): add more test cases to VariantManager test suite. --- .../variant/test_variant_manager.py | 287 ++++++++++++++++++ 1 file changed, 287 insertions(+) diff --git a/agenta-cli/tests/management/variant/test_variant_manager.py b/agenta-cli/tests/management/variant/test_variant_manager.py index eb4e2bac8a..660f105cbc 100644 --- a/agenta-cli/tests/management/variant/test_variant_manager.py +++ b/agenta-cli/tests/management/variant/test_variant_manager.py @@ -2,6 +2,8 @@ import pytest +from tests.management.deployment.fixtures import * + @pytest.mark.usefixtures("create_app_from_template") class TestVariantManager: @@ -104,3 +106,288 @@ async def test_configs_add_invalid_data(self, http_client): # ASSERT: Verify validation error assert response.status_code == 422, "Expected 422 validation error" + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_commit_success( + self, + http_client, + get_variant_revisions, + get_completion_app_from_list, + ): + # ARRANGE: Prepare test data + app_id = get_completion_app_from_list.get("app_id", None) + variant_revision = get_variant_revisions[0] + variant_revision_id = variant_revision.get("id", None) + variant_revision_config_name = variant_revision.get("config", {}).get( + "config_name", None + ) + variant_revision_version = variant_revision.get("revision") + + # ACT: Add configuration + response = await http_client.post( + "variants/configs/commit", + json={ + "config": { + "params": { + "model": "gpt-4", + "top_p": 1, + "inputs": [{"name": "country"}], + "force_json": 0, + "max_tokens": 1000, + "prompt_user": "What is the capital of {country}?", + "temperature": 0.65, + "prompt_system": "You are an expert in geography.", + "presence_penalty": 0, + "frequence_penalty": 0, + }, + "application_ref": { + "slug": None, + "version": None, + "id": app_id, + }, + "service_ref": None, + "variant_ref": { + "slug": variant_revision_config_name, + "version": variant_revision_version, + "id": variant_revision_id, + }, + "environment_ref": None, + } + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert "params" and "url" in response.json() + assert "variant_lifecycle" in response.json() + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_commit_missing_data( + self, http_client, get_completion_app_from_list + ): + # ARRANGE: Prepare test data + app_id = get_completion_app_from_list.get("app_id", None) + + # ACT: Add configuration + response = await http_client.post( + "variants/configs/commit", + json={ + "params": {}, + "url": "", + "application_ref": { + "slug": "test", + "version": None, + "id": app_id, + }, + }, + ) + + # ASSERT: Verify response + assert response.status_code == 422 + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_delete_success( + self, http_client, get_completion_app_from_list, list_app_variants + ): + # ARRANGE: Prepare test data + app_name = get_completion_app_from_list.get("app_name", None) + app_variant = list_app_variants[0] + + # ACT: Add configuration + variant_response = await http_client.post( + "variants/from-base", + json={ + "base_id": app_variant.get("base_id"), + "new_variant_name": "from_pytest_for_deletion", + "new_config_name": "from_base_config", + "parameters": {}, + }, + ) + variant_response.raise_for_status() + + response = await http_client.post( + "variants/configs/delete", + json={ + "variant_ref": { + "slug": "from_pytest_for_deletion", + "version": None, + "id": None, + }, + "application_ref": { + "slug": app_name, + "version": None, + "id": None, + }, + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert 204 == response.json() + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_delete_not_found( + self, http_client, get_completion_app_from_list + ): + # ARRANGE: Prepare test data + app_name = get_completion_app_from_list.get("app_name", None) + + # ACT: Add configuration + response = await http_client.post( + "variants/configs/delete", + json={ + "variant_ref": { + "slug": "non-existent-variant", + "version": None, + "id": None, + }, + "application_ref": { + "slug": app_name, + "version": None, + "id": None, + }, + }, + ) + + # ASSERT: Verify response + assert 204 == response.json() + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_list_success( + self, http_client, get_completion_app_from_list + ): + # ARRANGE: Prepare test data + app_name = get_completion_app_from_list.get("app_name", None) + + # ACT: Add configuration + response = await http_client.post( + "variants/configs/list", + json={ + "application_ref": { + "slug": app_name, + "version": None, + "id": None, + } + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert isinstance(response.json(), list) + assert len(response.json()) > 0 + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_list_not_found(self, http_client): + # ACT: Add configuration + response = await http_client.post( + "variants/configs/list", + json={ + "application_ref": { + "slug": "non_existent_app", + "version": None, + "id": None, + } + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert [] == response.json() + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_history_by_slug_and_appid_success( + self, + http_client, + get_completion_app_from_list, + get_variant_revisions, + ): + # ARRANGE: Prepare test data + app_id = get_completion_app_from_list.get("app_id", None) + variant_revision = get_variant_revisions[0] + variant_revision_config_name = variant_revision.get("config", {}).get( + "config_name", None + ) + + # ACT: Add configuration + response = await http_client.post( + "variants/configs/history", + json={ + "application_ref": { + "slug": None, + "version": None, + "id": app_id, + }, + "variant_ref": { + "slug": variant_revision_config_name, + "version": None, + "id": None, + }, + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert isinstance(response.json(), list) + assert len(response.json()) >= 1 + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_history_by_id_success(self, http_client, list_app_variants): + # ARRANGE: Prepare test data + app_variant = list_app_variants[0] + + # ACT: Add configuration + response = await http_client.post( + "variants/configs/history", + json={ + "variant_ref": { + "slug": None, + "version": None, + "id": app_variant.get("variant_id", None), + }, + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert isinstance(response.json(), list) + assert len(response.json()) > 0 + + @pytest.mark.asyncio + @pytest.mark.variant_manager + async def test_configs_history_not_found(self, http_client): + # ACT: Add configuration + app_not_found_response = await http_client.post( + "variants/configs/history", + json={ + "variant_ref": { + "slug": "non_existent_app", + "version": None, + "id": None, + } + }, + ) + variant_not_found_response = await http_client.post( + "variants/configs/history", + json={ + "variant_ref": { + "slug": None, + "version": None, + "id": str(uuid.uuid4()), + } + }, + ) + + # ASSERT: Verify response + assert app_not_found_response.status_code == 200 + assert variant_not_found_response.status_code == 200 + assert [] == ( + app_not_found_response.json() and variant_not_found_response.json() + ) From 035c0566e5c0def5b156110c20ac63289d67dfea Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 16:38:08 +0100 Subject: [PATCH 12/27] feat (sdk:tests): add fixture for Config Manager test suites and create test cases for it --- .../tests/management/config/__init__.py | 0 .../tests/management/config/fixtures.py | 12 ++ .../management/config/test_config_manager.py | 121 ++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 agenta-cli/tests/management/config/__init__.py create mode 100644 agenta-cli/tests/management/config/fixtures.py create mode 100644 agenta-cli/tests/management/config/test_config_manager.py diff --git a/agenta-cli/tests/management/config/__init__.py b/agenta-cli/tests/management/config/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/management/config/fixtures.py b/agenta-cli/tests/management/config/fixtures.py new file mode 100644 index 0000000000..91d196793e --- /dev/null +++ b/agenta-cli/tests/management/config/fixtures.py @@ -0,0 +1,12 @@ +import pytest_asyncio + + +@pytest_asyncio.fixture(scope="session") +async def get_production_environment_revision( + http_client, get_completion_app_from_list +): + app_id = get_completion_app_from_list.get("app_id", None) + response = await http_client.get(f"apps/{app_id}/revisions/production") + response.raise_for_status() + response_data = response.json() + return response_data diff --git a/agenta-cli/tests/management/config/test_config_manager.py b/agenta-cli/tests/management/config/test_config_manager.py new file mode 100644 index 0000000000..ba8b438cc3 --- /dev/null +++ b/agenta-cli/tests/management/config/test_config_manager.py @@ -0,0 +1,121 @@ +import pytest + +from tests.management.config.fixtures import * +from tests.management.deployment.fixtures import * + + +@pytest.mark.usefixtures("create_app_from_template") +class TestDConfigManager: + @pytest.mark.asyncio + @pytest.mark.config_manager + async def test_configs_fetch_by_variant_ref( + self, http_client, get_variant_revisions + ): + # ARRANGE: Prepare test data + variant_revision = get_variant_revisions[0] + variant_revision_id = variant_revision.get("id", None) + variant_revision_config_name = variant_revision.get("config", {}).get( + "config_name", None + ) + variant_revision_version = variant_revision.get("revision") + + # ACT: Add configuration + response = await http_client.post( + url="variants/configs/fetch", + json={ + "variant_ref": { + "slug": variant_revision_config_name, + "version": variant_revision_version, + "id": variant_revision_id, + } + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert "params" in response.json() + assert "application_ref" in response.json() + assert "variant_ref" in response.json() + assert "service_ref" in response.json() + assert "environment_ref" in response.json() + assert "variant_lifecycle" in response.json() + + @pytest.mark.asyncio + @pytest.mark.config_manager + async def test_configs_fetch_by_environment_and_application_ref( + self, http_client, get_completion_app_from_list + ): + # ARRANGE: Prepare test data + app_id = get_completion_app_from_list.get("app_id", None) + + # ACT: Add configuration + response = await http_client.post( + url="variants/configs/fetch", + json={ # type: ignore + "environment_ref": {"slug": "production", "version": 1, "id": None}, + "application_ref": { + "slug": None, + "version": None, + "id": app_id, + }, + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert "params" in response.json() + assert "application_ref" in response.json() + assert "variant_ref" in response.json() + assert "service_ref" in response.json() + assert "environment_ref" in response.json() + assert "variant_lifecycle" in response.json() + + @pytest.mark.asyncio + @pytest.mark.config_manager + async def test_configs_fetch_by_environment_ref( + self, http_client, get_production_environment_revision + ): + # ARRANGE: Prepare test data + environment_revision = get_production_environment_revision.get("revisions", [])[ + 0 + ] + + # ACT: Add configuration + response = await http_client.post( + url="variants/configs/fetch", + json={ # type: ignore + "environment_ref": { + "slug": None, + "version": None, + "id": environment_revision.get("id", None), + } + }, + ) + + # ASSERT: Verify response + assert response.status_code == 200 + assert "params" in response.json() + assert "application_ref" in response.json() + assert "variant_ref" in response.json() + assert "service_ref" in response.json() + assert "environment_ref" in response.json() + assert "variant_lifecycle" in response.json() + + @pytest.mark.asyncio + @pytest.mark.config_manager + async def test_configs_fetch_not_found(self, http_client): + # ACT: Add configuration + response = await http_client.post( + url="variants/configs/fetch", + params={ # type: ignore + "variant_ref": { + "slug": "non-existent", + "version": 1, + "id": "non-existent-id", + } + }, + ) + + # ASSERT: Verify response + assert response.status_code == 404 + assert response.json()["detail"] == "Config not found." From a4ddab10528e805da9c68d8aba48a51e439614af Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Dec 2024 16:38:55 +0100 Subject: [PATCH 13/27] chore (backend): add variant_revision id in `app_variant_db_revisions_to_output` api converter --- agenta-backend/agenta_backend/models/converters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 8f7a329fa1..cb9c22639d 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -318,6 +318,7 @@ async def app_variant_db_revisions_to_output( for app_variant_revision_db in app_variant_revisions_db: app_variant_revisions.append( AppVariantRevision( + id=str(app_variant_revision_db.id) or None, revision=app_variant_revision_db.revision, modified_by=app_variant_revision_db.modified_by.username, config={ From eec75b3c6517324dda7fd45ddd022cbcba63fc98 Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 20 Dec 2024 13:33:10 +0100 Subject: [PATCH 14/27] feat (sdk:tests): add fixture for SDK routing test suites and create test cases for it --- agenta-cli/tests/sdk_routing/__init__.py | 0 .../tests/sdk_routing/assets/greetings/app.py | 29 +++ .../sdk_routing/assets/greetings/main.py | 13 ++ .../assets/greetings/requirements.txt | 0 agenta-cli/tests/sdk_routing/conftest.py | 189 ++++++++++++++++++ agenta-cli/tests/sdk_routing/test_routers.py | 102 ++++++++++ 6 files changed, 333 insertions(+) create mode 100644 agenta-cli/tests/sdk_routing/__init__.py create mode 100644 agenta-cli/tests/sdk_routing/assets/greetings/app.py create mode 100644 agenta-cli/tests/sdk_routing/assets/greetings/main.py create mode 100644 agenta-cli/tests/sdk_routing/assets/greetings/requirements.txt create mode 100644 agenta-cli/tests/sdk_routing/conftest.py create mode 100644 agenta-cli/tests/sdk_routing/test_routers.py diff --git a/agenta-cli/tests/sdk_routing/__init__.py b/agenta-cli/tests/sdk_routing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/sdk_routing/assets/greetings/app.py b/agenta-cli/tests/sdk_routing/assets/greetings/app.py new file mode 100644 index 0000000000..34349302a6 --- /dev/null +++ b/agenta-cli/tests/sdk_routing/assets/greetings/app.py @@ -0,0 +1,29 @@ +from importlib.metadata import version + +import agenta as ag + + +ag.init(host="http://localhost") +ag.config.default( + flag=ag.BinaryParam(value=True), +) + + +class CustomException(Exception): + def __init__(self, message): + self.message = message + self.status_code = 401 + + def __str__(self): + return self.message + + +@ag.entrypoint +@ag.instrument(spankind="workflow") +async def greetings(name: str): + message = "Hello, World!" + + if ag.config.flag: + message = f"Hello, {name}! (version={version('agenta')})" + + return message diff --git a/agenta-cli/tests/sdk_routing/assets/greetings/main.py b/agenta-cli/tests/sdk_routing/assets/greetings/main.py new file mode 100644 index 0000000000..00a638ad69 --- /dev/null +++ b/agenta-cli/tests/sdk_routing/assets/greetings/main.py @@ -0,0 +1,13 @@ +from os import getenv + +from uvicorn import run # type: ignore + +import app # type: ignore +import agenta # pylint: disable=unused-import + + +HOST = getenv("HOST", "0.0.0.0") +PORT = int(getenv("PORT", "8888")) + +if __name__ == "__main__": + run("agenta:app", host=HOST, port=PORT) diff --git a/agenta-cli/tests/sdk_routing/assets/greetings/requirements.txt b/agenta-cli/tests/sdk_routing/assets/greetings/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/sdk_routing/conftest.py b/agenta-cli/tests/sdk_routing/conftest.py new file mode 100644 index 0000000000..0947279335 --- /dev/null +++ b/agenta-cli/tests/sdk_routing/conftest.py @@ -0,0 +1,189 @@ +import os +import sys +import time +import uuid +import socket +import random +import threading +import subprocess +from pathlib import Path +from importlib.metadata import version + +import httpx +import pytest + + +BASE_URL = os.getenv("BASE_URL", "http://127.0.0.1") +AGENTA_HOST = os.getenv("AGENTA_HOST", "http://localhost") +API_BASE_URL = f"{AGENTA_HOST}/api/" +API_KEY = os.getenv("AGENTA_API_KEY") + + +def get_free_port(start=8001, end=8999, max_attempts=100): + """ + Find an available port within the specified range with a maximum number of attempts. + """ + + attempts = 0 + + while attempts < max_attempts: + port = random.randint(start, end) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("127.0.0.1", port)) + return port + + except OSError: + attempts += 1 + + raise RuntimeError("Could not find a free port within the range") + + +@pytest.fixture(scope="class") +def get_agenta_version(): + return version("agenta") + + +@pytest.fixture(scope="class") +def execute_python(): + """ + Fixture to provide the current Python executable. + """ + + python_executable = sys.executable + return python_executable + + +@pytest.fixture(scope="class") +def get_port_number(): + port = get_free_port() + return port + + +@pytest.fixture(scope="class") +def http_client(get_port_number): + """ + Create an HTTP client for API testing. + """ + + with httpx.Client( + base_url=f"{BASE_URL}:{get_port_number}", + timeout=httpx.Timeout(timeout=6, read=None, write=5), + headers={ + "Authorization": f"ApiKey {API_KEY}", + "Content-Type": "application/json", + }, + ) as client: + yield client + + +@pytest.fixture(scope="class") +def create_application(http_client): + """ + Create an application and set the APP_ID in the environment + """ + + response = http_client.post( + f"{API_BASE_URL}apps/", json={"app_name": f"app_{uuid.uuid4().hex[:8]}"} + ) + response.raise_for_status() + response_data = response.json() + return response_data + + +@pytest.fixture(scope="class") +def fastapi_server( + request, get_port_number, create_application, http_client, execute_python +): + """ + Run the FastAPI server as a subprocess on a random port and return its base URL. + """ + + app_id = create_application.get("app_id", None) + app_file = request.param.get("app_file", "main.py") + env_vars = request.param.get("env_vars", {}) + + app_folder = Path(__file__).parent + + if not (app_folder / app_file).exists(): + raise FileNotFoundError(f"FastAPI app not found at: {app_folder / app_file}") + + env_vars.update( + { + "AGENTA_APP_ID": app_id, + "AGENTA_HOST": BASE_URL, + "HOST": "0.0.0.0", + "PORT": str(get_port_number), + } + ) + + command = [ + execute_python, + app_file, + ] + + process = subprocess.Popen( + command, + cwd=app_folder, + env=env_vars, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + def print_logs(pipe, prefix): + for line in iter(pipe.readline, ""): + print(f"{prefix}: {line.strip()}") + pipe.close() + + threading.Thread( + target=print_logs, + args=(process.stdout, "STDOUT"), + daemon=True, + ).start() + threading.Thread( + target=print_logs, + args=(process.stderr, "STDERR"), + daemon=True, + ).start() + + # Wait a bit for the server to start + time.sleep(2) + + yield BASE_URL, process + + process.terminate() + process.wait() + + # Remove application after server teardown + response = http_client.delete(f"{API_BASE_URL}apps/{app_id}") + response.raise_for_status() + + +@pytest.fixture(scope="class") +def ensure_server(fastapi_server, http_client): + """ + Ensure the server is running by checking the health endpoint. + """ + + _, process = fastapi_server + + for i in range(10): + try: + response = http_client.get("/") + if response.status_code == 200: + return + + print( + f"Health check attempt {i+1}/10 failed with status {response.status_code}" + ) + except (ConnectionError, TimeoutError) as e: + print(f"Health check attempt {i+1}/10 failed: {e}") + time.sleep(2) + + stdout, stderr = process.communicate(timeout=1) + raise RuntimeError( + f"Server failed to respond to health checks\nStdout: {stdout}\nStderr: {stderr}" + ) diff --git a/agenta-cli/tests/sdk_routing/test_routers.py b/agenta-cli/tests/sdk_routing/test_routers.py new file mode 100644 index 0000000000..68a4c09f29 --- /dev/null +++ b/agenta-cli/tests/sdk_routing/test_routers.py @@ -0,0 +1,102 @@ +import pytest + +from .conftest import * + + +@pytest.mark.parametrize( + "fastapi_server", + [{"app_file": "assets/greetings/main.py"}], + indirect=True, +) +class TestApplicationRoutes: + @pytest.fixture(autouse=True) + def _setup(self, fastapi_server): + self.base_url, _ = fastapi_server + + @pytest.mark.sdk_routing + def test_health_endpoint(self, http_client): + # ACT: Add configuration + response = http_client.get("/health") + + # ASSERT: Verify response + response.raise_for_status() + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + @pytest.mark.sdk_routing + def test_generate_endpoint_success(self, http_client, get_agenta_version): + # ARRANGE: Prepare test data + name = "Aloha" + + # ACT: Add configuration + response = http_client.post("/generate", json={"name": name}) + + # ASSERT: Verify response + response.raise_for_status() + response_data = response.json() + assert response.status_code == 200 + assert response_data["data"] == f"Hello, {name}! (version={get_agenta_version})" + assert type(response_data["tree"]) == dict and isinstance( + response_data.get("tree", {}).get("nodes"), list + ) + + @pytest.mark.sdk_routing + def test_generate_endpoint_authentication_failed(self, http_client): + # ARRANGE: Prepare test data + name = "Aloha" + + # ACT: Add configuration + response = http_client.post( + "/generate", + headers={"Authorization": "ApiKey dummy"}, + json={"name": name}, + ) + + # ASSERT: Verify response + assert response.status_code == 401 + assert response.text == "Unauthorized" + + @pytest.mark.sdk_routing + def test_generate_endpoint_invalid_payload(self, http_client): + # ACT: Add configuration + response = http_client.post("/generate") + + # ASSERT: Verify response + assert response.status_code == 422 + + @pytest.mark.sdk_routing + def test_generate_deployed_endpoint(self, http_client, get_agenta_version): + # ARRANGE: Prepare test data + name = "Aloha" + + # ACT: Add configuration + response = http_client.post("/generate_deployed", json={"name": name}) + + # ASSERT: Verify response + response_data = response.json() + assert response.status_code == 200 + assert response_data["data"] == f"Hello, {name}! (version={get_agenta_version})" + + @pytest.mark.sdk_routing + def test_generate_deployed_endpoint_authentication_failed(self, http_client): + # ARRANGE: Prepare test data + name = "Aloha" + + # ACT: Add configuration + response = http_client.post( + "/generate_deployed", + headers={"Authorization": "ApiKey dummy"}, + json={"name": name}, + ) + + # ASSERT: Verify response + assert response.status_code == 401 + assert response.text == "Unauthorized" + + @pytest.mark.sdk_routing + def test_generate_deployed_endpoint_invalid_payload(self, http_client): + # ACT: Add configuration + response = http_client.post("/generate_deployed") + + # ASSERT: Verify response + assert response.status_code == 422 From 9f6271b0e230efaef38c4378f4aa378930ac078b Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 20 Dec 2024 13:33:44 +0100 Subject: [PATCH 15/27] chore (sdk:tests): add markers for config_manager and sdk_routing test suites --- agenta-cli/tests/pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agenta-cli/tests/pytest.ini b/agenta-cli/tests/pytest.ini index 6f1bcf538a..4ca2447227 100644 --- a/agenta-cli/tests/pytest.ini +++ b/agenta-cli/tests/pytest.ini @@ -8,6 +8,8 @@ markers = asyncio: mark a test as an async test variant_manager: mark testcase as part of the SDK Variant Manager testsuite deployment_manager: mark testcase as part of the SDK Deployment Manager testsuite + config_manager: mark testcase as part of the SDK Config Manager testsuite + sdk_routing: mark testcase as part of the SDK routing testsuite [tool.pytest.ini_options] asyncio_mode = "auto" From 7b7d8f717b5e11de2e97e1bf7271423d39a4f5dc Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 20 Dec 2024 13:34:25 +0100 Subject: [PATCH 16/27] chore (deps): add uvicorn to dev dependencies in agenta-cli --- agenta-cli/poetry.lock | 21 ++++++++++++++++++++- agenta-cli/pyproject.toml | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/agenta-cli/poetry.lock b/agenta-cli/poetry.lock index df3b43f8a6..034118a18f 100644 --- a/agenta-cli/poetry.lock +++ b/agenta-cli/poetry.lock @@ -2445,6 +2445,25 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "uvicorn" +version = "0.34.0" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.9" +files = [ + {file = "uvicorn-0.34.0-py3-none-any.whl", hash = "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4"}, + {file = "uvicorn-0.34.0.tar.gz", hash = "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + [[package]] name = "wcwidth" version = "0.2.13" @@ -2653,4 +2672,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "522bb5aa006e2ab6dd86d1df90e8ef6ce3c0d6a4e9e179e024fece1167a6e451" +content-hash = "b1b8cc5b3129390cbd20bd57013df192d0c1fab88707bfa960c6379f5bbb3771" diff --git a/agenta-cli/pyproject.toml b/agenta-cli/pyproject.toml index 9b332546b2..b1cabc1b78 100644 --- a/agenta-cli/pyproject.toml +++ b/agenta-cli/pyproject.toml @@ -47,6 +47,7 @@ setuptools = "^71.1.0" pytest-asyncio = "^0.24.0" mypy = "^1.13.0" pytest-xdist = "^3.6.1" +uvicorn = "^0.34.0" [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" From e741644f8aa51952b323b92beedb1cb959f5c09b Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 20 Dec 2024 13:34:58 +0100 Subject: [PATCH 17/27] refactor (backend): retrieve api_key from headers properly --- agenta-backend/agenta_backend/routers/app_router.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index d4a54f16b5..7179d73848 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -210,10 +210,11 @@ async def create_app( """ try: if isCloudEE(): - api_key_from_headers = request.headers.get("Authorization") + api_key_from_headers = request.headers.get("Authorization", None) if api_key_from_headers is not None: + api_key = api_key_from_headers.split(" ")[-1] # ["ApiKey", "xxxxx.xxxxxx"] await check_apikey_action_access( - api_key_from_headers, + api_key, request.state.user_id, Permission.CREATE_APPLICATION, ) From b0a194887d7446c5c1af477ac07bd15fdb73261f Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 20 Dec 2024 13:38:23 +0100 Subject: [PATCH 18/27] refactor (sdk:tests): add command to run sdk_routing test suites --- agenta-cli/tests/run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agenta-cli/tests/run_tests.sh b/agenta-cli/tests/run_tests.sh index 57ccf07b30..66494be5bc 100755 --- a/agenta-cli/tests/run_tests.sh +++ b/agenta-cli/tests/run_tests.sh @@ -2,5 +2,5 @@ set -e -OPENAI_API_KEY=sk-xxxxxxxx AGENTA_HOST=http://localhost AGENTA_API_KEY=xxxxx.xxxxxxxxxxxxx pytest management/ -v -m variant_management # run variant_management tests - +OPENAI_API_KEY=sk-xxxxx AGENTA_HOST=http://localhost AGENTA_API_KEY=xxxx.xxxxxxxxxxxxxxx pytest -n 2 -v ./management/* +BASE_URL=http://127.0.0.1 AGENTA_HOST=http://localhost AGENTA_API_KEY=xxxx.xxxxxxxxxxxxxxx pytest -v ./sdk_routing/* From e05a72a4ce855bc86ed88b15f76a258b4b8d887f Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 20 Dec 2024 13:44:00 +0100 Subject: [PATCH 19/27] style (backend): format backend --- agenta-backend/agenta_backend/routers/app_router.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 7179d73848..14ace3bf65 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -212,7 +212,9 @@ async def create_app( if isCloudEE(): api_key_from_headers = request.headers.get("Authorization", None) if api_key_from_headers is not None: - api_key = api_key_from_headers.split(" ")[-1] # ["ApiKey", "xxxxx.xxxxxx"] + api_key = api_key_from_headers.split(" ")[ + -1 + ] # ["ApiKey", "xxxxx.xxxxxx"] await check_apikey_action_access( api_key, request.state.user_id, From afe804ec28e84ca15b9bc28c9d5980df12ea5907 Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 20 Dec 2024 14:02:31 +0100 Subject: [PATCH 20/27] minor refactor (sdk:tests): rename fixture to get executable_python --- agenta-cli/tests/sdk_routing/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agenta-cli/tests/sdk_routing/conftest.py b/agenta-cli/tests/sdk_routing/conftest.py index 0947279335..a76e55acd4 100644 --- a/agenta-cli/tests/sdk_routing/conftest.py +++ b/agenta-cli/tests/sdk_routing/conftest.py @@ -46,7 +46,7 @@ def get_agenta_version(): @pytest.fixture(scope="class") -def execute_python(): +def executable_python(): """ Fixture to provide the current Python executable. """ @@ -94,7 +94,7 @@ def create_application(http_client): @pytest.fixture(scope="class") def fastapi_server( - request, get_port_number, create_application, http_client, execute_python + request, get_port_number, create_application, http_client, executable_python ): """ Run the FastAPI server as a subprocess on a random port and return its base URL. @@ -119,7 +119,7 @@ def fastapi_server( ) command = [ - execute_python, + executable_python, app_file, ] From c862cbdfc18f2fb1ff30984a4169f0da2aa73513 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 23 Dec 2024 09:01:24 +0100 Subject: [PATCH 21/27] refactor (sdk:tests): update bash script to dynamically request for variables required to run cli tests --- agenta-cli/tests/run_tests.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/agenta-cli/tests/run_tests.sh b/agenta-cli/tests/run_tests.sh index 66494be5bc..5e1beb2478 100755 --- a/agenta-cli/tests/run_tests.sh +++ b/agenta-cli/tests/run_tests.sh @@ -2,5 +2,22 @@ set -e -OPENAI_API_KEY=sk-xxxxx AGENTA_HOST=http://localhost AGENTA_API_KEY=xxxx.xxxxxxxxxxxxxxx pytest -n 2 -v ./management/* -BASE_URL=http://127.0.0.1 AGENTA_HOST=http://localhost AGENTA_API_KEY=xxxx.xxxxxxxxxxxxxxx pytest -v ./sdk_routing/* +# Function to prompt for a variable if not already set +check_and_request_var() { + local var_name=$1 + local var_value=${!var_name} # Use indirect variable reference to get value + if [ -z "$var_value" ]; then + read -p "Enter value for $var_name: " var_value + export $var_name="$var_value" + fi +} + +# Check for required variables and prompt if missing +check_and_request_var "OPENAI_API_KEY" +check_and_request_var "AGENTA_HOST" +check_and_request_var "AGENTA_API_KEY" + +# Run test commands +pytest -n 2 -v ./management/* +BASE_URL="http://127.0.0.1" pytest -v ./sdk_routing/* +pytest cli/ From 078cb6457b3bef7264ca89183505ef38bfba037b Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 23 Dec 2024 09:01:45 +0100 Subject: [PATCH 22/27] refactor (sdk:tests): add marker for cli_testing --- agenta-cli/tests/pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/agenta-cli/tests/pytest.ini b/agenta-cli/tests/pytest.ini index 4ca2447227..f91e274ef0 100644 --- a/agenta-cli/tests/pytest.ini +++ b/agenta-cli/tests/pytest.ini @@ -10,6 +10,7 @@ markers = deployment_manager: mark testcase as part of the SDK Deployment Manager testsuite config_manager: mark testcase as part of the SDK Config Manager testsuite sdk_routing: mark testcase as part of the SDK routing testsuite + cli_testing: mark testcase as part of the CLI testing testsuite [tool.pytest.ini_options] asyncio_mode = "auto" From 8e8f9fab67d69aaf4c2470bd76bf2718d42e62b5 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 23 Dec 2024 09:04:40 +0100 Subject: [PATCH 23/27] feat (sdk:tests): add required fixtures required to run cli commands in python --- agenta-cli/tests/cli/__init__.py | 0 agenta-cli/tests/cli/fixtures.py | 139 +++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 agenta-cli/tests/cli/__init__.py create mode 100644 agenta-cli/tests/cli/fixtures.py diff --git a/agenta-cli/tests/cli/__init__.py b/agenta-cli/tests/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/cli/fixtures.py b/agenta-cli/tests/cli/fixtures.py new file mode 100644 index 0000000000..c4b74f0f7a --- /dev/null +++ b/agenta-cli/tests/cli/fixtures.py @@ -0,0 +1,139 @@ +import os +import toml +import shutil +import pexpect +from typing import List +from pathlib import Path + +import httpx +import pytest + + +AGENTA_API_KEY = os.environ.get("AGENTA_API_KEY") +AGENTA_HOST = os.environ.get("AGENTA_HOST", "http://localhost") +API_BASE_URL = f"{AGENTA_HOST}/api/" + + +def agenta_executable(): + """ + Fixture to provide the current Agenta executable. + """ + + executable_path = shutil.which("agenta") + return executable_path + + +def get_assets_folder(example_folder: str): + parent_folder = Path(__file__).parent + assets_folder = Path(f"{parent_folder}/assets/{example_folder}/") + return assets_folder + + +def retrieve_app_id_from_path_and_remove_application(assets_dir): + """ + Retrieve the app_id from the config.toml file and remove the application + """ + + config_file = assets_dir / "config.toml" + if config_file.exists(): + config = toml.load(config_file) + app_id = config["app_id"] + + # Delete application + response = httpx.delete( + f"{API_BASE_URL}apps/{app_id}", + headers={"Authorization": f"ApiKey {AGENTA_API_KEY}"}, + timeout=httpx.Timeout(timeout=6, read=None, write=5), + ) + response.raise_for_status() + + +def cleanup_created_test_files(assets_dir: Path): + """ + Clean up the test directory + """ + + if assets_dir.exists(): + # Remove .agentaignore file if it exists + agentaignore_file = assets_dir / ".agentaignore" + if agentaignore_file.exists(): + agentaignore_file.unlink() + print(f"Removed: {agentaignore_file}") + + # Remove config.toml file if it exists + config_file = assets_dir / "config.toml" + if config_file.exists(): + config_file.unlink() + print(f"Removed: {config_file}") + + +@pytest.fixture +def cleanup_application_and_files(): + """ + Factory fixture to ensure the application and test files are cleaned up after each test class, with support for dynamic folder input. + """ + + def _cleanup_application_and_files(folder_name): + assets_dir = get_assets_folder(folder_name) + retrieve_app_id_from_path_and_remove_application(assets_dir) + cleanup_created_test_files(assets_dir) + + yield "ok" + + return _cleanup_application_and_files + + +def run_agenta_init(user_inputs: List[str], example_folder: str): + """ + Run agenta init in assets/greetings directory with the given inputs using pexpect + """ + + # Ensure the directory exists + assets_dir = get_assets_folder(example_folder) + os.chdir(assets_dir) + + # Construct the command with the provided inputs + executable_path = agenta_executable() + child = pexpect.spawn( + command=f"{executable_path} init", encoding="utf-8", timeout=5 + ) + + for input in user_inputs: + child.send(input) + + # Give it time to finish + child.wait() + + # Capture the final output after the process finishes + output = child.read() + child.close() + + yield {"output": str(output).strip(" "), "exit_status": child.exitstatus} + + +def run_variant_serve(user_inputs: List[str], example_folder: str): + """ + Run agenta variant serve in assets/greetings directory with the given inputs using pexpect + """ + + # Ensure the directory exists + assets_dir = get_assets_folder(example_folder) + os.chdir(assets_dir) + + # Construct the command with the provided inputs + executable_path = agenta_executable() + child = pexpect.spawn( + command=f"{executable_path} variant serve app.py", encoding="utf-8", timeout=5 + ) + + for input in user_inputs: + child.send(input) + + # Give it time to finish + child.wait() + + # Capture the final output after the process finishes + output = child.read() + child.close() + + yield {"output": str(output).strip(" "), "exit_status": child.exitstatus} From d65f0fb715344375c645501576be0c5855867cfb Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 23 Dec 2024 09:05:05 +0100 Subject: [PATCH 24/27] feat (sdk:tests): add assets required to run the cli tests --- agenta-cli/tests/cli/assets/__init__.py | 0 agenta-cli/tests/cli/assets/greetings/app.py | 30 +++++++++++++++++++ .../cli/assets/greetings/requirements.txt | 0 .../tests/cli/assets/salutations/app.py | 30 +++++++++++++++++++ .../cli/assets/salutations/requirements.txt | 0 5 files changed, 60 insertions(+) create mode 100644 agenta-cli/tests/cli/assets/__init__.py create mode 100644 agenta-cli/tests/cli/assets/greetings/app.py create mode 100644 agenta-cli/tests/cli/assets/greetings/requirements.txt create mode 100644 agenta-cli/tests/cli/assets/salutations/app.py create mode 100644 agenta-cli/tests/cli/assets/salutations/requirements.txt diff --git a/agenta-cli/tests/cli/assets/__init__.py b/agenta-cli/tests/cli/assets/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/cli/assets/greetings/app.py b/agenta-cli/tests/cli/assets/greetings/app.py new file mode 100644 index 0000000000..490179937f --- /dev/null +++ b/agenta-cli/tests/cli/assets/greetings/app.py @@ -0,0 +1,30 @@ +import os +from importlib.metadata import version + +import agenta as ag + +ag.init(host=os.environ.get("AGENTA_HOST", "http://localhost")) + +ag.config.default( + flag=ag.BinaryParam(value=True), +) + + +class CustomException(Exception): + def __init__(self, message): + self.message = message + self.status_code = 401 + + def __str__(self): + return self.message + + +@ag.entrypoint +@ag.instrument(spankind="workflow") +async def greetings(name: str): + message = "Hello, World!" + + if ag.config.flag: + message = f"Hello, {name}! (version={version('agenta')})" + + return message diff --git a/agenta-cli/tests/cli/assets/greetings/requirements.txt b/agenta-cli/tests/cli/assets/greetings/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/cli/assets/salutations/app.py b/agenta-cli/tests/cli/assets/salutations/app.py new file mode 100644 index 0000000000..490179937f --- /dev/null +++ b/agenta-cli/tests/cli/assets/salutations/app.py @@ -0,0 +1,30 @@ +import os +from importlib.metadata import version + +import agenta as ag + +ag.init(host=os.environ.get("AGENTA_HOST", "http://localhost")) + +ag.config.default( + flag=ag.BinaryParam(value=True), +) + + +class CustomException(Exception): + def __init__(self, message): + self.message = message + self.status_code = 401 + + def __str__(self): + return self.message + + +@ag.entrypoint +@ag.instrument(spankind="workflow") +async def greetings(name: str): + message = "Hello, World!" + + if ag.config.flag: + message = f"Hello, {name}! (version={version('agenta')})" + + return message diff --git a/agenta-cli/tests/cli/assets/salutations/requirements.txt b/agenta-cli/tests/cli/assets/salutations/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 From 97b1bb2468d68c7cefa9d329c33487287835ec88 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 23 Dec 2024 09:07:05 +0100 Subject: [PATCH 25/27] feat (sdk:tests): create TestAgentaInitCommand and TestAgentaVariantServeCommand test suite and test cases for each --- agenta-cli/tests/cli/test_init.py | 120 +++++++++++++++++++ agenta-cli/tests/cli/test_variant_serve.py | 127 +++++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 agenta-cli/tests/cli/test_init.py create mode 100644 agenta-cli/tests/cli/test_variant_serve.py diff --git a/agenta-cli/tests/cli/test_init.py b/agenta-cli/tests/cli/test_init.py new file mode 100644 index 0000000000..7662f7617c --- /dev/null +++ b/agenta-cli/tests/cli/test_init.py @@ -0,0 +1,120 @@ +import os +import toml +import uuid +from pathlib import Path + +import pytest + +from .fixtures import * + + +class TestAgentaInitCommand: + @pytest.fixture(autouse=True) + def _setup(self): + self.asset_example_folder = "greetings" + self.assets_folder = str( + get_assets_folder(example_folder=self.asset_example_folder) + ) + + @pytest.mark.cli_testing + def test_cloud_blank_app_success(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "n" + provide_api_key = os.environ.get("AGENTA_API_KEY") + + # ACT: Add configuration + inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + provide_api_key, + ] + result = run_agenta_init(inputs, self.asset_example_folder) + cli_output = next(result) + + # ASSERT: Verify response + assert cli_output["exit_status"] == 0 + assert "App initialized successfully" in cli_output["output"] + + config_path = Path(f"{self.assets_folder}/config.toml") + assert config_path.exists() + + config = toml.load(config_path) + assert config["app_id"] is not None + assert config["app_name"] == app_name + assert config["backend_host"] == os.environ.get("AGENTA_HOST") + + agentaignore_path = Path(f"{self.assets_folder}/.agentaignore") + assert agentaignore_path.exists() + + # CLEANUP: Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files(self.asset_example_folder) + assert next(cleanup) == "ok" + + @pytest.mark.cli_testing + def test_cloud_blank_app_already_exists(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "N" + provide_api_key = os.environ.get("AGENTA_API_KEY") + + # ACT: Add configuration + inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + provide_api_key, + ] + result_1 = run_agenta_init( + inputs, self.asset_example_folder + ) # create app the first time + _ = next(result_1) + result_2 = run_agenta_init( + inputs, self.asset_example_folder + ) # tries to create app with the same name + cli_output = next(result_2) + + # ASSERT: Verify response + assert cli_output["exit_status"] == 1 + assert "App with the same name already exists" in cli_output["output"] + + # CLEANUP: Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files(self.asset_example_folder) + assert next(cleanup) == "ok" + + @pytest.mark.cli_testing + def test_cloud_blank_app_with_invalid_credential(self): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + provide_api_key = "dummy_key\n" + environ_keys = os.environ.copy() + os.environ["AGENTA_API_KEY"] = "dummy_key" + + # ACT: Add configuration + inputs = [ + f"{app_name}\n", + where_to_run_agenta, + provide_api_key, + ] + result = run_agenta_init(inputs, self.asset_example_folder) + cli_output = next(result) + + # ASSERT: Verify response + assert ( + cli_output["exit_status"] == 1 + ) # Ensure non-zero exit status indicating failure + assert "Unauthorized" in cli_output["output"] + + config_path = Path(f"{self.assets_folder}/config.toml") + assert not config_path.exists() + + agentaignore_path = Path(f"{self.assets_folder}/.agentaignore") + assert not agentaignore_path.exists() + + # CLEANUP: Reset environment variables + os.environ.clear() + os.environ.update(environ_keys) diff --git a/agenta-cli/tests/cli/test_variant_serve.py b/agenta-cli/tests/cli/test_variant_serve.py new file mode 100644 index 0000000000..e88b251953 --- /dev/null +++ b/agenta-cli/tests/cli/test_variant_serve.py @@ -0,0 +1,127 @@ +import os +import toml +import uuid +from pathlib import Path + +import pytest + +from .fixtures import * + + +class TestAgentaVariantServeCommand: + @pytest.fixture(autouse=True) + def _setup(self): + self.asset_example_folder = "salutations" + self.assets_folder = str( + get_assets_folder(example_folder=self.asset_example_folder) + ) + + @pytest.mark.cli_testing + def test_variant_serve_success(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "n" + provide_api_key = os.environ.get("AGENTA_API_KEY") + + # ACT: Add configuration + init_inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + provide_api_key, + ] + result = run_agenta_init(init_inputs, self.asset_example_folder) + cli_output = next(result) + + if cli_output["exit_status"] == 1: + pytest.fail("Creating an app from the CLI failed.") + + serve_inputs = [] + result = run_variant_serve(serve_inputs, self.asset_example_folder) + cli_serve_output = next(result) + + # ASSERT: Verify response + assert cli_serve_output["exit_status"] == 0 + assert "Adding app.default to server..." in cli_serve_output["output"] + assert "Waiting for the variant to be ready" in cli_serve_output["output"] + assert "Variant added successfully!" in cli_serve_output["output"] + assert "Congratulations!" in cli_serve_output["output"] + assert ( + "Your app has been deployed locally as an API." + in cli_serve_output["output"] + ) + assert "Read the API documentation." in cli_serve_output["output"] + assert ( + "Start experimenting with your app in the playground." + in cli_serve_output["output"] + ) + + config_path = Path(f"{self.assets_folder}/config.toml") + assert config_path.exists() + + config = toml.load(config_path) + assert config["app_id"] is not None + assert config["app_name"] == app_name + assert config["backend_host"] == os.environ.get("AGENTA_HOST") + + agentaignore_path = Path(f"{self.assets_folder}/.agentaignore") + assert agentaignore_path.exists() + + # CLEANUP: Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files(self.asset_example_folder) + assert next(cleanup) == "ok" + + @pytest.mark.cli_testing + def test_variant_reserve_success(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "n" + provide_api_key = os.environ.get("AGENTA_API_KEY") + + # ACT: Add configuration + init_inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + provide_api_key, + ] + result = run_agenta_init(init_inputs, self.asset_example_folder) + cli_output = next(result) + + if cli_output["exit_status"] == 1: + pytest.fail("Creating an app from the CLI failed.") + + serve_inputs = [] + serve_result = run_variant_serve(serve_inputs, self.asset_example_folder) + cli_serve_output = next(serve_result) + + if cli_serve_output["exit_status"] == 1: + pytest.fail("Serving a variant from the CLI failed.") + + reserve_inputs = ["y\n"] + reserve_result = run_variant_serve(reserve_inputs, self.asset_example_folder) + cli_reserve_output = next(reserve_result) + + # ASSERT: Verify response + assert cli_reserve_output["exit_status"] == 0 + assert ( + f"Variant app.default for App {app_name} updated successfully" + in cli_reserve_output["output"] + ) + + config_path = Path(f"{self.assets_folder}/config.toml") + assert config_path.exists() + + config = toml.load(config_path) + assert config["app_id"] is not None + assert config["app_name"] == app_name + assert config["backend_host"] == os.environ.get("AGENTA_HOST") + + agentaignore_path = Path(f"{self.assets_folder}/.agentaignore") + assert agentaignore_path.exists() + + # CLEANUP: Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files(self.asset_example_folder) + assert next(cleanup) == "ok" From e492d7022e58b8e56ecd59b48d95737730e178e9 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 25 Dec 2024 12:19:54 +0100 Subject: [PATCH 26/27] refactor (sdk:tests): add and integrated fixtures for programmatic access to sdk (management, sdk routing & cli commands) --- agenta-cli/.env.example | 5 + agenta-cli/poetry.lock | 114 +++++++++++++++++- agenta-cli/pyproject.toml | 3 + .../tests/cli/assets/greetings/.env.example | 0 .../tests/cli/assets/salutations/.env.example | 0 agenta-cli/tests/cli/fixtures.py | 29 +++-- agenta-cli/tests/cli/test_init.py | 29 +++-- agenta-cli/tests/cli/test_variant_serve.py | 27 +++-- agenta-cli/tests/conftest.py | 97 +++++++++++++++ agenta-cli/tests/management/conftest.py | 8 +- agenta-cli/tests/run_tests.sh | 7 +- agenta-cli/tests/sdk_routing/conftest.py | 8 +- agenta-cli/tests/sdk_routing/test_routers.py | 2 +- 13 files changed, 281 insertions(+), 48 deletions(-) create mode 100644 agenta-cli/.env.example create mode 100644 agenta-cli/tests/cli/assets/greetings/.env.example create mode 100644 agenta-cli/tests/cli/assets/salutations/.env.example create mode 100644 agenta-cli/tests/conftest.py diff --git a/agenta-cli/.env.example b/agenta-cli/.env.example new file mode 100644 index 0000000000..b26165b3b2 --- /dev/null +++ b/agenta-cli/.env.example @@ -0,0 +1,5 @@ +# rename [environ] to the environment you're working/testing on +# for local, rename it to local --- .env.local +_SECRET_KEY=xxxxxxx +AWS_PROFILE_NAME=xxxxxxxxxx +AGENTA_AUTH_KEY_SECRET_ARN=xxxxxxxxxxxxx diff --git a/agenta-cli/poetry.lock b/agenta-cli/poetry.lock index 034118a18f..7ad23f8810 100644 --- a/agenta-cli/poetry.lock +++ b/agenta-cli/poetry.lock @@ -197,6 +197,47 @@ files = [ {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] +[[package]] +name = "boto3" +version = "1.35.87" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.35.87-py3-none-any.whl", hash = "sha256:588ab05e2771c50fca5c242be14e7a25200ffd3dd95c45950ce40993473864c7"}, + {file = "boto3-1.35.87.tar.gz", hash = "sha256:341c58602889078a4a25dc4331b832b5b600a33acd73471d2532c6f01b16fbb4"}, +] + +[package.dependencies] +botocore = ">=1.35.87,<1.36.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.35.87" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.35.87-py3-none-any.whl", hash = "sha256:81cf84f12030d9ab3829484b04765d5641697ec53c2ac2b3987a99eefe501692"}, + {file = "botocore-1.35.87.tar.gz", hash = "sha256:3062d073ce4170a994099270f469864169dc1a1b8b3d4a21c14ce0ae995e0f89"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = [ + {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, + {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}, +] + +[package.extras] +crt = ["awscrt (==0.22.0)"] + [[package]] name = "cachetools" version = "5.5.0" @@ -934,6 +975,17 @@ files = [ {file = "jiter-0.7.1.tar.gz", hash = "sha256:448cf4f74f7363c34cdef26214da527e8eeffd88ba06d0b80b485ad0667baf5d"}, ] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "jsonschema" version = "4.23.0" @@ -1410,6 +1462,20 @@ files = [ {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] +[[package]] +name = "pexpect" +version = "4.9.0" +description = "Pexpect allows easy control of interactive console applications." +optional = false +python-versions = "*" +files = [ + {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, + {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, +] + +[package.dependencies] +ptyprocess = ">=0.5" + [[package]] name = "pluggy" version = "1.5.0" @@ -1589,6 +1655,17 @@ files = [ {file = "protobuf-5.28.3.tar.gz", hash = "sha256:64badbc49180a5e401f373f9ce7ab1d18b63f7dd4a9cdc43c92b9f0b481cef7b"}, ] +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +optional = false +python-versions = "*" +files = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] + [[package]] name = "pydantic" version = "2.9.2" @@ -1604,8 +1681,8 @@ files = [ annotated-types = ">=0.6.0" pydantic-core = "2.23.4" typing-extensions = [ - {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, {version = ">=4.6.1", markers = "python_version < \"3.13\""}, + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, ] [package.extras] @@ -2142,6 +2219,23 @@ files = [ {file = "rpds_py-0.21.0.tar.gz", hash = "sha256:ed6378c9d66d0de903763e7706383d60c33829581f0adff47b6535f1802fa6db"}, ] +[[package]] +name = "s3transfer" +version = "0.10.4" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.8" +files = [ + {file = "s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e"}, + {file = "s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + [[package]] name = "setuptools" version = "71.1.0" @@ -2428,6 +2522,22 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "urllib3" +version = "1.26.20" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e"}, + {file = "urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "urllib3" version = "2.2.3" @@ -2672,4 +2782,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "b1b8cc5b3129390cbd20bd57013df192d0c1fab88707bfa960c6379f5bbb3771" +content-hash = "63f1b85746ca4ca56ea55d70f4bf1028b96aa7bb19d4d4be80609e6fbd6df2c5" diff --git a/agenta-cli/pyproject.toml b/agenta-cli/pyproject.toml index b1cabc1b78..ef1cac3233 100644 --- a/agenta-cli/pyproject.toml +++ b/agenta-cli/pyproject.toml @@ -48,6 +48,9 @@ pytest-asyncio = "^0.24.0" mypy = "^1.13.0" pytest-xdist = "^3.6.1" uvicorn = "^0.34.0" +requests = "^2.32.3" +pexpect = "^4.9.0" +boto3 = "^1.35.87" [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" diff --git a/agenta-cli/tests/cli/assets/greetings/.env.example b/agenta-cli/tests/cli/assets/greetings/.env.example new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/cli/assets/salutations/.env.example b/agenta-cli/tests/cli/assets/salutations/.env.example new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agenta-cli/tests/cli/fixtures.py b/agenta-cli/tests/cli/fixtures.py index c4b74f0f7a..f87ac3bd2a 100644 --- a/agenta-cli/tests/cli/fixtures.py +++ b/agenta-cli/tests/cli/fixtures.py @@ -1,3 +1,4 @@ +import sys import os import toml import shutil @@ -8,10 +9,7 @@ import httpx import pytest - -AGENTA_API_KEY = os.environ.get("AGENTA_API_KEY") -AGENTA_HOST = os.environ.get("AGENTA_HOST", "http://localhost") -API_BASE_URL = f"{AGENTA_HOST}/api/" +from tests.conftest import get_admin_user_credentials, API_BASE_URL def agenta_executable(): @@ -29,7 +27,9 @@ def get_assets_folder(example_folder: str): return assets_folder -def retrieve_app_id_from_path_and_remove_application(assets_dir): +def retrieve_app_id_from_path_and_remove_application( + assets_dir: Path, access_token: str +): """ Retrieve the app_id from the config.toml file and remove the application """ @@ -42,7 +42,7 @@ def retrieve_app_id_from_path_and_remove_application(assets_dir): # Delete application response = httpx.delete( f"{API_BASE_URL}apps/{app_id}", - headers={"Authorization": f"ApiKey {AGENTA_API_KEY}"}, + headers={"Authorization": f"ApiKey {access_token}"}, timeout=httpx.Timeout(timeout=6, read=None, write=5), ) response.raise_for_status() @@ -67,15 +67,24 @@ def cleanup_created_test_files(assets_dir: Path): print(f"Removed: {config_file}") +def get_programmatic_access_credentials(): + """ + Retrieve the admin user's credentials for API testing. + """ + + user_credentials = get_admin_user_credentials() + return str(user_credentials).strip("ApiKey ") + + @pytest.fixture def cleanup_application_and_files(): """ Factory fixture to ensure the application and test files are cleaned up after each test class, with support for dynamic folder input. """ - def _cleanup_application_and_files(folder_name): + def _cleanup_application_and_files(folder_name, access_token): assets_dir = get_assets_folder(folder_name) - retrieve_app_id_from_path_and_remove_application(assets_dir) + retrieve_app_id_from_path_and_remove_application(assets_dir, access_token) cleanup_created_test_files(assets_dir) yield "ok" @@ -95,7 +104,7 @@ def run_agenta_init(user_inputs: List[str], example_folder: str): # Construct the command with the provided inputs executable_path = agenta_executable() child = pexpect.spawn( - command=f"{executable_path} init", encoding="utf-8", timeout=5 + command=f"{executable_path} init", encoding="utf-8", timeout=10 ) for input in user_inputs: @@ -123,7 +132,7 @@ def run_variant_serve(user_inputs: List[str], example_folder: str): # Construct the command with the provided inputs executable_path = agenta_executable() child = pexpect.spawn( - command=f"{executable_path} variant serve app.py", encoding="utf-8", timeout=5 + command=f"{executable_path} variant serve app.py", encoding="utf-8", timeout=10 ) for input in user_inputs: diff --git a/agenta-cli/tests/cli/test_init.py b/agenta-cli/tests/cli/test_init.py index 7662f7617c..195595540b 100644 --- a/agenta-cli/tests/cli/test_init.py +++ b/agenta-cli/tests/cli/test_init.py @@ -9,12 +9,13 @@ class TestAgentaInitCommand: - @pytest.fixture(autouse=True) - def _setup(self): - self.asset_example_folder = "greetings" - self.assets_folder = str( - get_assets_folder(example_folder=self.asset_example_folder) + @pytest.fixture(scope="class", autouse=True) + def _setup(self, request): + request.cls.asset_example_folder = "greetings" + request.cls.assets_folder = str( + get_assets_folder(example_folder=request.cls.asset_example_folder) ) + request.cls.api_key = get_programmatic_access_credentials() @pytest.mark.cli_testing def test_cloud_blank_app_success(self, cleanup_application_and_files): @@ -22,14 +23,14 @@ def test_cloud_blank_app_success(self, cleanup_application_and_files): app_name = f"greetings_{uuid.uuid4().hex[:6]}" where_to_run_agenta = "\n" use_this_key = "n" - provide_api_key = os.environ.get("AGENTA_API_KEY") + provide_api_key = self.api_key # ACT: Add configuration inputs = [ f"{app_name}\n", where_to_run_agenta, use_this_key, - provide_api_key, + f"{provide_api_key}\n", ] result = run_agenta_init(inputs, self.asset_example_folder) cli_output = next(result) @@ -50,7 +51,9 @@ def test_cloud_blank_app_success(self, cleanup_application_and_files): assert agentaignore_path.exists() # CLEANUP: Remove application from backend, db and local filesystem - cleanup = cleanup_application_and_files(self.asset_example_folder) + cleanup = cleanup_application_and_files( + self.asset_example_folder, provide_api_key + ) assert next(cleanup) == "ok" @pytest.mark.cli_testing @@ -59,14 +62,14 @@ def test_cloud_blank_app_already_exists(self, cleanup_application_and_files): app_name = f"greetings_{uuid.uuid4().hex[:6]}" where_to_run_agenta = "\n" use_this_key = "N" - provide_api_key = os.environ.get("AGENTA_API_KEY") + provide_api_key = self.api_key # ACT: Add configuration inputs = [ f"{app_name}\n", where_to_run_agenta, use_this_key, - provide_api_key, + f"{provide_api_key}\n", ] result_1 = run_agenta_init( inputs, self.asset_example_folder @@ -82,7 +85,9 @@ def test_cloud_blank_app_already_exists(self, cleanup_application_and_files): assert "App with the same name already exists" in cli_output["output"] # CLEANUP: Remove application from backend, db and local filesystem - cleanup = cleanup_application_and_files(self.asset_example_folder) + cleanup = cleanup_application_and_files( + self.asset_example_folder, provide_api_key + ) assert next(cleanup) == "ok" @pytest.mark.cli_testing @@ -98,7 +103,7 @@ def test_cloud_blank_app_with_invalid_credential(self): inputs = [ f"{app_name}\n", where_to_run_agenta, - provide_api_key, + f"{provide_api_key}\n", ] result = run_agenta_init(inputs, self.asset_example_folder) cli_output = next(result) diff --git a/agenta-cli/tests/cli/test_variant_serve.py b/agenta-cli/tests/cli/test_variant_serve.py index e88b251953..df346f905f 100644 --- a/agenta-cli/tests/cli/test_variant_serve.py +++ b/agenta-cli/tests/cli/test_variant_serve.py @@ -9,12 +9,13 @@ class TestAgentaVariantServeCommand: - @pytest.fixture(autouse=True) - def _setup(self): - self.asset_example_folder = "salutations" - self.assets_folder = str( - get_assets_folder(example_folder=self.asset_example_folder) + @pytest.fixture(scope="class", autouse=True) + def _setup(self, request): + request.cls.asset_example_folder = "salutations" + request.cls.assets_folder = str( + get_assets_folder(example_folder=request.cls.asset_example_folder) ) + request.cls.api_key = get_programmatic_access_credentials() @pytest.mark.cli_testing def test_variant_serve_success(self, cleanup_application_and_files): @@ -22,14 +23,14 @@ def test_variant_serve_success(self, cleanup_application_and_files): app_name = f"greetings_{uuid.uuid4().hex[:6]}" where_to_run_agenta = "\n" use_this_key = "n" - provide_api_key = os.environ.get("AGENTA_API_KEY") + provide_api_key = self.api_key # ACT: Add configuration init_inputs = [ f"{app_name}\n", where_to_run_agenta, use_this_key, - provide_api_key, + f"{provide_api_key}\n", ] result = run_agenta_init(init_inputs, self.asset_example_folder) cli_output = next(result) @@ -69,7 +70,9 @@ def test_variant_serve_success(self, cleanup_application_and_files): assert agentaignore_path.exists() # CLEANUP: Remove application from backend, db and local filesystem - cleanup = cleanup_application_and_files(self.asset_example_folder) + cleanup = cleanup_application_and_files( + self.asset_example_folder, provide_api_key + ) assert next(cleanup) == "ok" @pytest.mark.cli_testing @@ -78,14 +81,14 @@ def test_variant_reserve_success(self, cleanup_application_and_files): app_name = f"greetings_{uuid.uuid4().hex[:6]}" where_to_run_agenta = "\n" use_this_key = "n" - provide_api_key = os.environ.get("AGENTA_API_KEY") + provide_api_key = self.api_key # ACT: Add configuration init_inputs = [ f"{app_name}\n", where_to_run_agenta, use_this_key, - provide_api_key, + f"{provide_api_key}\n", ] result = run_agenta_init(init_inputs, self.asset_example_folder) cli_output = next(result) @@ -123,5 +126,7 @@ def test_variant_reserve_success(self, cleanup_application_and_files): assert agentaignore_path.exists() # CLEANUP: Remove application from backend, db and local filesystem - cleanup = cleanup_application_and_files(self.asset_example_folder) + cleanup = cleanup_application_and_files( + self.asset_example_folder, provide_api_key + ) assert next(cleanup) == "ok" diff --git a/agenta-cli/tests/conftest.py b/agenta-cli/tests/conftest.py new file mode 100644 index 0000000000..5afbc6e069 --- /dev/null +++ b/agenta-cli/tests/conftest.py @@ -0,0 +1,97 @@ +import os +import uuid +import logging +from json import loads +from traceback import format_exc +from typing import Optional, Any + +import httpx +import boto3 +from dotenv import load_dotenv + + +# Load environment variables +load_dotenv("../.env") + +# Set global variables +AGENTA_SECRET_KEY = os.environ.get("_SECRET_KEY", "AGENTA_AUTH_KEY") +AGENTA_AWS_PROFILE_NAME = os.environ.get("AWS_PROFILE_NAME", "staging") +AGENTA_SECRET_ARN = os.environ.get("AGENTA_AUTH_KEY_SECRET_ARN", None) +AGENTA_HOST = os.environ.get("AGENTA_HOST", "http://localhost") +API_BASE_URL = f"{AGENTA_HOST}/api/" + +session = boto3.Session(profile_name=AGENTA_AWS_PROFILE_NAME) +sm_client = session.client("secretsmanager") + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def fetch_secret( + secret_arn: str, + secret_key: Optional[str] = None, +) -> Optional[Any]: + try: + response = sm_client.get_secret_value(SecretId=secret_arn) + + secrets = None + + if "SecretString" in response: + secrets = response["SecretString"] + elif "SecretBinary" in response: + secrets = response["SecretBinary"].decode("utf-8") + + if not secrets: + return None + + secrets = loads(secrets) + + if not secret_key: + return secrets + + secret = None + + if secret_key: + secret = secrets.get(secret_key, None) + + return secret + + except: # pylint: disable=bare-except + logger.error("Failed to fetch secrets with: %s", format_exc()) + return None + + +def http_client(): + access_key = fetch_secret( + secret_arn=AGENTA_SECRET_ARN, secret_key=AGENTA_SECRET_KEY + ) + client = httpx.Client( + base_url=API_BASE_URL, + timeout=httpx.Timeout(timeout=6, read=None, write=5), + headers={"Authorization": f"Access {access_key}"}, + ) + return client + + +def create_programmatic_user(): + client = http_client() + randomness = uuid.uuid4().hex[:8] + response = client.post( + "admin/accounts", + json={ + "user": { + "name": f"Test_{randomness}", + "email": f"test_{randomness}@agenta.ai", + }, + "scope": {"name": "tests"}, + }, + ) + response.raise_for_status() + return response.json() + + +def get_admin_user_credentials(): + programmatic_user = create_programmatic_user() + scopes = programmatic_user.get("scopes", []) + credentials = scopes[0].get("credentials", None) + return credentials diff --git a/agenta-cli/tests/management/conftest.py b/agenta-cli/tests/management/conftest.py index 2403258801..980bde41d6 100644 --- a/agenta-cli/tests/management/conftest.py +++ b/agenta-cli/tests/management/conftest.py @@ -7,11 +7,10 @@ import pytest_asyncio from pytest_asyncio import is_async_test +from tests.conftest import get_admin_user_credentials, API_BASE_URL + OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", None) -AGENTA_API_KEY = os.environ.get("AGENTA_API_KEY", None) -AGENTA_HOST = os.environ.get("AGENTA_HOST", "http://localhost") -API_BASE_URL = f"{AGENTA_HOST}/api/" def pytest_collection_modifyitems(items): @@ -33,11 +32,12 @@ async def http_client(): Create an HTTP client for API testing. """ + programmatic_access = get_admin_user_credentials() async with httpx.AsyncClient( base_url=API_BASE_URL, timeout=httpx.Timeout(timeout=6, read=None, write=5), headers={ - "Authorization": f"ApiKey {AGENTA_API_KEY}", + "Authorization": f"{programmatic_access}", "Content-Type": "application/json", }, ) as client: diff --git a/agenta-cli/tests/run_tests.sh b/agenta-cli/tests/run_tests.sh index 5e1beb2478..fc2987bf60 100755 --- a/agenta-cli/tests/run_tests.sh +++ b/agenta-cli/tests/run_tests.sh @@ -9,15 +9,14 @@ check_and_request_var() { if [ -z "$var_value" ]; then read -p "Enter value for $var_name: " var_value export $var_name="$var_value" + export BASE_URL="http://127.0.0.1" # required for sdk routing test suites fi } # Check for required variables and prompt if missing check_and_request_var "OPENAI_API_KEY" check_and_request_var "AGENTA_HOST" -check_and_request_var "AGENTA_API_KEY" # Run test commands -pytest -n 2 -v ./management/* -BASE_URL="http://127.0.0.1" pytest -v ./sdk_routing/* -pytest cli/ +pytest -n 2 -v ./management/* ./sdk_routing/* +pytest -v ./cli/* diff --git a/agenta-cli/tests/sdk_routing/conftest.py b/agenta-cli/tests/sdk_routing/conftest.py index a76e55acd4..33c7f06331 100644 --- a/agenta-cli/tests/sdk_routing/conftest.py +++ b/agenta-cli/tests/sdk_routing/conftest.py @@ -12,11 +12,10 @@ import httpx import pytest +from tests.conftest import get_admin_user_credentials, API_BASE_URL + BASE_URL = os.getenv("BASE_URL", "http://127.0.0.1") -AGENTA_HOST = os.getenv("AGENTA_HOST", "http://localhost") -API_BASE_URL = f"{AGENTA_HOST}/api/" -API_KEY = os.getenv("AGENTA_API_KEY") def get_free_port(start=8001, end=8999, max_attempts=100): @@ -67,11 +66,12 @@ def http_client(get_port_number): Create an HTTP client for API testing. """ + programmatic_access = get_admin_user_credentials() with httpx.Client( base_url=f"{BASE_URL}:{get_port_number}", timeout=httpx.Timeout(timeout=6, read=None, write=5), headers={ - "Authorization": f"ApiKey {API_KEY}", + "Authorization": f"{programmatic_access}", "Content-Type": "application/json", }, ) as client: diff --git a/agenta-cli/tests/sdk_routing/test_routers.py b/agenta-cli/tests/sdk_routing/test_routers.py index 68a4c09f29..7484337f21 100644 --- a/agenta-cli/tests/sdk_routing/test_routers.py +++ b/agenta-cli/tests/sdk_routing/test_routers.py @@ -5,7 +5,7 @@ @pytest.mark.parametrize( "fastapi_server", - [{"app_file": "assets/greetings/main.py"}], + [{"app_file": "./assets/greetings/main.py"}], indirect=True, ) class TestApplicationRoutes: From 9336875c73ee13cd9c4c9c0abcd187ae6a78cb89 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Dec 2024 12:04:26 +0100 Subject: [PATCH 27/27] refactor (sdk:tests): add test case for grumpy path --- serve variant with no env file --- agenta-cli/tests/cli/test_variant_serve.py | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/agenta-cli/tests/cli/test_variant_serve.py b/agenta-cli/tests/cli/test_variant_serve.py index df346f905f..3581920644 100644 --- a/agenta-cli/tests/cli/test_variant_serve.py +++ b/agenta-cli/tests/cli/test_variant_serve.py @@ -130,3 +130,45 @@ def test_variant_reserve_success(self, cleanup_application_and_files): self.asset_example_folder, provide_api_key ) assert next(cleanup) == "ok" + + @pytest.mark.cli_testing + def test_variant_serve_with_no_env_file(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "n" + provide_api_key = self.api_key + if Path(f"{self.assets_folder}/.env").exists(): + os.rename(f"{self.assets_folder}/.env", f"{self.assets_folder}/.env.dummy") + + # ACT: Add configuration + init_inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + f"{provide_api_key}\n", + ] + result = run_agenta_init(init_inputs, self.asset_example_folder) + cli_output = next(result) + + if cli_output["exit_status"] == 1: + pytest.fail("Creating an app from the CLI failed.") + + serve_inputs = ["n"] # No .env file found! Are you sure you [...] + result = run_variant_serve(serve_inputs, self.asset_example_folder) + cli_serve_output = next(result) + + # ASSERT: Verify response + assert cli_serve_output["exit_status"] == 0 + assert "Operation cancelled." in cli_serve_output["output"] + + # CLEANUP: + # i). Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files( + self.asset_example_folder, provide_api_key + ) + assert next(cleanup) == "ok" + + # ii). Rename the.env.dummy back to.env if it exists + if Path(f"{self.assets_folder}/.env.dummy").exists(): + os.rename(f"{self.assets_folder}/.env.dummy", f"{self.assets_folder}/.env")