From d5774bf1574bb7760dd1f8dc81b56960ef378fc6 Mon Sep 17 00:00:00 2001 From: Abram Date: Sun, 10 Sep 2023 19:39:05 +0100 Subject: [PATCH 001/103] Update - added restrictedpython --- agenta-backend/poetry.lock | 96 +++++++++++++++++++++++------------ agenta-backend/pyproject.toml | 1 + 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/agenta-backend/poetry.lock b/agenta-backend/poetry.lock index 5b8f69d042..eed7ee8bf5 100644 --- a/agenta-backend/poetry.lock +++ b/agenta-backend/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "aiodocker" @@ -154,24 +154,24 @@ uvloop = ["uvloop (>=0.13,<0.15)"] [[package]] name = "anyio" -version = "3.7.1" +version = "4.0.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, - {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, + {file = "anyio-4.0.0-py3-none-any.whl", hash = "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f"}, + {file = "anyio-4.0.0.tar.gz", hash = "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a"}, ] [package.dependencies] -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" [package.extras] -doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] -test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] -trio = ["trio (<0.22)"] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.22)"] [[package]] name = "asgiref" @@ -584,18 +584,21 @@ test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==23.1.0)", "coverage[toml] (>=6 [[package]] name = "filelock" -version = "3.12.2" +version = "3.12.3" description = "A platform independent file lock." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, - {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, + {file = "filelock-3.12.3-py3-none-any.whl", hash = "sha256:f067e40ccc40f2b48395a80fcbd4728262fab54e232e090a4063ab804179efeb"}, + {file = "filelock-3.12.3.tar.gz", hash = "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d"}, ] +[package.dependencies] +typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.11\""} + [package.extras] -docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] [[package]] name = "frozenlist" @@ -679,6 +682,7 @@ files = [ {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, + {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, @@ -687,6 +691,7 @@ files = [ {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, + {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, @@ -716,6 +721,7 @@ files = [ {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, + {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, @@ -724,6 +730,7 @@ files = [ {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, + {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, @@ -858,13 +865,13 @@ text-helpers = ["chardet (>=5.1.0,<6.0.0)"] [[package]] name = "langsmith" -version = "0.0.26" +version = "0.0.35" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langsmith-0.0.26-py3-none-any.whl", hash = "sha256:61c1d4582104d96edde04e1eea1dae347645b691c44489a5871341a2a1a2a1eb"}, - {file = "langsmith-0.0.26.tar.gz", hash = "sha256:80a4ef1b663a24a460d25b9986ab2010c5d06b6061c65be473abafc0647d191a"}, + {file = "langsmith-0.0.35-py3-none-any.whl", hash = "sha256:96b5cf69952a218d881a27817381fa67ebd283eba9c6814b72180530f9748348"}, + {file = "langsmith-0.0.35.tar.gz", hash = "sha256:127fee806b475430b530bdf9bc397ea1c65ec144a23fa1b5bba2bba31d9d1e76"}, ] [package.dependencies] @@ -1123,13 +1130,13 @@ test = ["async-asgi-testclient (>=1.4.4,<1.5.0)", "asyncmock (>=0.4.2,<0.5.0)", [[package]] name = "openai" -version = "0.27.9" +version = "0.27.10" description = "Python client library for the OpenAI API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-0.27.9-py3-none-any.whl", hash = "sha256:6a3cf8e276d1a6262b50562fbc0cba7967cfebb78ed827d375986b48fdad6475"}, - {file = "openai-0.27.9.tar.gz", hash = "sha256:b687761c82f5ebb6f61efc791b2083d2d068277b94802d4d1369efe39851813d"}, + {file = "openai-0.27.10-py3-none-any.whl", hash = "sha256:beabd1757e3286fa166dde3b70ebb5ad8081af046876b47c14c41e203ed22a14"}, + {file = "openai-0.27.10.tar.gz", hash = "sha256:60e09edf7100080283688748c6803b7b3b52d5a55d21890f3815292a0552d83b"}, ] [package.dependencies] @@ -1430,13 +1437,13 @@ zstd = ["zstandard"] [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -1466,13 +1473,13 @@ dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatc [[package]] name = "pytz" -version = "2023.3" +version = "2023.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] [[package]] @@ -1510,6 +1517,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1517,8 +1525,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1535,6 +1550,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1542,6 +1558,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1601,6 +1618,21 @@ files = [ requests = ">=1.0.0" six = "*" +[[package]] +name = "restrictedpython" +version = "6.2" +description = "RestrictedPython is a defined subset of the Python language which allows to provide a program input into a trusted environment." +optional = false +python-versions = ">=3.6, <3.12" +files = [ + {file = "RestrictedPython-6.2-py3-none-any.whl", hash = "sha256:7c2ffa4904300d67732f841d8a975dcdc53eba4c1cdc9d84b97684ef12304a3d"}, + {file = "RestrictedPython-6.2.tar.gz", hash = "sha256:db73eb7e3b39650f0d21d10cc8dda9c0e2986e621c94b0c5de32fb0dee3a08af"}, +] + +[package.extras] +docs = ["Sphinx", "sphinx-rtd-theme"] +test = ["pytest", "pytest-mock"] + [[package]] name = "six" version = "1.16.0" @@ -1674,7 +1706,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] @@ -1747,13 +1779,13 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyam [[package]] name = "supertokens-python" -version = "0.15.2" +version = "0.15.3" description = "SuperTokens SDK for Python" optional = false python-versions = ">=3.7" files = [ - {file = "supertokens_python-0.15.2-py3-none-any.whl", hash = "sha256:6bf4028bfca54e0e5820232a185c37d770205cad8ac920defebc231778691610"}, - {file = "supertokens_python-0.15.2.tar.gz", hash = "sha256:e7129504913ac6d2a33969e607a852bf95abda12fd013cfcfd61d31eacb2247c"}, + {file = "supertokens_python-0.15.3-py3-none-any.whl", hash = "sha256:153173fd93b12a023f96f547208e27692fd08471bfc92dc94c6e23be5f98e629"}, + {file = "supertokens_python-0.15.3.tar.gz", hash = "sha256:233e774f09e7470af025a22fa124c09bcddf4b59aeab86d87211e5f3afb8dac3"}, ] [package.dependencies] @@ -2115,4 +2147,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "2ceaece27051d32064ce72c40947152c46f57f44c8bedf92c1d30b3f526b6d05" +content-hash = "fb7b1efa4fa55cb9d4a1a7a0a736a688d385447052f606d7e997a087d2c968d7" diff --git a/agenta-backend/pyproject.toml b/agenta-backend/pyproject.toml index 4e796a47ac..81360bbf2f 100644 --- a/agenta-backend/pyproject.toml +++ b/agenta-backend/pyproject.toml @@ -23,6 +23,7 @@ openai = "^0.27.8" langchain = "^0.0.256" odmantic = "^0.9.2" supertokens-python = "^0.15.1" +restrictedpython = { version = "^6.2", python = ">=3.9,<3.12" } [tool.poetry.group.dev.dependencies] From e2d57bd0b57378a2155e6f8c165c3c146531acae Mon Sep 17 00:00:00 2001 From: Abram Date: Sun, 10 Sep 2023 19:39:49 +0100 Subject: [PATCH 002/103] Feat - created security module --- agenta-backend/agenta_backend/services/security/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 agenta-backend/agenta_backend/services/security/__init__.py diff --git a/agenta-backend/agenta_backend/services/security/__init__.py b/agenta-backend/agenta_backend/services/security/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 3cefed176dbdc0bafe116f85919d2a22a9a45601 Mon Sep 17 00:00:00 2001 From: Abram Date: Sun, 10 Sep 2023 19:40:57 +0100 Subject: [PATCH 003/103] Feat - implemented execute_code_safely function --- .../services/security/sandbox.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 agenta-backend/agenta_backend/services/security/sandbox.py diff --git a/agenta-backend/agenta_backend/services/security/sandbox.py b/agenta-backend/agenta_backend/services/security/sandbox.py new file mode 100644 index 0000000000..0c9374a9fb --- /dev/null +++ b/agenta-backend/agenta_backend/services/security/sandbox.py @@ -0,0 +1,47 @@ +from typing import List, Union, Text + +from RestrictedPython import safe_builtins, compile_restricted +from RestrictedPython.Eval import default_guarded_getiter, default_guarded_getitem +from RestrictedPython.Guards import guarded_iter_unpack_sequence, full_write_guard + + +def execute_code_safely(code: Text, allowed_imports: List[str], inputs: List[str]) -> Union[float, None]: + """ + Execute the provided Python code safely using RestrictedPython. + + Args: + - code (str): The Python code to be executed. + - allowed_imports (list): List of modules or objects that can be imported. + - inputs (dict): Inputs to be used during code execution. + + Returns: + - (float): Result of the execution if successful. Should be between 0 and 1. + - None if execution fails or result is not a float between 0 and 1. + """ + # Define the available built-ins + local_builtins = safe_builtins.copy() + + # Allow certain imports + for item in allowed_imports: + local_builtins[item] = __import__(item) + + # Define the environment for the code execution + environment = { + '_getiter_': default_guarded_getiter, + '_getitem_': default_guarded_getitem, + '_iter_unpack_sequence_': guarded_iter_unpack_sequence, + '_write_': full_write_guard, + 'inputs': inputs + } + + # Compile the code in a restricted environment + byte_code = compile_restricted(code, filename='', mode='exec') + + # Execute the code + exec(byte_code, environment) + + # Extract the result if it exists and is a float between 0 and 1 + result = environment.get('result', None) + if isinstance(result, float) and 0 <= result <= 1: + return result + return None From 7e74d5aeb08dbfed640c6de6df98aa6760a92bc7 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 03:59:21 +0100 Subject: [PATCH 004/103] Feat - created custom evaluation db collection --- .../agenta_backend/models/db_models.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index c3d07bb20c..dc3a298e30 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -110,6 +110,26 @@ class EvaluationScenarioDB(Model): class Config: collection = "evaluation_scenarios" + + +class CustomEvaluationTarget(EmbeddedModel): + inputs: Dict[str, Any] + output: float + target: float + + +class CustomEvaluationDB(Model): + evaluation_name: str + python_code: str + app_name: str + user: UserDB = Reference() + allowed_imports: List[str] + parameters: CustomEvaluationTarget + created_at: Optional[datetime] = Field(default=datetime.utcnow()) + updated_at: Optional[datetime] = Field(default=datetime.utcnow()) + + class Config: + collection = "custom_evaluations" class TestSetDB(Model): From 3e0ee169ca4faa5b2c27ea98b4e0341a9dca488e Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 04:00:18 +0100 Subject: [PATCH 005/103] Feat - created custom evaluation type and store custom evaluation api models --- .../models/api/evaluation_model.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/models/api/evaluation_model.py b/agenta-backend/agenta_backend/models/api/evaluation_model.py index 3a2fa3c5c2..f7a0065ce4 100644 --- a/agenta-backend/agenta_backend/models/api/evaluation_model.py +++ b/agenta-backend/agenta_backend/models/api/evaluation_model.py @@ -1,5 +1,5 @@ from pydantic import BaseModel, Field -from typing import Optional, List, Dict +from typing import Optional, List, Dict, Any from datetime import datetime from enum import Enum @@ -82,3 +82,18 @@ class NewEvaluation(BaseModel): class DeleteEvaluation(BaseModel): evaluations_ids: List[str] + + +class CustomEvaluationTarget(BaseModel): + inputs: Dict[str, Any] + output: float + target: float + + +class StoreCustomEvaluation(BaseModel): + evaluation_name: str + python_code: str + app_name: str + parameters: CustomEvaluationTarget + allowed_imports: List[str] + \ No newline at end of file From ad4fe0d0c69eddbfa30623941a51d32fabcbb025 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 04:01:29 +0100 Subject: [PATCH 006/103] Feat - implemented store and execute custom code evaluation logics --- .../services/evaluation_service.py | 91 +++++++++++++++++-- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 1b52a1c1d4..818c021fa3 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -11,7 +11,9 @@ NewEvaluation, EvaluationScenarioUpdate, EvaluationStatus, + StoreCustomEvaluation, ) +from agenta_backend.services.security.sandbox import execute_code_safely from agenta_backend.services.db_manager import engine, query, get_user_object from agenta_backend.models.db_models import ( EvaluationDB, @@ -20,6 +22,8 @@ EvaluationTypeSettings, EvaluationScenarioInput, EvaluationScenarioOutput, + CustomEvaluationDB, + CustomEvaluationTarget, ) from langchain.chains import LLMChain @@ -33,7 +37,9 @@ class UpdateEvaluationScenarioError(Exception): pass -async def create_new_evaluation(payload: NewEvaluation, **kwargs: dict) -> Dict: +async def create_new_evaluation( + payload: NewEvaluation, **kwargs: dict +) -> Dict: # Get user object user = await get_user_object(kwargs["uid"]) @@ -43,7 +49,9 @@ async def create_new_evaluation(payload: NewEvaluation, **kwargs: dict) -> Dict: evaluation_dict["updated_at"] = datetime.utcnow() # Initialize evaluation type settings embedded model - similarity_threshold = payload.evaluation_type_settings.similarity_threshold + similarity_threshold = ( + payload.evaluation_type_settings.similarity_threshold + ) evaluation_type_settings = EvaluationTypeSettings( similarity_threshold=0.0 if similarity_threshold is None @@ -72,7 +80,9 @@ async def create_new_evaluation(payload: NewEvaluation, **kwargs: dict) -> Dict: # Get testset using the provided _id testsetId = eval_instance.testset["_id"] - testset = await engine.find_one(TestSetDB, TestSetDB.id == ObjectId(testsetId)) + testset = await engine.find_one( + TestSetDB, TestSetDB.id == ObjectId(testsetId) + ) csvdata = testset.csvdata for datum in csvdata: @@ -169,9 +179,9 @@ async def update_evaluation_status( user = await get_user_object(kwargs["uid"]) # Construct query expression for evaluation - query_expression = query.eq(EvaluationDB.id, ObjectId(evaluation_id)) & query.eq( - EvaluationDB.user, user.id - ) + query_expression = query.eq( + EvaluationDB.id, ObjectId(evaluation_id) + ) & query.eq(EvaluationDB.user, user.id) result = await engine.find_one(EvaluationDB, query_expression) if result is not None: @@ -192,7 +202,9 @@ async def update_evaluation_status( updated_at=result.updated_at, ) else: - raise UpdateEvaluationScenarioError("Failed to update evaluation status") + raise UpdateEvaluationScenarioError( + "Failed to update evaluation status" + ) async def update_evaluation_scenario( @@ -241,7 +253,9 @@ async def update_evaluation_scenario( for scenario_input in current_evaluation_scenario.inputs ], correct_answer=current_evaluation_scenario.correct_answer, - app_variant_output=new_evaluation_set["outputs"][0]["variant_output"], + app_variant_output=new_evaluation_set["outputs"][0][ + "variant_output" + ], evaluation_prompt_template=evaluation_scenario_dict[ "evaluation_prompt_template" ], @@ -250,7 +264,9 @@ async def update_evaluation_scenario( new_evaluation_set["evaluation"] = evaluation # Get an evaluation scenario with the provided id - result = await engine.find_one(EvaluationScenarioDB, query_expression_eval_scen) + result = await engine.find_one( + EvaluationScenarioDB, query_expression_eval_scen + ) # Loop through the evaluation set outputs, create an evaluation scenario # output instance and append the instance in the list @@ -384,3 +400,60 @@ def extend_with_correct_answer(evaluation_type: EvaluationType, row: dict): if row["correct_answer"]: correct_answer["correct_answer"] = row["correct_answer"] return correct_answer + + +async def store_custom_code_evaluation( + payload: StoreCustomEvaluation, **kwargs: dict +) -> str: + # Get user object + user = await get_user_object(kwargs["uid"]) + + # Set payload as dictionary + payload_dict = payload.dict() + + # Set evaluation target payload + eval_target_dict = payload_dict["parameters"] + + # Instantiate custom evaluation + eval_target = CustomEvaluationTarget(**eval_target_dict) + + del payload_dict["parameters"] + custom_eval = CustomEvaluationDB( + **payload_dict, parameters=eval_target, user=user + ) + + await engine.save(custom_eval) + return str(custom_eval.id) + + +async def execute_cusom_code_evaluation( + evaluation_id: str, app_name: str, **kwargs: dict +): + # Get user object + user = await get_user_object(kwargs["uid"]) + + # Build query expression + query_expression = ( + query.eq(CustomEvaluationDB.id, ObjectId(evaluation_id)) + & query.eq(CustomEvaluationDB.user, user.id) + & query.eq(CustomEvaluationDB.app_name, app_name) + ) + + # Get custom evaluation + custom_eval = await engine.find_one(CustomEvaluationDB, query_expression) + if not custom_eval: + raise HTTPException(status_code=404, detail="Evaluation not found") + + # Execute the Python code with the provided inputs and allowed imports + try: + result = execute_code_safely( + custom_eval.python_code, + custom_eval.allowed_imports, + custom_eval.parameters.inputs + ) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to execute custom code evaluation: {str(e)}", + ) + return result From f4bd4ead9b3a15b7eed30d5b486f1af0a418e829 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 04:02:27 +0100 Subject: [PATCH 007/103] Feat - implemented function to check if module import is safe to ensure code safe execution --- .../services/security/sandbox.py | 71 +++++++++++++++---- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/agenta-backend/agenta_backend/services/security/sandbox.py b/agenta-backend/agenta_backend/services/security/sandbox.py index 0c9374a9fb..6ac55b1a9c 100644 --- a/agenta-backend/agenta_backend/services/security/sandbox.py +++ b/agenta-backend/agenta_backend/services/security/sandbox.py @@ -1,11 +1,44 @@ -from typing import List, Union, Text +import re +from typing import List, Union, Text, Dict, Any from RestrictedPython import safe_builtins, compile_restricted -from RestrictedPython.Eval import default_guarded_getiter, default_guarded_getitem -from RestrictedPython.Guards import guarded_iter_unpack_sequence, full_write_guard +from RestrictedPython.Eval import ( + default_guarded_getiter, + default_guarded_getitem, +) +from RestrictedPython.Guards import ( + guarded_iter_unpack_sequence, + full_write_guard, +) -def execute_code_safely(code: Text, allowed_imports: List[str], inputs: List[str]) -> Union[float, None]: +def is_import_safe(module: str) -> bool: + """Checks if a given package import contains any disallowed patterns of code. + + Args: + module (str) -- The module to check for potentially dangerous code + + Returns: + bool - module is secured or not + """ + + # Define patterns to disallow dangerous code + disallowed_patterns = [ + r"import\s+os", + r"import\s+subprocess", + # Add more patterns as needed + ] + + # Check if any disallowed patterns are present + for pattern in disallowed_patterns: + if re.search(pattern, module): + return False + return True + + +def execute_code_safely( + code: Text, allowed_imports: List[str], inputs: Dict[str, Any] +) -> Union[float, None]: """ Execute the provided Python code safely using RestrictedPython. @@ -20,28 +53,40 @@ def execute_code_safely(code: Text, allowed_imports: List[str], inputs: List[str """ # Define the available built-ins local_builtins = safe_builtins.copy() + + # Add the __import__ built-in function to the local builtins + local_builtins["__import__"] = __import__ - # Allow certain imports + # Create a dictionary to simulate allowed imports + allowed_modules = {} for item in allowed_imports: - local_builtins[item] = __import__(item) + module_safe = is_import_safe(item) + if not module_safe: + raise Exception("") + else: + allowed_modules[item] = __import__(item) + + # Add the allowed modules to the local built-ins + local_builtins.update(allowed_modules) # Define the environment for the code execution environment = { - '_getiter_': default_guarded_getiter, - '_getitem_': default_guarded_getitem, - '_iter_unpack_sequence_': guarded_iter_unpack_sequence, - '_write_': full_write_guard, - 'inputs': inputs + "_getiter_": default_guarded_getiter, + "_getitem_": default_guarded_getitem, + "_iter_unpack_sequence_": guarded_iter_unpack_sequence, + "_write_": full_write_guard, + "inputs": inputs, + "__builtins__": local_builtins } # Compile the code in a restricted environment - byte_code = compile_restricted(code, filename='', mode='exec') + byte_code = compile_restricted(code, filename="", mode="exec") # Execute the code exec(byte_code, environment) # Extract the result if it exists and is a float between 0 and 1 - result = environment.get('result', None) + result = environment.get("result", None) if isinstance(result, float) and 0 <= result <= 1: return result return None From 05107f17e1595bf67f415e101542194725118b46 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 04:04:16 +0100 Subject: [PATCH 008/103] Cleanup - remove app_name from execute_custom_code_evaluation --- .../routers/evaluation_router.py | 72 ++++++++++++++++--- .../services/evaluation_service.py | 5 +- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 6b6ea716b6..5555d5bffa 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -4,6 +4,7 @@ from typing import List, Optional from fastapi import HTTPException, APIRouter, Body, Depends +from fastapi.responses import JSONResponse from agenta_backend.models.api.evaluation_model import ( Evaluation, @@ -13,6 +14,7 @@ DeleteEvaluation, EvaluationType, EvaluationStatus, + StoreCustomEvaluation, ) from agenta_backend.services.results_service import ( fetch_results_for_human_a_b_testing_evaluation, @@ -26,6 +28,8 @@ update_evaluation_status, create_new_evaluation, create_new_evaluation_scenario, + store_custom_code_evaluation, + execute_cusom_code_evaluation, ) from agenta_backend.services.db_manager import engine, query, get_user_object from agenta_backend.models.db_models import EvaluationDB, EvaluationScenarioDB @@ -85,7 +89,9 @@ async def update_evaluation_status_router( try: # Get user and organization id kwargs: dict = await get_user_and_org_id(stoken_session) - return await update_evaluation_status(evaluation_id, update_data, **kwargs) + return await update_evaluation_status( + evaluation_id, update_data, **kwargs + ) except KeyError: raise HTTPException( status_code=400, @@ -138,7 +144,9 @@ async def fetch_evaluation_scenarios( return eval_scenarios -@router.post("/{evaluation_id}/evaluation_scenario", response_model=EvaluationScenario) +@router.post( + "/{evaluation_id}/evaluation_scenario", response_model=EvaluationScenario +) async def create_evaluation_scenario( evaluation_id: str, evaluation_scenario: EvaluationScenario, @@ -262,9 +270,9 @@ async def fetch_evaluation( user = await get_user_object(kwargs["uid"]) # Construct query expression builder - query_expression = query.eq(EvaluationDB.id, ObjectId(evaluation_id)) & query.eq( - EvaluationDB.user, user.id - ) + query_expression = query.eq( + EvaluationDB.id, ObjectId(evaluation_id) + ) & query.eq(EvaluationDB.user, user.id) evaluation = await engine.find_one(EvaluationDB, query_expression) if evaluation is not None: return Evaluation( @@ -344,9 +352,9 @@ async def fetch_results( user = await get_user_object(kwargs["uid"]) # Construct query expression builder and retrieve evaluation from database - query_expression = query.eq(EvaluationDB.id, ObjectId(evaluation_id)) & query.eq( - EvaluationDB.user, user.id - ) + query_expression = query.eq( + EvaluationDB.id, ObjectId(evaluation_id) + ) & query.eq(EvaluationDB.user, user.id) evaluation = await engine.find_one(EvaluationDB, query_expression) if evaluation.evaluation_type == EvaluationType.human_a_b_testing: @@ -371,3 +379,51 @@ async def fetch_results( elif evaluation.evaluation_type == EvaluationType.auto_ai_critique: results = await fetch_results_for_auto_ai_critique(evaluation_id) return {"results_data": results} + + +@router.post("/custom_evaluation/store/") +async def store_custom_evaluation( + custom_evaluation_payload: StoreCustomEvaluation, + stoken_session: SessionContainer = Depends(verify_session()), +): + """Store evaluation with custom python code. + + Args: + \n custom_evaluation_payload (StoreCustomEvaluation): payload schema required + \n stoken_session (SessionContainer, optional): session token to verify user session. Defaults to Depends(verify_session()). + """ + + # Get user and organization id + kwargs: dict = await get_user_and_org_id(stoken_session) + + # Store custom evaluation in database + evaluation_id = await store_custom_code_evaluation( + custom_evaluation_payload, **kwargs + ) + + return JSONResponse( + { + "status": "success", + "message": "Evaluation stored successfully.", + "evaluation_id": evaluation_id, + }, + status_code=200, + ) + + +@router.post( + "/custom_evaluation/execute/{app_name}/{evaluation_id}/" +) +async def execute_evaluation( + app_name: str, + evaluation_id: str, + stoken_session: SessionContainer = Depends(verify_session()), +): + # Get user and organization id + kwargs: dict = await get_user_and_org_id(stoken_session) + + # Excute custom code evaluation + result = await execute_cusom_code_evaluation( + evaluation_id, app_name, **kwargs + ) + return result diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 818c021fa3..faa25b5dec 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -426,8 +426,8 @@ async def store_custom_code_evaluation( return str(custom_eval.id) -async def execute_cusom_code_evaluation( - evaluation_id: str, app_name: str, **kwargs: dict +async def execute_custom_code_evaluation( + evaluation_id: str, **kwargs: dict ): # Get user object user = await get_user_object(kwargs["uid"]) @@ -436,7 +436,6 @@ async def execute_cusom_code_evaluation( query_expression = ( query.eq(CustomEvaluationDB.id, ObjectId(evaluation_id)) & query.eq(CustomEvaluationDB.user, user.id) - & query.eq(CustomEvaluationDB.app_name, app_name) ) # Get custom evaluation From 182c8122b8c148e2a2337d6f1c7a36300396a299 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 04:04:59 +0100 Subject: [PATCH 009/103] Feat - implemented store and execute custom evaluation routers --- .../agenta_backend/routers/evaluation_router.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 5555d5bffa..d5977070e0 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -29,7 +29,7 @@ create_new_evaluation, create_new_evaluation_scenario, store_custom_code_evaluation, - execute_cusom_code_evaluation, + execute_custom_code_evaluation, ) from agenta_backend.services.db_manager import engine, query, get_user_object from agenta_backend.models.db_models import EvaluationDB, EvaluationScenarioDB @@ -412,10 +412,9 @@ async def store_custom_evaluation( @router.post( - "/custom_evaluation/execute/{app_name}/{evaluation_id}/" + "/custom_evaluation/execute/{evaluation_id}/" ) -async def execute_evaluation( - app_name: str, +async def execute_custom_evaluation( evaluation_id: str, stoken_session: SessionContainer = Depends(verify_session()), ): @@ -423,7 +422,7 @@ async def execute_evaluation( kwargs: dict = await get_user_and_org_id(stoken_session) # Excute custom code evaluation - result = await execute_cusom_code_evaluation( - evaluation_id, app_name, **kwargs + result = await execute_custom_code_evaluation( + evaluation_id, **kwargs ) return result From 7e8e80539a0d50a04dd92fc78e4d4dfba4dbc5b9 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 14:41:32 +0100 Subject: [PATCH 010/103] Update - added custom_code_run to evaluation type and labels --- agenta-web/src/lib/enums.ts | 1 + agenta-web/src/lib/helpers/utils.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/agenta-web/src/lib/enums.ts b/agenta-web/src/lib/enums.ts index 20013fd15e..798e361ddc 100644 --- a/agenta-web/src/lib/enums.ts +++ b/agenta-web/src/lib/enums.ts @@ -11,4 +11,5 @@ export enum EvaluationType { auto_exact_match = "auto_exact_match", auto_similarity_match = "auto_similarity_match", auto_ai_critique = "auto_ai_critique", + custom_code_run = "custom_code_run", } diff --git a/agenta-web/src/lib/helpers/utils.ts b/agenta-web/src/lib/helpers/utils.ts index 59793513ea..49d3410414 100644 --- a/agenta-web/src/lib/helpers/utils.ts +++ b/agenta-web/src/lib/helpers/utils.ts @@ -20,6 +20,7 @@ export const EvaluationTypeLabels: Record = { [EvaluationType.auto_ai_critique]: "AI Critic", [EvaluationType.human_a_b_testing]: "A/B testing", [EvaluationType.human_scoring]: "Scoring single variant", + [EvaluationType.custom_code_run]: "Custom Code Run" } export const saveOpenAIKey = (key: string) => { From 1e6c7d2862a84b90bb4f789e5b274f016d6bc479 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 14:41:53 +0100 Subject: [PATCH 011/103] Feat - upload custom_code image --- agenta-web/src/media/custom_code.png | Bin 0 -> 2144 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 agenta-web/src/media/custom_code.png diff --git a/agenta-web/src/media/custom_code.png b/agenta-web/src/media/custom_code.png new file mode 100644 index 0000000000000000000000000000000000000000..4ebb0b4bff6c99b6cd54421e150ec68e89e81439 GIT binary patch literal 2144 zcmbVOdpOgJ8~@H`Om0!9nPKEoIhD&|F0tJ2PFBK5=5%VZj3tw?xkfJ0m6PaKOXV`; z7Q<#qLgUCKmo*hB_lnGIetORDuix|h{`j5uc`wiBdEfVW-@iWZoA#TBqnr#v1^@s# zXD2(at=Rt4q@i2B%!gI56(AHF=VQ`aLX-YIbqmADPQDZX*rxo`K)|2*Z~%~&aJD;g zjG8|yAeADItG)ahHlM5&g7<^jlz2bzR!X?v>XS)OHpppXAdb>fbtml8bVCSoW436! zubR3}8Gadro_l*7&_$2;SnW^_p@qn%pa}Zj_Qm_RJ4ZjOn@*>gp0A_u`A;Xi=4Rro zUp0Q_k5a@j794?D6u*n1?R@PYD7yL$10OGW@8bufwh2Y@;ffU5Tc^(k>yY)x5h=Ay z|MMWeN=;6^Y;tHD`RuoRxl)aq93;q0(^l0BR^muPIbxYlPGgx#!7{b6hsfbAKOV9K z-0v>v8Pn7oJ;$(o$VE_kk~OUGc61ZE@<7pQ((4}&m(DTh%Vl_g-(G!k*nPOlqUE9+0Z@TA72z**Q(?Nm0gHUAWr2?1;Az!mqlFq~i~748$qmQcNz|6r({ zivh|j0hFU8``@MiGgaMfJ|+s^WvDOf6q7GMDnGM(bXF|*xc`g^Jo_R>{TU2H`Xan4 z#e(oP@b<<$M)h-hE^gmRI_^z!7Xp0kMB2z6HE^Te)467ejVG5uDgq=K7IY>SF4C+p z30^CgDmeM|l9VX4o8u|bEI+C+3J!g~DNHlyROsK`P9CxnGilA`0zA}s*uybVn z#*}Rp5jO)eH|q+)9?+)wooHH?C%dFr@!hLAm1o#zv}-MZ#sjwduXpLWxo7X>)+9a8 zJDx+52P>TZJbkTw+5#qo2VB)2_K_O}{K1xnw(f}DUt<$PA#=K7q9B0vs%J1=X*P`Y zR{RMODUfV5uq9n@vn@5e1GWMzJI3xVY~zNjtgc^VZoGXH>j{bTdeH31=*(q3kOCVy zRK`;`BeH2PSGz&K?A~b9`AVU1*PiSeCnC72*@h};#rXpqpx{=e@Swx$kZTiV|3T=F zzQ-|s&;=+nNDwc&rxv>X88Wx~?Xn+lj9KScS)*;(k0dzh)g$cc zNFWR=gZIR&9X=~%4ry#UrFPqXHCB<@n@ZA<^5041&{tGue{PyECEpsR=YRcZ$45rY z?%2Sam4zp^%MU`EgkY|u%G&-{j2#s+BAF$tqjfm(zUH!=W(^(M^N9;GWLG9fJkiPR z=-Qb418amof_d*fiCe6Z>oazQYoF*}=ue0}>C$94czz6)Jk{R9`2C_}V$_wpT%2n0 zWP+WVj>0s`jkrR&zd08X{odRm2x~NAOLO!AtlZvFR1??Dy*jf}Rs)@LX?Zg3g^1#b zN&{5hi=yxg3{zA8&Y?6ve#o<1`pPxGqWwD!^5+VM#H|h8Ju{2xuDi?UwUD0<1r2$r z%{zsD?@kHH+f#+ep4SM zdW(gplnD92{qV|JrXVq?YQ4VStoyG3mR{Gs^UC1fdkuw8=|I66d^qrkw9WB5Co+6u zZf5pb;KaZ}t;B|D3j;kf#4U$6;toR^&$ZXq*$6`*9Z$ zSrh$psIJpp(7J@9ux`1wz1Eul+nW^7(go@38Y7m9p5Z3*kos*@*g`l@^Z3nZ9`hxu z3iIv!OjCrLi<{uhltByb;*){m(qjrVc5 zKgm#L<18{dtfVQ4qGhK3R}oLvDLS2thbatO-XtH;8VoEymK&V?X+|vygeVaHA5?uK z|1~?3nB&(O_BZpJNkq_7OqfdY8N=8t#ta}X_?Oz=M|a|597x)ZSap=;^xEV&Co>aW qYMb=(bpU40<{uo=t(HFXO)!C3p?*}SRCen{0G#bT>{vFZ68;T^+uRKR literal 0 HcmV?d00001 From b9c705d5014ca87d96916e50b9a326f49eb63398 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 14:58:03 +0100 Subject: [PATCH 012/103] Feat - created store custom evaluation type interface --- agenta-web/src/lib/Types.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/agenta-web/src/lib/Types.ts b/agenta-web/src/lib/Types.ts index 15f6c1e9d3..f2edb958bc 100644 --- a/agenta-web/src/lib/Types.ts +++ b/agenta-web/src/lib/Types.ts @@ -61,6 +61,13 @@ export interface Evaluation { appName: string } + +export interface StoreCustomEvaluation { + evaluation_name: string + python_code: string + app_name: string +} + export interface Parameter { name: string type: string From 45e3f0de4c7aba7bf8a1c6612aabd9b53b120736 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 15:26:04 +0100 Subject: [PATCH 013/103] Feat - created type interface for store custom evaluation success response --- agenta-web/src/lib/Types.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/agenta-web/src/lib/Types.ts b/agenta-web/src/lib/Types.ts index f2edb958bc..424b52f638 100644 --- a/agenta-web/src/lib/Types.ts +++ b/agenta-web/src/lib/Types.ts @@ -1,3 +1,4 @@ +import { AbsoluteString } from "next/dist/lib/metadata/types/metadata-types" import {EvaluationFlow, EvaluationType} from "./enums" export interface testset { @@ -68,6 +69,12 @@ export interface StoreCustomEvaluation { app_name: string } +export interface StoreCustomEvaluationSuccessResponse { + status: string + message: string + evaluation_id: string +} + export interface Parameter { name: string type: string From 37ab07f2c55bfa181bcf4b2404769a0dcdbfffa9 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 15:26:54 +0100 Subject: [PATCH 014/103] Feat - implemented save custom code evaluation api logic --- agenta-web/src/lib/services/api.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/agenta-web/src/lib/services/api.ts b/agenta-web/src/lib/services/api.ts index b7c5ecda17..d5ec8b91c9 100644 --- a/agenta-web/src/lib/services/api.ts +++ b/agenta-web/src/lib/services/api.ts @@ -11,6 +11,7 @@ import { TemplateImage, RestartVariantDocker, RestartVariantDockerResponse, + StoreCustomEvaluation } from "@/lib/Types" import { fromEvaluationResponseToEvaluation, @@ -388,6 +389,19 @@ export const fetchEvaluationResults = async (evaluationId: string) => { return response.data } + +export const saveCutomCodeEvaluation = async ( + payload: StoreCustomEvaluation, + ignoreAxiosError: boolean = false, +) => { + const response = await axios.post( + `${process.env.NEXT_PUBLIC_AGENTA_API_URL}/api/evaluations/custom_evaluation/store/`, + payload, + {_ignoreError: ignoreAxiosError} as any, + ) + return response +} + export const fetchApps = () => { const {data, error, isLoading} = useSWR( `${process.env.NEXT_PUBLIC_AGENTA_API_URL}/api/app_variant/list_apps/`, From db97338cb44919dc7952616bffc10f3d7b01bef8 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 18:12:43 +0100 Subject: [PATCH 015/103] Feat - implemented custom evaluation dropdown component --- .../Evaluations/CustomEvaluationsDropdown.tsx | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 agenta-web/src/components/Evaluations/CustomEvaluationsDropdown.tsx diff --git a/agenta-web/src/components/Evaluations/CustomEvaluationsDropdown.tsx b/agenta-web/src/components/Evaluations/CustomEvaluationsDropdown.tsx new file mode 100644 index 0000000000..6079f40af4 --- /dev/null +++ b/agenta-web/src/components/Evaluations/CustomEvaluationsDropdown.tsx @@ -0,0 +1,49 @@ +import React, { useState, useEffect } from 'react' +import { Select } from 'antd' +import { fetchCustomEvaluations } from '@/lib/services/api'; +import { SingleCustomEvaluation } from '@/lib/Types'; + +const { Option } = Select; + + + +const EvaluationDropdown: React.FC = ({ classes, appName }) => { + const [evaluation, setEvaluation] = useState(''); + const [evaluationList, setEvaluationList] = useState() + + useEffect(() => { + fetchCustomEvaluations(appName).then(res => { + if (res.status === 200) { + setEvaluationList(res.data); + } + }); + }, [appName]); + + const handleChange = (e: string) => { + setEvaluation(e) + }; + + const setOptionName = () => { + if (!evaluation){ + return "List Custom Evaluations" + } else { + return evaluation + } + } + + return ( + + ) +} + +export default EvaluationDropdown From 296b461900d21da1d6004ada11e03533d41eb441 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 18:13:33 +0100 Subject: [PATCH 016/103] Update - added type dropdown component --- .../components/Evaluations/CustomEvaluationsDropdown.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/agenta-web/src/components/Evaluations/CustomEvaluationsDropdown.tsx b/agenta-web/src/components/Evaluations/CustomEvaluationsDropdown.tsx index 6079f40af4..761e45a130 100644 --- a/agenta-web/src/components/Evaluations/CustomEvaluationsDropdown.tsx +++ b/agenta-web/src/components/Evaluations/CustomEvaluationsDropdown.tsx @@ -5,9 +5,12 @@ import { SingleCustomEvaluation } from '@/lib/Types'; const { Option } = Select; +interface EVDropdownI { + classes: any + appName: string; +} - -const EvaluationDropdown: React.FC = ({ classes, appName }) => { +const EvaluationDropdown: React.FC = ({ classes, appName }) => { const [evaluation, setEvaluation] = useState(''); const [evaluationList, setEvaluationList] = useState() From e97c9f6cfe69dca9f0b5e75d2c64b114bfbbed17 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 11 Sep 2023 18:15:04 +0100 Subject: [PATCH 017/103] Feat - implemented custom python code component --- .../Evaluations/CustomPythonCode.tsx | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 agenta-web/src/components/Evaluations/CustomPythonCode.tsx diff --git a/agenta-web/src/components/Evaluations/CustomPythonCode.tsx b/agenta-web/src/components/Evaluations/CustomPythonCode.tsx new file mode 100644 index 0000000000..ae611ffaa2 --- /dev/null +++ b/agenta-web/src/components/Evaluations/CustomPythonCode.tsx @@ -0,0 +1,104 @@ +import React, { useState } from 'react'; +import { Input, Form, Button, Row, Col, Typography, notification } from 'antd'; +import { StoreCustomEvaluationSuccessResponse } from '@/lib/Types' +import { saveCutomCodeEvaluation } from '@/lib/services/api'; + + +interface ICustomPythonProps { + classes: any + appName: string; +} + +const CustomPythonCode: React.FC = ({classes, appName}) => { + const { TextArea } = Input + const { Title } = Typography + const [form] = Form.useForm() + + const [submitting, setSubmittingData] = useState(false) + + let prevKey = "" + const showNotification = (config: Parameters[0]) => { + if (prevKey) notification.destroy(prevKey) + prevKey = (config.key || "") as string + notification.open(config) + } + + + const handlerToSubmitFormData = async (values: any) => { + setSubmittingData(true) + const data = { + evaluation_name: values.evaluationName, + python_code: values.pythonCode, + app_name: appName + } + const response = await saveCutomCodeEvaluation(data) + if (response.status === 200) { + const data: StoreCustomEvaluationSuccessResponse = response.data + + // Diable submitting form data + setSubmittingData(false) + showNotification({ + type:"success", + message: "Custom Evaluation", + description: data.message, + key: data.evaluation_id + }) + + // Reset form fields + form.resetFields() + } + }; + + + const isSaveButtonDisabled = () => { + return ( + !form.isFieldsTouched(true) || + form.getFieldsError().filter(({ errors }) => errors.length).length > 0 + ) + } + + return ( +
+ Save Python Code Evaluation +
+ + + + + + + + +