From c450447a14da49a4584d09cec0c5cd46733ef8d5 Mon Sep 17 00:00:00 2001 From: Neosoft Date: Wed, 9 Oct 2024 11:50:55 +0530 Subject: [PATCH 01/47] Python version upgraded from 3.8 to 3.11 --- AzureTRE | 1 + airlock_processor/Dockerfile | 2 +- api_app/Dockerfile | 2 +- resource_processor/vmss_porter/Dockerfile | 2 +- templates/shared_services/certs/Dockerfile.tmpl | 2 +- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) create mode 160000 AzureTRE diff --git a/AzureTRE b/AzureTRE new file mode 160000 index 0000000000..400766bd39 --- /dev/null +++ b/AzureTRE @@ -0,0 +1 @@ +Subproject commit 400766bd39ff6b5216c24679dfeb4fb1a2252076 diff --git a/airlock_processor/Dockerfile b/airlock_processor/Dockerfile index e7f56ab82b..27885be658 100644 --- a/airlock_processor/Dockerfile +++ b/airlock_processor/Dockerfile @@ -1,6 +1,6 @@ # To enable ssh & remote debugging on app service change the base image to the one below # FROM mcr.microsoft.com/azure-functions/python:4-python3.8-appservice as base -FROM mcr.microsoft.com/azure-functions/python:4-python3.8-slim as base +FROM mcr.microsoft.com/azure-functions/python:4-python3.11-slim as base COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt diff --git a/api_app/Dockerfile b/api_app/Dockerfile index ebf227f5ee..a510f17688 100644 --- a/api_app/Dockerfile +++ b/api_app/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-bullseye as base +FROM python:3.11-slim-bullseye as base COPY requirements.txt /. RUN pip3 install --no-cache-dir -r requirements.txt diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index 9c40ab5ee8..b41181510d 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM python:3.8-slim-bullseye +FROM python:3.11-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 0f39713944..623ea42e44 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -1,5 +1,5 @@ # syntax=docker/dockerfile-upstream:1.4.0 -FROM --platform=linux/amd64 python:3.8-slim-bullseye +FROM --platform=linux/amd64 python:3.11-slim-bullseye # PORTER_INIT diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index 839b35fe0c..990083f5f7 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-bullseye +FROM python:3.11-bullseye # Install MLflow Python Packages RUN pip install --no-cache-dir psycopg2==2.9.5 mlflow==2.0.1 azure-storage-blob==12.14.1 From 66a6321943568a79b33cd6cc611bcde1b4f1101d Mon Sep 17 00:00:00 2001 From: Neosoft Date: Wed, 9 Oct 2024 15:47:52 +0530 Subject: [PATCH 02/47] Python version changes done in build_docker_images.yml file --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a6d7a953ca..16c1a303e8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.166.1/containers/python-3/.devcontainer/base.Dockerfile # [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6 -ARG VARIANT="3.8" +ARG VARIANT="3.11" ARG TARGETPLATFORM="linux/amd64" FROM --platform="${TARGETPLATFORM}" mcr.microsoft.com/vscode/devcontainers/python:dev-${VARIANT}-bullseye From ebe46584b68109e909f879401025e9a12c236bea Mon Sep 17 00:00:00 2001 From: Neosoft Date: Wed, 9 Oct 2024 19:38:14 +0530 Subject: [PATCH 03/47] Testng 3.8 version test results --- .devcontainer/Dockerfile | 2 +- api_app/Dockerfile | 2 +- resource_processor/vmss_porter/Dockerfile | 2 +- templates/shared_services/certs/Dockerfile.tmpl | 2 +- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 16c1a303e8..a6d7a953ca 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.166.1/containers/python-3/.devcontainer/base.Dockerfile # [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6 -ARG VARIANT="3.11" +ARG VARIANT="3.8" ARG TARGETPLATFORM="linux/amd64" FROM --platform="${TARGETPLATFORM}" mcr.microsoft.com/vscode/devcontainers/python:dev-${VARIANT}-bullseye diff --git a/api_app/Dockerfile b/api_app/Dockerfile index a510f17688..ebf227f5ee 100644 --- a/api_app/Dockerfile +++ b/api_app/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bullseye as base +FROM python:3.8-slim-bullseye as base COPY requirements.txt /. RUN pip3 install --no-cache-dir -r requirements.txt diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index b41181510d..9c40ab5ee8 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM python:3.11-slim-bullseye +FROM python:3.8-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 623ea42e44..0f39713944 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -1,5 +1,5 @@ # syntax=docker/dockerfile-upstream:1.4.0 -FROM --platform=linux/amd64 python:3.11-slim-bullseye +FROM --platform=linux/amd64 python:3.8-slim-bullseye # PORTER_INIT diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index 990083f5f7..839b35fe0c 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-bullseye +FROM python:3.8-bullseye # Install MLflow Python Packages RUN pip install --no-cache-dir psycopg2==2.9.5 mlflow==2.0.1 azure-storage-blob==12.14.1 From 730ce90066317261a8dd51343d5f08f030788578 Mon Sep 17 00:00:00 2001 From: Neosoft Date: Tue, 15 Oct 2024 13:21:41 +0530 Subject: [PATCH 04/47] Testing 3.8 tests results --- airlock_processor/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airlock_processor/Dockerfile b/airlock_processor/Dockerfile index 27885be658..e7f56ab82b 100644 --- a/airlock_processor/Dockerfile +++ b/airlock_processor/Dockerfile @@ -1,6 +1,6 @@ # To enable ssh & remote debugging on app service change the base image to the one below # FROM mcr.microsoft.com/azure-functions/python:4-python3.8-appservice as base -FROM mcr.microsoft.com/azure-functions/python:4-python3.11-slim as base +FROM mcr.microsoft.com/azure-functions/python:4-python3.8-slim as base COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt From d777c07abe0896466751639d8612fb9d34fd4075 Mon Sep 17 00:00:00 2001 From: Neosoft Date: Tue, 15 Oct 2024 17:38:58 +0530 Subject: [PATCH 05/47] python 3.11 testing --- .devcontainer/Dockerfile | 2 +- airlock_processor/Dockerfile | 2 +- api_app/Dockerfile | 2 +- resource_processor/vmss_porter/Dockerfile | 2 +- templates/shared_services/certs/Dockerfile.tmpl | 2 +- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a6d7a953ca..16c1a303e8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.166.1/containers/python-3/.devcontainer/base.Dockerfile # [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6 -ARG VARIANT="3.8" +ARG VARIANT="3.11" ARG TARGETPLATFORM="linux/amd64" FROM --platform="${TARGETPLATFORM}" mcr.microsoft.com/vscode/devcontainers/python:dev-${VARIANT}-bullseye diff --git a/airlock_processor/Dockerfile b/airlock_processor/Dockerfile index e7f56ab82b..27885be658 100644 --- a/airlock_processor/Dockerfile +++ b/airlock_processor/Dockerfile @@ -1,6 +1,6 @@ # To enable ssh & remote debugging on app service change the base image to the one below # FROM mcr.microsoft.com/azure-functions/python:4-python3.8-appservice as base -FROM mcr.microsoft.com/azure-functions/python:4-python3.8-slim as base +FROM mcr.microsoft.com/azure-functions/python:4-python3.11-slim as base COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt diff --git a/api_app/Dockerfile b/api_app/Dockerfile index ebf227f5ee..a510f17688 100644 --- a/api_app/Dockerfile +++ b/api_app/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-bullseye as base +FROM python:3.11-slim-bullseye as base COPY requirements.txt /. RUN pip3 install --no-cache-dir -r requirements.txt diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index 9c40ab5ee8..b41181510d 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM python:3.8-slim-bullseye +FROM python:3.11-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 0f39713944..623ea42e44 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -1,5 +1,5 @@ # syntax=docker/dockerfile-upstream:1.4.0 -FROM --platform=linux/amd64 python:3.8-slim-bullseye +FROM --platform=linux/amd64 python:3.11-slim-bullseye # PORTER_INIT diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index 839b35fe0c..990083f5f7 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-bullseye +FROM python:3.11-bullseye # Install MLflow Python Packages RUN pip install --no-cache-dir psycopg2==2.9.5 mlflow==2.0.1 azure-storage-blob==12.14.1 From 3aff35b28f83909ae8c9d89dce66cfd9215d7229 Mon Sep 17 00:00:00 2001 From: gauravmishrack Date: Thu, 17 Oct 2024 10:31:15 +0000 Subject: [PATCH 06/47] Final Commit for python upgrade --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9d49130458..33601795a5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,8 +10,8 @@ "context": "..", "dockerfile": "Dockerfile", "args": { - // To ensure that the group ID for the docker group in the container - // matches the group ID on the host, add this to your .bash_profile on the host + // To ensure that the group ID for the docker group in container + // matches the group ID on the host, add this to your .bash_profile on host // export DOCKER_GROUP_ID=$(getent group docker | awk -F ":" '{ print $3 }') "DOCKER_GROUP_ID": "${localEnv:DOCKER_GROUP_ID}", "INTERACTIVE": "true" From 9c19e5a34c627e6807364827942a2cacbe7d395b Mon Sep 17 00:00:00 2001 From: gauravmishrack Date: Mon, 21 Oct 2024 07:38:36 +0000 Subject: [PATCH 07/47] requirements.txt updated for testing --- api_app/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api_app/requirements.txt b/api_app/requirements.txt index 5c773326ec..a6d209a1bb 100644 --- a/api_app/requirements.txt +++ b/api_app/requirements.txt @@ -1,15 +1,15 @@ aiohttp==3.9.4 azure-core==1.30.0 -azure-cosmos==4.5.1 +azure-cosmos==4.6.0 azure-eventgrid==4.15.0 azure-identity==1.16.1 azure-mgmt-compute==30.3.0 -azure-mgmt-cosmosdb==9.3.0 +azure-mgmt-cosmosdb==9.4.0 azure-mgmt-costmanagement==4.0.1 azure-mgmt-resource==23.0.1 azure-mgmt-storage==21.1.0 azure-monitor-opentelemetry~=1.6 -azure-servicebus==7.11.3 +azure-servicebus==7.17.5 azure-storage-blob==12.19.0 fastapi==0.110.0 fastapi-utils==0.2.1 @@ -19,7 +19,7 @@ msal==1.26.0 opentelemetry.instrumentation.logging>=0.44b0 pandas==2.0.3 PyJWT==2.8.0 -pytz==2022.7 -python-dateutil==2.8.2 +pytz==2023.4 +python-dateutil==2.9.0 semantic-version==2.10.0 uvicorn[standard]==0.23.2 From ae384dec157befcbfe8e487c3bec93519a0861c6 Mon Sep 17 00:00:00 2001 From: gauravmishrack Date: Mon, 21 Oct 2024 09:45:56 +0000 Subject: [PATCH 08/47] bullseye to bookworm python update --- api_app/Dockerfile | 2 +- resource_processor/vmss_porter/Dockerfile | 2 +- templates/shared_services/certs/Dockerfile.tmpl | 2 +- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api_app/Dockerfile b/api_app/Dockerfile index a510f17688..139815f347 100644 --- a/api_app/Dockerfile +++ b/api_app/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bullseye as base +FROM python:3.11-slim-bookworm as base COPY requirements.txt /. RUN pip3 install --no-cache-dir -r requirements.txt diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index b41181510d..a63f2d3aff 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM python:3.11-slim-bullseye +FROM python:3.11-slim-bookworm SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 623ea42e44..141a4d10ff 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -1,5 +1,5 @@ # syntax=docker/dockerfile-upstream:1.4.0 -FROM --platform=linux/amd64 python:3.11-slim-bullseye +FROM --platform=linux/amd64 python:3.11-slim-bookworm # PORTER_INIT diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index 990083f5f7..ff529fe33b 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-bullseye +FROM python:3.11-bookworm # Install MLflow Python Packages RUN pip install --no-cache-dir psycopg2==2.9.5 mlflow==2.0.1 azure-storage-blob==12.14.1 From ec44e25ce925971a1c7263bfad78c22f547acc40 Mon Sep 17 00:00:00 2001 From: Neosoft Date: Mon, 21 Oct 2024 18:22:23 +0530 Subject: [PATCH 09/47] service bus version update --- api_app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/requirements.txt b/api_app/requirements.txt index a6d209a1bb..56ddce1dc1 100644 --- a/api_app/requirements.txt +++ b/api_app/requirements.txt @@ -9,7 +9,7 @@ azure-mgmt-costmanagement==4.0.1 azure-mgmt-resource==23.0.1 azure-mgmt-storage==21.1.0 azure-monitor-opentelemetry~=1.6 -azure-servicebus==7.17.5 +azure-servicebus==7.12.3 azure-storage-blob==12.19.0 fastapi==0.110.0 fastapi-utils==0.2.1 From ffef58cca8b377db5ad8549211ab0c15c083daba Mon Sep 17 00:00:00 2001 From: Neosoft Date: Mon, 21 Oct 2024 18:45:34 +0530 Subject: [PATCH 10/47] rolledback requirement.txt file --- api_app/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api_app/requirements.txt b/api_app/requirements.txt index 56ddce1dc1..5c773326ec 100644 --- a/api_app/requirements.txt +++ b/api_app/requirements.txt @@ -1,15 +1,15 @@ aiohttp==3.9.4 azure-core==1.30.0 -azure-cosmos==4.6.0 +azure-cosmos==4.5.1 azure-eventgrid==4.15.0 azure-identity==1.16.1 azure-mgmt-compute==30.3.0 -azure-mgmt-cosmosdb==9.4.0 +azure-mgmt-cosmosdb==9.3.0 azure-mgmt-costmanagement==4.0.1 azure-mgmt-resource==23.0.1 azure-mgmt-storage==21.1.0 azure-monitor-opentelemetry~=1.6 -azure-servicebus==7.12.3 +azure-servicebus==7.11.3 azure-storage-blob==12.19.0 fastapi==0.110.0 fastapi-utils==0.2.1 @@ -19,7 +19,7 @@ msal==1.26.0 opentelemetry.instrumentation.logging>=0.44b0 pandas==2.0.3 PyJWT==2.8.0 -pytz==2023.4 -python-dateutil==2.9.0 +pytz==2022.7 +python-dateutil==2.8.2 semantic-version==2.10.0 uvicorn[standard]==0.23.2 From f892564b8886829839ba1174fb6fc40f791e3aa7 Mon Sep 17 00:00:00 2001 From: gauravmishrack Date: Tue, 22 Oct 2024 05:49:12 +0000 Subject: [PATCH 11/47] resource procesor change --- resource_processor/vmss_porter/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index a63f2d3aff..b41181510d 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM python:3.11-slim-bookworm +FROM python:3.11-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-c"] From 50e7d70fc0dea4076cb21dffddf9b08d58ccc906 Mon Sep 17 00:00:00 2001 From: gauravmishrack Date: Tue, 22 Oct 2024 06:01:12 +0000 Subject: [PATCH 12/47] Bookworm to Bullseye Changes --- api_app/Dockerfile | 2 +- templates/shared_services/certs/Dockerfile.tmpl | 2 +- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api_app/Dockerfile b/api_app/Dockerfile index 139815f347..a510f17688 100644 --- a/api_app/Dockerfile +++ b/api_app/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bookworm as base +FROM python:3.11-slim-bullseye as base COPY requirements.txt /. RUN pip3 install --no-cache-dir -r requirements.txt diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 141a4d10ff..623ea42e44 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -1,5 +1,5 @@ # syntax=docker/dockerfile-upstream:1.4.0 -FROM --platform=linux/amd64 python:3.11-slim-bookworm +FROM --platform=linux/amd64 python:3.11-slim-bullseye # PORTER_INIT diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index ff529fe33b..9e076cfbb2 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-bookworm +FROM python:3.11-slim-bullseye # Install MLflow Python Packages RUN pip install --no-cache-dir psycopg2==2.9.5 mlflow==2.0.1 azure-storage-blob==12.14.1 From 6c9e93b2737689b36433f1104a90c1d2c485bbbe Mon Sep 17 00:00:00 2001 From: gauravmishrack Date: Tue, 22 Oct 2024 12:44:01 +0530 Subject: [PATCH 13/47] 3.12 version upgrade for python --- .devcontainer/Dockerfile | 2 +- airlock_processor/Dockerfile | 2 +- api_app/Dockerfile | 2 +- resource_processor/vmss_porter/Dockerfile | 2 +- templates/shared_services/certs/Dockerfile.tmpl | 2 +- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 16c1a303e8..b68f6a09e8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.166.1/containers/python-3/.devcontainer/base.Dockerfile # [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6 -ARG VARIANT="3.11" +ARG VARIANT="3.12" ARG TARGETPLATFORM="linux/amd64" FROM --platform="${TARGETPLATFORM}" mcr.microsoft.com/vscode/devcontainers/python:dev-${VARIANT}-bullseye diff --git a/airlock_processor/Dockerfile b/airlock_processor/Dockerfile index 27885be658..d345faff1a 100644 --- a/airlock_processor/Dockerfile +++ b/airlock_processor/Dockerfile @@ -1,6 +1,6 @@ # To enable ssh & remote debugging on app service change the base image to the one below # FROM mcr.microsoft.com/azure-functions/python:4-python3.8-appservice as base -FROM mcr.microsoft.com/azure-functions/python:4-python3.11-slim as base +FROM mcr.microsoft.com/azure-functions/python:4-python3.12-slim-bullseye as base COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt diff --git a/api_app/Dockerfile b/api_app/Dockerfile index a510f17688..3c6f814100 100644 --- a/api_app/Dockerfile +++ b/api_app/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bullseye as base +FROM python:3.12-slim-bullseye as base COPY requirements.txt /. RUN pip3 install --no-cache-dir -r requirements.txt diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index b41181510d..ddadfdfa11 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM python:3.11-slim-bullseye +FROM python:3.12-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 623ea42e44..baff9db2bf 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -1,5 +1,5 @@ # syntax=docker/dockerfile-upstream:1.4.0 -FROM --platform=linux/amd64 python:3.11-slim-bullseye +FROM --platform=linux/amd64 python:3.12-slim-bullseye # PORTER_INIT diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index 9e076cfbb2..02ff09f37d 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bullseye +FROM python:3.12slim-bullseye # Install MLflow Python Packages RUN pip install --no-cache-dir psycopg2==2.9.5 mlflow==2.0.1 azure-storage-blob==12.14.1 From b50090e44992ad70849a909eb7401f3ac263b5c2 Mon Sep 17 00:00:00 2001 From: gauravmishrack Date: Tue, 22 Oct 2024 07:55:38 +0000 Subject: [PATCH 14/47] airlockfiles changes --- airlock_processor/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airlock_processor/Dockerfile b/airlock_processor/Dockerfile index d345faff1a..1b2ab0de6d 100644 --- a/airlock_processor/Dockerfile +++ b/airlock_processor/Dockerfile @@ -1,6 +1,6 @@ # To enable ssh & remote debugging on app service change the base image to the one below # FROM mcr.microsoft.com/azure-functions/python:4-python3.8-appservice as base -FROM mcr.microsoft.com/azure-functions/python:4-python3.12-slim-bullseye as base +FROM mcr.microsoft.com/azure-functions/python:4-python3.12-slim as base COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt From 72a52e8c70fc85db29f506c1fdaf1ed332fc2166 Mon Sep 17 00:00:00 2001 From: gauravmishrack Date: Tue, 22 Oct 2024 08:17:55 +0000 Subject: [PATCH 15/47] mlflow changes --- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index 02ff09f37d..cda806cf26 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12slim-bullseye +FROM python:3.12-slim-bullseye # Install MLflow Python Packages RUN pip install --no-cache-dir psycopg2==2.9.5 mlflow==2.0.1 azure-storage-blob==12.14.1 From 9388e3ef49016367429c46ebb28d0d3137310f9e Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 11:33:29 +0000 Subject: [PATCH 16/47] remove stuff --- AzureTRE | 1 - airlock_processor/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 160000 AzureTRE diff --git a/AzureTRE b/AzureTRE deleted file mode 160000 index 400766bd39..0000000000 --- a/AzureTRE +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 400766bd39ff6b5216c24679dfeb4fb1a2252076 diff --git a/airlock_processor/Dockerfile b/airlock_processor/Dockerfile index 1b2ab0de6d..a2db28c349 100644 --- a/airlock_processor/Dockerfile +++ b/airlock_processor/Dockerfile @@ -1,6 +1,6 @@ # To enable ssh & remote debugging on app service change the base image to the one below # FROM mcr.microsoft.com/azure-functions/python:4-python3.8-appservice as base -FROM mcr.microsoft.com/azure-functions/python:4-python3.12-slim as base +FROM mcr.microsoft.com/azure-functions/python:4-python3.12 as base COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt From a66f7dcf12e41a6727a275ea75d376d09d4dd25f Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 11:34:55 +0000 Subject: [PATCH 17/47] add in 'the' --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 33601795a5..9d49130458 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,8 +10,8 @@ "context": "..", "dockerfile": "Dockerfile", "args": { - // To ensure that the group ID for the docker group in container - // matches the group ID on the host, add this to your .bash_profile on host + // To ensure that the group ID for the docker group in the container + // matches the group ID on the host, add this to your .bash_profile on the host // export DOCKER_GROUP_ID=$(getent group docker | awk -F ":" '{ print $3 }') "DOCKER_GROUP_ID": "${localEnv:DOCKER_GROUP_ID}", "INTERACTIVE": "true" From 25b54228b1919b0b1209d52a3f2f08611bae90aa Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 11:36:53 +0000 Subject: [PATCH 18/47] update --- airlock_processor/_version.py | 2 +- api_app/_version.py | 2 +- resource_processor/_version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airlock_processor/_version.py b/airlock_processor/_version.py index bc8c296f6a..777f190df0 100644 --- a/airlock_processor/_version.py +++ b/airlock_processor/_version.py @@ -1 +1 @@ -__version__ = "0.7.2" +__version__ = "0.8.0" diff --git a/api_app/_version.py b/api_app/_version.py index aa070c2c9c..5f4bb0b345 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.19.2" +__version__ = "0.20.0" diff --git a/resource_processor/_version.py b/resource_processor/_version.py index a2fecb4576..61fb31cae0 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.9.2" +__version__ = "0.10.0" From 8c0ea440a8a850e550bbc2c06ff7031ee1ce6a0b Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 11:42:40 +0000 Subject: [PATCH 19/47] update requirements --- airlock_processor/requirements-dev.txt | 4 +- airlock_processor/requirements.txt | 12 +++--- api_app/requirements-dev.txt | 6 +-- api_app/requirements.txt | 40 +++++++++---------- .../vmss_porter/requirements.txt | 8 ++-- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/airlock_processor/requirements-dev.txt b/airlock_processor/requirements-dev.txt index 3bec256653..062d9aa5c0 100644 --- a/airlock_processor/requirements-dev.txt +++ b/airlock_processor/requirements-dev.txt @@ -1,3 +1,3 @@ # Dev requirements -pytest==7.2.0 -mock==5.0.0 +pytest==8.3.3 +mock==5.1.0 diff --git a/airlock_processor/requirements.txt b/airlock_processor/requirements.txt index 0b984bab32..ff75b1615b 100644 --- a/airlock_processor/requirements.txt +++ b/airlock_processor/requirements.txt @@ -1,8 +1,8 @@ # Do not include azure-functions-worker as it may conflict with the Azure Functions platform -azure-core==1.30.0 +azure-core==1.31.0 azure-functions==1.17.0 -azure-storage-blob==12.19.0 -azure-identity==1.16.1 -azure-mgmt-storage==21.1.0 -azure-mgmt-resource==23.0.1 -pydantic==1.10.13 +azure-storage-blob==12.23.1 +azure-identity==1.19.0 +azure-mgmt-storage==21.2.1 +azure-mgmt-resource==23.2.0 +pydantic==2.9.2 diff --git a/api_app/requirements-dev.txt b/api_app/requirements-dev.txt index b02905b89a..b9d1190cf0 100644 --- a/api_app/requirements-dev.txt +++ b/api_app/requirements-dev.txt @@ -1,5 +1,5 @@ # Dev requirements -pytest-asyncio==0.21.1 -httpx==0.25.0 +pytest-asyncio==0.24.0 +httpx==0.27.2 mock==5.1.0 -pytest==7.4.3 +pytest==8.3.3 diff --git a/api_app/requirements.txt b/api_app/requirements.txt index 5c773326ec..3271a1411e 100644 --- a/api_app/requirements.txt +++ b/api_app/requirements.txt @@ -1,25 +1,25 @@ -aiohttp==3.9.4 -azure-core==1.30.0 -azure-cosmos==4.5.1 -azure-eventgrid==4.15.0 -azure-identity==1.16.1 -azure-mgmt-compute==30.3.0 -azure-mgmt-cosmosdb==9.3.0 +aiohttp==3.10.10 +azure-core==1.31.0 +azure-cosmos==4.7.0 +azure-eventgrid==4.20.0 +azure-identity==1.19.0 +azure-mgmt-compute==33.0.0 +azure-mgmt-cosmosdb==9.6.0 azure-mgmt-costmanagement==4.0.1 -azure-mgmt-resource==23.0.1 -azure-mgmt-storage==21.1.0 +azure-mgmt-resource==23.2.0 +azure-mgmt-storage==21.2.1 azure-monitor-opentelemetry~=1.6 -azure-servicebus==7.11.3 -azure-storage-blob==12.19.0 -fastapi==0.110.0 -fastapi-utils==0.2.1 -gunicorn==22.0.0 +azure-servicebus==7.12.3 +azure-storage-blob==12.23.1 +fastapi==0.115.3 +fastapi-utils==0.7.0 +gunicorn==23.0.0 jsonschema[format_nongpl]==4.19.1 -msal==1.26.0 +msal==1.31.0 opentelemetry.instrumentation.logging>=0.44b0 -pandas==2.0.3 -PyJWT==2.8.0 -pytz==2022.7 -python-dateutil==2.8.2 +pandas==2.2.3 +PyJWT==2.9.0 +pytz==2024.2 +python-dateutil==2.9.0 semantic-version==2.10.0 -uvicorn[standard]==0.23.2 +uvicorn[standard]==0.31.0 diff --git a/resource_processor/vmss_porter/requirements.txt b/resource_processor/vmss_porter/requirements.txt index 6c2771ffde..fcc4415807 100644 --- a/resource_processor/vmss_porter/requirements.txt +++ b/resource_processor/vmss_porter/requirements.txt @@ -1,6 +1,6 @@ -aiohttp==3.9.4 -azure-cli-core==2.57.0 -azure-identity==1.16.1 +aiohttp==3.10.10 +azure-cli-core==2.65.0 +azure-identity==1.19.0 azure-monitor-opentelemetry~=1.6 -azure-servicebus==7.11.3 +azure-servicebus==7.12.3 opentelemetry.instrumentation.logging>=0.44b0 From ad445e8d9a81d5dbcf9017c6041814a36fdb6173 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 11:49:53 +0000 Subject: [PATCH 20/47] updaet --- api_app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/requirements.txt b/api_app/requirements.txt index 3271a1411e..a208a31b19 100644 --- a/api_app/requirements.txt +++ b/api_app/requirements.txt @@ -17,7 +17,7 @@ gunicorn==23.0.0 jsonschema[format_nongpl]==4.19.1 msal==1.31.0 opentelemetry.instrumentation.logging>=0.44b0 -pandas==2.2.3 +pandas==2.2.2 PyJWT==2.9.0 pytz==2024.2 python-dateutil==2.9.0 From ccfb11d7c8a8d7a5338ce39047279000cbbc2518 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 12:03:46 +0000 Subject: [PATCH 21/47] update --- api_app/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/api_app/requirements.txt b/api_app/requirements.txt index a208a31b19..af38ff9ba6 100644 --- a/api_app/requirements.txt +++ b/api_app/requirements.txt @@ -22,4 +22,5 @@ PyJWT==2.9.0 pytz==2024.2 python-dateutil==2.9.0 semantic-version==2.10.0 +setuptools==75.3.0 uvicorn[standard]==0.31.0 From fadd855ec30f8fa7aeead6396dd93920a8343f29 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 12:14:03 +0000 Subject: [PATCH 22/47] update --- api_app/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/requirements-dev.txt b/api_app/requirements-dev.txt index b9d1190cf0..824dd2c0f8 100644 --- a/api_app/requirements-dev.txt +++ b/api_app/requirements-dev.txt @@ -1,5 +1,5 @@ # Dev requirements pytest-asyncio==0.24.0 -httpx==0.27.2 +httpx==0.25.0 mock==5.1.0 pytest==8.3.3 From 0b143cdaa81634ed8e3ce0c7b3bb2ccffa83636a Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 12:23:12 +0000 Subject: [PATCH 23/47] update pytest versions --- e2e_tests/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e_tests/requirements.txt b/e2e_tests/requirements.txt index 7587b0d58a..0fa7a2a927 100644 --- a/e2e_tests/requirements.txt +++ b/e2e_tests/requirements.txt @@ -1,7 +1,7 @@ # API httpx==0.25.0 -pytest==7.4.3 -pytest-asyncio==0.21.1 +pytest==8.3.3 +pytest-asyncio==0.24.0 starlette==0.36.3 pytest-timeout==2.2.0 pytest-xdist==3.3.1 From 5ca4af3c0fc771f4144b11265919d32f35b60718 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 12:26:38 +0000 Subject: [PATCH 24/47] update starlette --- e2e_tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_tests/requirements.txt b/e2e_tests/requirements.txt index 0fa7a2a927..9e16ce1bc8 100644 --- a/e2e_tests/requirements.txt +++ b/e2e_tests/requirements.txt @@ -2,7 +2,7 @@ httpx==0.25.0 pytest==8.3.3 pytest-asyncio==0.24.0 -starlette==0.36.3 +starlette==0.41.2 pytest-timeout==2.2.0 pytest-xdist==3.3.1 backoff==2.2.1 From 679c468815ebe0d4cef9b3517f8a9295d53c1101 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 13:24:19 +0000 Subject: [PATCH 25/47] update --- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index cda806cf26..692dd76599 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.12-slim-bullseye # Install MLflow Python Packages -RUN pip install --no-cache-dir psycopg2==2.9.5 mlflow==2.0.1 azure-storage-blob==12.14.1 +RUN pip install --no-cache-dir psycopg2==2.9.10 mlflow==2.17.2 azure-storage-blob==12.23.1 RUN apt-get update \ && apt-get install openssh-server -y --no-install-recommends \ From fb7c31ed0b3ce4fc2c32d8330c5fbdc3cad9978d Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 5 Nov 2024 13:56:49 +0000 Subject: [PATCH 26/47] update mlflow --- .../workspace_services/mlflow/mlflow-server/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile index 692dd76599..53595fb1e3 100644 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.12-slim-bullseye # Install MLflow Python Packages -RUN pip install --no-cache-dir psycopg2==2.9.10 mlflow==2.17.2 azure-storage-blob==12.23.1 +RUN pip install --no-cache-dir psycopg2-binary==2.9.10 mlflow==2.17.2 azure-storage-blob==12.23.1 RUN apt-get update \ && apt-get install openssh-server -y --no-install-recommends \ From 9972ef758c48473135c0d2031806499125231e1c Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Wed, 6 Nov 2024 10:58:10 +0000 Subject: [PATCH 27/47] update --- api_app/Dockerfile | 8 ++++---- api_app/_version.py | 2 +- api_app/models/domain/airlock_operations.py | 3 +-- api_app/models/domain/airlock_request.py | 9 ++++----- api_app/models/domain/azuretremodel.py | 4 ++-- api_app/models/domain/resource.py | 4 ++-- api_app/models/domain/shared_service.py | 2 +- api_app/models/domain/user_resource.py | 2 +- api_app/models/domain/user_resource_template.py | 2 +- api_app/models/domain/workspace.py | 2 +- api_app/models/domain/workspace_service.py | 2 +- 11 files changed, 19 insertions(+), 21 deletions(-) diff --git a/api_app/Dockerfile b/api_app/Dockerfile index 3c6f814100..bc39dba7c5 100644 --- a/api_app/Dockerfile +++ b/api_app/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.12-slim-bullseye as base +FROM python:3.12-slim-bullseye AS base COPY requirements.txt /. RUN pip3 install --no-cache-dir -r requirements.txt -FROM base as test +FROM base AS test COPY requirements-dev.txt /. RUN pip3 install --no-cache-dir -r requirements-dev.txt COPY . /api @@ -10,11 +10,11 @@ WORKDIR /api RUN ./run_tests_and_exit_succesfully.sh -FROM scratch as test-results +FROM scratch AS test-results COPY --from=test /test-results/* /. -FROM base as runtime +FROM base AS runtime COPY . /api WORKDIR /api RUN groupadd -r api && useradd -r -s /bin/false -g api api_user diff --git a/api_app/_version.py b/api_app/_version.py index 5f4bb0b345..ac82404646 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.20.0" +__version__ = "0.20.1" diff --git a/api_app/models/domain/airlock_operations.py b/api_app/models/domain/airlock_operations.py index be447f3961..eda5d6f494 100644 --- a/api_app/models/domain/airlock_operations.py +++ b/api_app/models/domain/airlock_operations.py @@ -1,8 +1,7 @@ from pydantic import Field from pydantic.types import UUID4 -from pydantic.schema import Optional from models.domain.azuretremodel import AzureTREModel -from typing import List +from typing import List, Optional from models.domain.airlock_request import AirlockFile diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py index 14d6901b6e..d56bf3b756 100644 --- a/api_app/models/domain/airlock_request.py +++ b/api_app/models/domain/airlock_request.py @@ -1,9 +1,8 @@ from enum import Enum -from typing import List, Dict +from typing import List, Dict, Optional from models.domain.azuretremodel import AzureTREModel -from pydantic import Field, validator -from pydantic.schema import Optional +from pydantic import Field, field_validator from resources import strings @@ -92,14 +91,14 @@ class AirlockRequest(AzureTREModel): files: List[AirlockFile] = Field([], title="Files of the request") title: str = Field("Airlock Request", title="Brief title for the request") businessJustification: str = Field("Business Justification", title="Explanation that will be provided to the request reviewer") - status = AirlockRequestStatus.Draft + status: AirlockRequestStatus = AirlockRequestStatus.Draft statusMessage: Optional[str] = Field(title="Optional - contains additional information about the current status.") reviews: Optional[List[AirlockReview]] etag: Optional[str] = Field(title="_etag", alias="_etag") reviewUserResources: Dict[str, AirlockReviewUserResource] = Field({}, title="User resources created for Airlock Reviews") # SQL API CosmosDB saves ETag as an escaped string: https://github.com/microsoft/AzureTRE/issues/1931 - @validator("etag", pre=True) + @field_validator("etag") def parse_etag_to_remove_escaped_quotes(cls, value): if value: return value.replace('\"', '') diff --git a/api_app/models/domain/azuretremodel.py b/api_app/models/domain/azuretremodel.py index dd7dde690c..4f24bb2098 100644 --- a/api_app/models/domain/azuretremodel.py +++ b/api_app/models/domain/azuretremodel.py @@ -1,7 +1,7 @@ -from pydantic import BaseConfig, BaseModel +from pydantic import ConfigDict, BaseModel class AzureTREModel(BaseModel): - class Config(BaseConfig): + class Config(ConfigDict): allow_population_by_field_name = True arbitrary_types_allowed = True diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index 36485fb07f..a47eafe9c3 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -1,6 +1,6 @@ from enum import Enum from typing import Optional, Union, List -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, Field, field_validator from models.domain.azuretremodel import AzureTREModel from models.domain.request_action import RequestAction from resources import strings @@ -76,7 +76,7 @@ def get_resource_request_message_payload(self, operation_id: str, step_id: str, # SQL API CosmosDB saves etag as an escaped string by default, with no apparent way to change it. # Removing escaped quotes on pydantic deserialization. https://github.com/microsoft/AzureTRE/issues/1931 - @validator("etag", pre=True) + @field_validator("etag") def parse_etag_to_remove_escaped_quotes(cls, value): return value.replace('\"', '') diff --git a/api_app/models/domain/shared_service.py b/api_app/models/domain/shared_service.py index 4ec14f26c2..2ce76665ab 100644 --- a/api_app/models/domain/shared_service.py +++ b/api_app/models/domain/shared_service.py @@ -5,4 +5,4 @@ class SharedService(Resource): """ Shared service request """ - resourceType = ResourceType.SharedService + resourceType: ResourceType = ResourceType.SharedService diff --git a/api_app/models/domain/user_resource.py b/api_app/models/domain/user_resource.py index 8b4706abc4..08a415c9dc 100644 --- a/api_app/models/domain/user_resource.py +++ b/api_app/models/domain/user_resource.py @@ -11,4 +11,4 @@ class UserResource(Resource): ownerId: str = Field("", title="Owner of the user resource") parentWorkspaceServiceId: str = Field("", title="Parent Workspace Service ID", description="Service target Workspace Service id") azureStatus: dict = Field({}, title="Azure Status", description="Azure status, varies per user resource") - resourceType = ResourceType.UserResource + resourceType: ResourceType = ResourceType.UserResource diff --git a/api_app/models/domain/user_resource_template.py b/api_app/models/domain/user_resource_template.py index 131aa408bb..d5aa4d3bba 100644 --- a/api_app/models/domain/user_resource_template.py +++ b/api_app/models/domain/user_resource_template.py @@ -6,4 +6,4 @@ class UserResourceTemplate(ResourceTemplate): parentWorkspaceService: str = Field("", title="Parent Workspace Service", description="The parent workspace service under which services with this template can be created") - resourceType = ResourceType.UserResource + resourceType: ResourceType = ResourceType.UserResource diff --git a/api_app/models/domain/workspace.py b/api_app/models/domain/workspace.py index 407afd6d1e..9a041a13fe 100644 --- a/api_app/models/domain/workspace.py +++ b/api_app/models/domain/workspace.py @@ -16,7 +16,7 @@ class Workspace(Resource): Workspace request """ workspaceURL: str = Field("", title="Workspace URL", description="Main endpoint for workspace users") - resourceType = ResourceType.Workspace + resourceType: ResourceType = ResourceType.Workspace class WorkspaceAuth(AzureTREModel): diff --git a/api_app/models/domain/workspace_service.py b/api_app/models/domain/workspace_service.py index 8f9472b894..1ef32a4c22 100644 --- a/api_app/models/domain/workspace_service.py +++ b/api_app/models/domain/workspace_service.py @@ -8,4 +8,4 @@ class WorkspaceService(Resource): Workspace service request """ workspaceId: str = Field("", title="Workspace ID", description="Service target Workspace id") - resourceType = ResourceType.WorkspaceService + resourceType: ResourceType = ResourceType.WorkspaceService From 543110f9727ec3a0db8a045ae5256a8048f85323 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Wed, 6 Nov 2024 15:29:55 +0000 Subject: [PATCH 28/47] update vmss_porter --- api_app/_version.py | 2 +- api_app/db/repositories/resource_templates.py | 16 ++++++++-------- core/terraform/.terraform.lock.hcl | 3 ++- resource_processor/vmss_porter/Dockerfile | 10 +++++----- .../base/terraform/.terraform.lock.hcl | 3 ++- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/api_app/_version.py b/api_app/_version.py index ac82404646..be6f6da961 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.20.1" +__version__ = "0.20.2" diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index 66674e7cf0..a43fceeadd 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -1,7 +1,7 @@ import uuid from typing import List, Optional, Union -from pydantic import parse_obj_as +from pydantic import TypeAdapter from core import config from db.errors import DuplicateEntity, EntityDoesNotExist, EntityVersionExist, InvalidInput @@ -46,7 +46,7 @@ async def get_templates_information(self, resource_type: ResourceType, user_role if resource_type == ResourceType.UserResource: query += f' AND c.parentWorkspaceService = "{parent_service_name}"' template_infos = await self.query(query=query) - templates = [parse_obj_as(ResourceTemplateInformation, info) for info in template_infos] + templates = [TypeAdapter(ResourceTemplateInformation).validate_python(info) for info in template_infos] if not user_roles: return templates @@ -66,9 +66,9 @@ async def get_current_template(self, template_name: str, resource_type: Resource if len(templates) > 1: raise DuplicateEntity if resource_type == ResourceType.UserResource: - return parse_obj_as(UserResourceTemplate, templates[0]) + return TypeAdapter(UserResourceTemplate).validate_python(templates[0]) else: - return parse_obj_as(ResourceTemplate, templates[0]) + return TypeAdapter(ResourceTemplate).validate_python(templates[0]) async def get_template_by_name_and_version(self, name: str, version: str, resource_type: ResourceType, parent_service_name: Optional[str] = None) -> Union[ResourceTemplate, UserResourceTemplate]: """ @@ -90,9 +90,9 @@ async def get_template_by_name_and_version(self, name: str, version: str, resour if len(templates) != 1: raise EntityDoesNotExist if resource_type == ResourceType.UserResource: - return parse_obj_as(UserResourceTemplate, templates[0]) + return TypeAdapter(UserResourceTemplate).validate_python(templates[0]) else: - return parse_obj_as(ResourceTemplate, templates[0]) + return TypeAdapter(ResourceTemplate).validate_python(templates[0]) async def get_all_template_versions(self, template_name: str) -> List[str]: query = 'SELECT VALUE c.version FROM c where c.name = @template_name' @@ -131,9 +131,9 @@ async def create_template(self, template_input: ResourceTemplateInCreate, resour if resource_type == ResourceType.UserResource: template["parentWorkspaceService"] = parent_service_name - template = parse_obj_as(UserResourceTemplate, template) + return TypeAdapter(UserResourceTemplate).validate_python(template) else: - template = parse_obj_as(ResourceTemplate, template) + return TypeAdapter(ResourceTemplate).validate_python(template) await self.save_item(template) return template diff --git a/core/terraform/.terraform.lock.hcl b/core/terraform/.terraform.lock.hcl index 0539020952..bc7c2d0508 100644 --- a/core/terraform/.terraform.lock.hcl +++ b/core/terraform/.terraform.lock.hcl @@ -3,8 +3,9 @@ provider "registry.terraform.io/azure/azapi" { version = "1.15.0" - constraints = "1.15.0" + constraints = ">= 1.15.0, ~> 1.15.0" hashes = [ + "h1:Y7ruMuPh8UJRTRl4rm+cdpGtmURx2taqiuqfYaH3o48=", "h1:gIOgxVmFSxHrR+XOzgUEA+ybOmp8kxZlZH3eYeB/eFI=", "zh:0627a8bc77254debc25dc0c7b62e055138217c97b03221e593c3c56dc7550671", "zh:2fe045f07070ef75d0bec4b0595a74c14394daa838ddb964e2fd23cc98c40c34", diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index ddadfdfa11..320e60d664 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -6,7 +6,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache # Install Azure CLI -ARG AZURE_CLI_VERSION=2.50.0-1~bullseye +ARG AZURE_CLI_VERSION=2.65.0-1~bullseye COPY scripts/azure-cli.sh /tmp/ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \ @@ -34,10 +34,10 @@ ENV PORTER_HOME_V0 ${PORTER_HOME} # can't be in a non default path # ARG PORTER_HOME_V1=/home/$USERNAME/.porter-v1/ ARG PORTER_HOME_V1=/root/.porter/ -ARG PORTER_VERSION=v1.0.15 -ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.2 -ARG PORTER_AZ_MIXIN_VERSION=v1.0.1 -ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.0 +ARG PORTER_VERSION=v1.1.1 +ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.3 +ARG PORTER_AZ_MIXIN_VERSION=v1.0.2 +ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.3 COPY scripts/porter-v1.sh /tmp/ RUN export PORTER_VERSION=${PORTER_VERSION} \ PORTER_TERRAFORM_MIXIN_VERSION=${PORTER_TERRAFORM_MIXIN_VERSION} \ diff --git a/templates/workspaces/base/terraform/.terraform.lock.hcl b/templates/workspaces/base/terraform/.terraform.lock.hcl index 3b9b428015..4d6d7dab37 100644 --- a/templates/workspaces/base/terraform/.terraform.lock.hcl +++ b/templates/workspaces/base/terraform/.terraform.lock.hcl @@ -3,8 +3,9 @@ provider "registry.terraform.io/azure/azapi" { version = "1.15.0" - constraints = "1.15.0" + constraints = ">= 1.15.0, 1.15.0" hashes = [ + "h1:Y7ruMuPh8UJRTRl4rm+cdpGtmURx2taqiuqfYaH3o48=", "h1:gIOgxVmFSxHrR+XOzgUEA+ybOmp8kxZlZH3eYeB/eFI=", "zh:0627a8bc77254debc25dc0c7b62e055138217c97b03221e593c3c56dc7550671", "zh:2fe045f07070ef75d0bec4b0595a74c14394daa838ddb964e2fd23cc98c40c34", From f8aeaa0788731672f1e3f28c58fb29f1277c17f8 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Wed, 6 Nov 2024 15:44:26 +0000 Subject: [PATCH 29/47] update --- core/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/version.txt b/core/version.txt index 47f57bfd12..a37fec72b0 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.10.11" +__version__ = "0.10.12" From ca179c00ffc76884138936ba96fb585d27320a92 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Thu, 7 Nov 2024 11:39:53 +0000 Subject: [PATCH 30/47] update core version --- core/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/version.txt b/core/version.txt index a37fec72b0..f63b48ab36 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.10.12" +__version__ = "0.10.13" From a8e7452298043ad8dc559524986c92eb3a4a2c1a Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Fri, 8 Nov 2024 11:43:28 +0000 Subject: [PATCH 31/47] python updates --- airlock_processor/requirements.txt | 1 + resource_processor/vmss_porter/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/airlock_processor/requirements.txt b/airlock_processor/requirements.txt index ff75b1615b..249dcde1a3 100644 --- a/airlock_processor/requirements.txt +++ b/airlock_processor/requirements.txt @@ -6,3 +6,4 @@ azure-identity==1.19.0 azure-mgmt-storage==21.2.1 azure-mgmt-resource==23.2.0 pydantic==2.9.2 +setuptools==75.3.0 diff --git a/resource_processor/vmss_porter/requirements.txt b/resource_processor/vmss_porter/requirements.txt index fcc4415807..b53e65aba8 100644 --- a/resource_processor/vmss_porter/requirements.txt +++ b/resource_processor/vmss_porter/requirements.txt @@ -4,3 +4,4 @@ azure-identity==1.19.0 azure-monitor-opentelemetry~=1.6 azure-servicebus==7.12.3 opentelemetry.instrumentation.logging>=0.44b0 +setuptools==75.3.0 From 3ce4e29a464ffea0a042906b73b442e1ed34f348 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 11 Nov 2024 12:34:09 +0000 Subject: [PATCH 32/47] update StrEnum --- api_app/db/repositories/resource_templates.py | 16 ++++++++-------- api_app/models/domain/airlock_request.py | 12 ++++++------ api_app/models/domain/costs.py | 6 +++--- api_app/models/domain/operation.py | 4 ++-- api_app/models/domain/request_action.py | 4 ++-- api_app/models/domain/resource.py | 10 +++++----- api_app/models/schemas/status.py | 4 ++-- api_app/models/schemas/workspace.py | 4 ++-- resource_processor/vmss_porter/runner.py | 2 +- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index a43fceeadd..66674e7cf0 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -1,7 +1,7 @@ import uuid from typing import List, Optional, Union -from pydantic import TypeAdapter +from pydantic import parse_obj_as from core import config from db.errors import DuplicateEntity, EntityDoesNotExist, EntityVersionExist, InvalidInput @@ -46,7 +46,7 @@ async def get_templates_information(self, resource_type: ResourceType, user_role if resource_type == ResourceType.UserResource: query += f' AND c.parentWorkspaceService = "{parent_service_name}"' template_infos = await self.query(query=query) - templates = [TypeAdapter(ResourceTemplateInformation).validate_python(info) for info in template_infos] + templates = [parse_obj_as(ResourceTemplateInformation, info) for info in template_infos] if not user_roles: return templates @@ -66,9 +66,9 @@ async def get_current_template(self, template_name: str, resource_type: Resource if len(templates) > 1: raise DuplicateEntity if resource_type == ResourceType.UserResource: - return TypeAdapter(UserResourceTemplate).validate_python(templates[0]) + return parse_obj_as(UserResourceTemplate, templates[0]) else: - return TypeAdapter(ResourceTemplate).validate_python(templates[0]) + return parse_obj_as(ResourceTemplate, templates[0]) async def get_template_by_name_and_version(self, name: str, version: str, resource_type: ResourceType, parent_service_name: Optional[str] = None) -> Union[ResourceTemplate, UserResourceTemplate]: """ @@ -90,9 +90,9 @@ async def get_template_by_name_and_version(self, name: str, version: str, resour if len(templates) != 1: raise EntityDoesNotExist if resource_type == ResourceType.UserResource: - return TypeAdapter(UserResourceTemplate).validate_python(templates[0]) + return parse_obj_as(UserResourceTemplate, templates[0]) else: - return TypeAdapter(ResourceTemplate).validate_python(templates[0]) + return parse_obj_as(ResourceTemplate, templates[0]) async def get_all_template_versions(self, template_name: str) -> List[str]: query = 'SELECT VALUE c.version FROM c where c.name = @template_name' @@ -131,9 +131,9 @@ async def create_template(self, template_input: ResourceTemplateInCreate, resour if resource_type == ResourceType.UserResource: template["parentWorkspaceService"] = parent_service_name - return TypeAdapter(UserResourceTemplate).validate_python(template) + template = parse_obj_as(UserResourceTemplate, template) else: - return TypeAdapter(ResourceTemplate).validate_python(template) + template = parse_obj_as(ResourceTemplate, template) await self.save_item(template) return template diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py index d56bf3b756..9cef0792f5 100644 --- a/api_app/models/domain/airlock_request.py +++ b/api_app/models/domain/airlock_request.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import List, Dict, Optional from models.domain.azuretremodel import AzureTREModel @@ -6,7 +6,7 @@ from resources import strings -class AirlockRequestStatus(str, Enum): +class AirlockRequestStatus(StrEnum): """ Airlock Resource status """ @@ -24,12 +24,12 @@ class AirlockRequestStatus(str, Enum): Failed = strings.AIRLOCK_RESOURCE_STATUS_FAILED -class AirlockRequestType(str, Enum): +class AirlockRequestType(StrEnum): Import = strings.AIRLOCK_REQUEST_TYPE_IMPORT Export = strings.AIRLOCK_REQUEST_TYPE_EXPORT -class AirlockActions(str, Enum): +class AirlockActions(StrEnum): Review = strings.AIRLOCK_ACTION_REVIEW Cancel = strings.AIRLOCK_ACTION_CANCEL Submit = strings.AIRLOCK_ACTION_SUBMIT @@ -40,7 +40,7 @@ class AirlockFile(AzureTREModel): size: float = Field(title="size", description="size of the file in bytes") -class AirlockReviewDecision(str, Enum): +class AirlockReviewDecision(StrEnum): Approved = strings.AIRLOCK_REVIEW_DECISION_APPROVED Rejected = strings.AIRLOCK_REVIEW_DECISION_REJECTED @@ -91,7 +91,7 @@ class AirlockRequest(AzureTREModel): files: List[AirlockFile] = Field([], title="Files of the request") title: str = Field("Airlock Request", title="Brief title for the request") businessJustification: str = Field("Business Justification", title="Explanation that will be provided to the request reviewer") - status: AirlockRequestStatus = AirlockRequestStatus.Draft + status: str = AirlockRequestStatus.Draft statusMessage: Optional[str] = Field(title="Optional - contains additional information about the current status.") reviews: Optional[List[AirlockReview]] etag: Optional[str] = Field(title="_etag", alias="_etag") diff --git a/api_app/models/domain/costs.py b/api_app/models/domain/costs.py index a484dd86eb..192454b177 100644 --- a/api_app/models/domain/costs.py +++ b/api_app/models/domain/costs.py @@ -1,17 +1,17 @@ from datetime import datetime, timedelta, date from typing import List, Optional from pydantic import BaseModel -from enum import Enum +from enum import StrEnum import random import uuid -class GranularityEnum(str, Enum): +class GranularityEnum(StrEnum): daily = "Daily" none = "None" -class CurrencyEnum(str, Enum): +class CurrencyEnum(StrEnum): USD = "USD" ILS = "ILS" diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 1577eaaa61..f8c2dd36dc 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import List, Optional from pydantic import Field @@ -9,7 +9,7 @@ from resources import strings -class Status(str, Enum): +class Status(StrEnum): """ Operation status """ diff --git a/api_app/models/domain/request_action.py b/api_app/models/domain/request_action.py index f52994fa8b..a325dc4621 100644 --- a/api_app/models/domain/request_action.py +++ b/api_app/models/domain/request_action.py @@ -1,7 +1,7 @@ -from enum import Enum +from enum import StrEnum -class RequestAction(str, Enum): +class RequestAction(StrEnum): Install = "install" UnInstall = "uninstall" Upgrade = "upgrade" diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index a47eafe9c3..f4484fa1a9 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import Optional, Union, List from pydantic import BaseModel, Field, field_validator from models.domain.azuretremodel import AzureTREModel @@ -6,7 +6,7 @@ from resources import strings -class ResourceType(str, Enum): +class ResourceType(StrEnum): """ Type of resource to deploy """ @@ -43,11 +43,11 @@ class Resource(AzureTREModel): templateName: str = Field(title="Resource template name", description="The resource template (bundle) to deploy") templateVersion: str = Field(title="Resource template version", description="The version of the resource template (bundle) to deploy") properties: dict = Field({}, title="Resource template parameters", description="Parameters for the deployment") - availableUpgrades: Optional[List[AvailableUpgrade]] = Field(title="Available template upgrades", description="Versions of the template that are available for upgrade") + availableUpgrades: Optional[List[AvailableUpgrade]] = Field([], title="Available template upgrades", description="Versions of the template that are available for upgrade") isEnabled: bool = True # Must be set before a resource can be deleted resourceType: ResourceType - deploymentStatus: Optional[str] = Field(title="Deployment Status", description="Overall deployment status of the resource") - etag: str = Field(title="_etag", description="eTag of the document", alias="_etag") + deploymentStatus: Optional[str] = Field("", title="Deployment Status", description="Overall deployment status of the resource") + etag: str = Field("", title="_etag", description="eTag of the document", alias="_etag") resourcePath: str = "" resourceVersion: int = 0 user: dict = {} diff --git a/api_app/models/schemas/status.py b/api_app/models/schemas/status.py index 392bce2311..c274e69cf0 100644 --- a/api_app/models/schemas/status.py +++ b/api_app/models/schemas/status.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import List from pydantic import BaseModel @@ -6,7 +6,7 @@ from resources import strings -class StatusEnum(str, Enum): +class StatusEnum(StrEnum): ok = strings.OK not_ok = strings.NOT_OK diff --git a/api_app/models/schemas/workspace.py b/api_app/models/schemas/workspace.py index 424c82b46a..94c6bf861e 100644 --- a/api_app/models/schemas/workspace.py +++ b/api_app/models/schemas/workspace.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import List from pydantic import BaseModel, Field @@ -23,7 +23,7 @@ def get_sample_workspace(workspace_id: str, spec_workspace_id: str = "0001") -> } -class AuthProvider(str, Enum): +class AuthProvider(StrEnum): """ Auth Provider """ diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index 12a12bdbfe..3de4ac06fe 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -289,4 +289,4 @@ async def check_runners(processes: list, httpserver: Process): asyncio.run(check_runners(processes, httpserver)) - logger.warn("Exiting main...") + logger.warning("Exiting main...") From 7dd21e4e77fec9615aab06a6222cbaddc84e732f Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 12 Nov 2024 09:04:15 +0000 Subject: [PATCH 33/47] fix api messages --- api_app/models/domain/azuretremodel.py | 2 +- api_app/models/domain/resource.py | 8 ++++---- api_app/models/schemas/airlock_request.py | 12 ++++++------ api_app/models/schemas/airlock_request_url.py | 2 +- api_app/models/schemas/operation.py | 4 ++-- api_app/models/schemas/resource.py | 4 ++-- api_app/models/schemas/resource_template.py | 2 +- api_app/models/schemas/shared_service.py | 10 +++++----- api_app/models/schemas/shared_service_template.py | 4 ++-- api_app/models/schemas/user_resource.py | 6 +++--- api_app/models/schemas/user_resource_template.py | 4 ++-- api_app/models/schemas/users.py | 2 +- api_app/models/schemas/workspace.py | 10 +++++----- api_app/models/schemas/workspace_service.py | 6 +++--- api_app/models/schemas/workspace_service_template.py | 4 ++-- api_app/models/schemas/workspace_template.py | 4 ++-- 16 files changed, 42 insertions(+), 42 deletions(-) diff --git a/api_app/models/domain/azuretremodel.py b/api_app/models/domain/azuretremodel.py index 4f24bb2098..d84ed78827 100644 --- a/api_app/models/domain/azuretremodel.py +++ b/api_app/models/domain/azuretremodel.py @@ -3,5 +3,5 @@ class AzureTREModel(BaseModel): class Config(ConfigDict): - allow_population_by_field_name = True + populate_by_name = True arbitrary_types_allowed = True diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index f4484fa1a9..b694af4009 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -22,7 +22,7 @@ class ResourceHistoryItem(AzureTREModel): """ id: str = Field(title="Id", description="GUID identifying the resource request") resourceId: str = Field(title="Id", description="GUID identifying the resource request") - properties: dict = Field({}, title="Resource template parameters", description="Parameters for the deployment") + properties: dict = Field(None, title="Resource template parameters", description="Parameters for the deployment") isEnabled: bool = True resourceVersion: int = 0 updatedWhen: float = 0 @@ -42,12 +42,12 @@ class Resource(AzureTREModel): id: str = Field(title="Id", description="GUID identifying the resource request") templateName: str = Field(title="Resource template name", description="The resource template (bundle) to deploy") templateVersion: str = Field(title="Resource template version", description="The version of the resource template (bundle) to deploy") - properties: dict = Field({}, title="Resource template parameters", description="Parameters for the deployment") + properties: dict = Field(None, title="Resource template parameters", description="Parameters for the deployment") availableUpgrades: Optional[List[AvailableUpgrade]] = Field([], title="Available template upgrades", description="Versions of the template that are available for upgrade") isEnabled: bool = True # Must be set before a resource can be deleted resourceType: ResourceType - deploymentStatus: Optional[str] = Field("", title="Deployment Status", description="Overall deployment status of the resource") - etag: str = Field("", title="_etag", description="eTag of the document", alias="_etag") + deploymentStatus: Optional[str] = Field(None, title="Deployment Status", description="Overall deployment status of the resource") + etag: str = Field(None, title="_etag", description="eTag of the document", alias="_etag") resourcePath: str = "" resourceVersion: int = 0 user: dict = {} diff --git a/api_app/models/schemas/airlock_request.py b/api_app/models/schemas/airlock_request.py index 2500b2462f..1e9bed7269 100644 --- a/api_app/models/schemas/airlock_request.py +++ b/api_app/models/schemas/airlock_request.py @@ -46,7 +46,7 @@ class AirlockRequestInResponse(BaseModel): airlockRequest: AirlockRequest class Config: - schema_extra = { + json_schema_extra = { "example": { "airlockRequest": get_sample_airlock_request("933ad738-7265-4b5f-9eae-a1a62928772e", "121e921f-a4aa-44b3-90a9-e8da030495ef") } @@ -58,7 +58,7 @@ class AirlockRequestAndOperationInResponse(BaseModel): operation: Operation class Config: - schema_extra = { + json_schema_extra = { "example": { "airlockRequest": get_sample_airlock_request("933ad738-7265-4b5f-9eae-a1a62928772e", "121e921f-a4aa-44b3-90a9-e8da030495ef"), "operation": get_sample_operation("121e921f-a4aa-44b3-90a9-e8da030495ef") @@ -71,7 +71,7 @@ class AirlockRequestWithAllowedUserActions(BaseModel): allowedUserActions: List[str] = Field([], title="actions that the requesting user can do on the request") class Config: - schema_extra = { + json_schema_extra = { "example": get_sample_airlock_request_with_allowed_user_actions("933ad738-7265-4b5f-9eae-a1a62928772e"), } @@ -80,7 +80,7 @@ class AirlockRequestWithAllowedUserActionsInList(BaseModel): airlockRequests: List[AirlockRequestWithAllowedUserActions] = Field([], title="Airlock Requests") class Config: - schema_extra = { + json_schema_extra = { "example": { "airlockRequests": [ get_sample_airlock_request_with_allowed_user_actions("933ad738-7265-4b5f-9eae-a1a62928772e"), @@ -97,7 +97,7 @@ class AirlockRequestInCreate(BaseModel): properties: dict = Field({}, title="Airlock request parameters", description="Values for the parameters required by the Airlock request specification") class Config: - schema_extra = { + json_schema_extra = { "example": { "type": "import", "title": "a request title", @@ -111,7 +111,7 @@ class AirlockReviewInCreate(BaseModel): decisionExplanation: str = Field("Decision Explanation", title="Explanation of the reviewer for the reviews decision") class Config: - schema_extra = { + json_schema_extra = { "example": { "approval": "True", "decisionExplanation": "the reason why this request was approved/rejected" diff --git a/api_app/models/schemas/airlock_request_url.py b/api_app/models/schemas/airlock_request_url.py index a83e3ccfdd..aed07f92c3 100644 --- a/api_app/models/schemas/airlock_request_url.py +++ b/api_app/models/schemas/airlock_request_url.py @@ -11,7 +11,7 @@ class AirlockRequestTokenInResponse(BaseModel): containerUrl: str class Config: - schema_extra = { + json_schema_extra = { "example": { "container_url": get_sample_airlock_request_container_url("container_url") } diff --git a/api_app/models/schemas/operation.py b/api_app/models/schemas/operation.py index 4b4c833c14..e3a85a8887 100644 --- a/api_app/models/schemas/operation.py +++ b/api_app/models/schemas/operation.py @@ -33,7 +33,7 @@ class OperationInResponse(BaseModel): operation: Operation class Config: - schema_extra = { + json_schema_extra = { "example": { "operation": get_sample_operation("7ac667f0-fd3f-4a6c-815b-82d0cb7a2132") } @@ -44,7 +44,7 @@ class OperationInList(BaseModel): operations: List[Operation] = Field([], title="Operations") class Config: - schema_extra = { + json_schema_extra = { "example": { "operations": [ get_sample_operation("7ac667f0-fd3f-4a6c-815b-82d0cb7a2132"), diff --git a/api_app/models/schemas/resource.py b/api_app/models/schemas/resource.py index 34d84c9cb6..d6ae60ed19 100644 --- a/api_app/models/schemas/resource.py +++ b/api_app/models/schemas/resource.py @@ -11,7 +11,7 @@ class ResourcePatch(BaseModel): class Config: extra = Extra.forbid - schema_extra = { + json_schema_extra = { "example": { "isEnabled": False, "templateVersion": "1.0.1", @@ -45,7 +45,7 @@ class ResourceHistoryInList(BaseModel): resource_history: List[ResourceHistoryItem] = Field([], title="Resource history") class Config: - schema_extra = { + json_schema_extra = { "example": { "resource_history": [ get_sample_resource_history("2fdc9fba-726e-4db6-a1b8-9018a2165748"), diff --git a/api_app/models/schemas/resource_template.py b/api_app/models/schemas/resource_template.py index dd3b75722e..04e9ca3ee8 100644 --- a/api_app/models/schemas/resource_template.py +++ b/api_app/models/schemas/resource_template.py @@ -28,7 +28,7 @@ class ResourceTemplateInformationInList(BaseModel): templates: List[ResourceTemplateInformation] class Config: - schema_extra = { + json_schema_extra = { "example": { "templates": [ { diff --git a/api_app/models/schemas/shared_service.py b/api_app/models/schemas/shared_service.py index 6c194f3faf..c1ed7d6654 100644 --- a/api_app/models/schemas/shared_service.py +++ b/api_app/models/schemas/shared_service.py @@ -25,7 +25,7 @@ class SharedServiceInResponse(BaseModel): sharedService: SharedService class Config: - schema_extra = { + json_schema_extra = { "example": { "shared_service": get_sample_shared_service("2fdc9fba-726e-4db6-a1b8-9018a2165748") } @@ -36,7 +36,7 @@ class RestrictedSharedServiceInResponse(BaseModel): sharedService: RestrictedResource class Config: - schema_extra = { + json_schema_extra = { "example": { "shared_service": get_sample_shared_service("2fdc9fba-726e-4db6-a1b8-9018a2165748") } @@ -47,7 +47,7 @@ class RestrictedSharedServicesInList(BaseModel): sharedServices: List[RestrictedResource] = Field([], title="shared services") class Config: - schema_extra = { + json_schema_extra = { "example": { "sharedServices": [ get_sample_shared_service("2fdc9fba-726e-4db6-a1b8-9018a2165748"), @@ -61,7 +61,7 @@ class SharedServicesInList(BaseModel): sharedServices: List[SharedService] = Field([], title="shared services") class Config: - schema_extra = { + json_schema_extra = { "example": { "sharedServices": [ get_sample_shared_service("2fdc9fba-726e-4db6-a1b8-9018a2165748"), @@ -76,7 +76,7 @@ class SharedServiceInCreate(BaseModel): properties: dict = Field({}, title="Shared service parameters", description="Values for the parameters required by the shared service resource specification") class Config: - schema_extra = { + json_schema_extra = { "example": { "templateName": "tre-shared-service-firewall", "properties": { diff --git a/api_app/models/schemas/shared_service_template.py b/api_app/models/schemas/shared_service_template.py index 048973b374..f76a1217a9 100644 --- a/api_app/models/schemas/shared_service_template.py +++ b/api_app/models/schemas/shared_service_template.py @@ -38,7 +38,7 @@ def get_sample_shared_service_template_in_response() -> dict: class SharedServiceTemplateInCreate(ResourceTemplateInCreate): class Config: - schema_extra = { + json_schema_extra = { "example": { "name": "my-tre-shared-service", "version": "0.0.1", @@ -65,6 +65,6 @@ class Config: class SharedServiceTemplateInResponse(ResourceTemplateInResponse): class Config: - schema_extra = { + json_schema_extra = { "example": get_sample_shared_service_template_in_response() } diff --git a/api_app/models/schemas/user_resource.py b/api_app/models/schemas/user_resource.py index a39f69759f..d324788451 100644 --- a/api_app/models/schemas/user_resource.py +++ b/api_app/models/schemas/user_resource.py @@ -29,7 +29,7 @@ class UserResourceInResponse(BaseModel): userResource: UserResource class Config: - schema_extra = { + json_schema_extra = { "example": { "user_resource": get_sample_user_resource("933ad738-7265-4b5f-9eae-a1a62928772e") } @@ -40,7 +40,7 @@ class UserResourcesInList(BaseModel): userResources: List[UserResource] = Field([], title="User resources") class Config: - schema_extra = { + json_schema_extra = { "example": { "userResources": [ get_sample_user_resource("2fdc9fba-726e-4db6-a1b8-9018a2165748"), @@ -55,7 +55,7 @@ class UserResourceInCreate(BaseModel): properties: dict = Field({}, title="User resource parameters", description="Values for the parameters required by the user resource specification") class Config: - schema_extra = { + json_schema_extra = { "example": { "templateName": "user-resource-type", "properties": { diff --git a/api_app/models/schemas/user_resource_template.py b/api_app/models/schemas/user_resource_template.py index cc0013cd75..bcae61b26f 100644 --- a/api_app/models/schemas/user_resource_template.py +++ b/api_app/models/schemas/user_resource_template.py @@ -38,7 +38,7 @@ def get_sample_user_resource_template_in_response() -> dict: class UserResourceTemplateInCreate(ResourceTemplateInCreate): class Config: - schema_extra = { + json_schema_extra = { "example": { "name": "my-tre-user-resource", "version": "0.0.1", @@ -71,6 +71,6 @@ class UserResourceTemplateInResponse(ResourceTemplateInResponse): parentWorkspaceService: str = Field(title="Workspace type", description="Bundle name") class Config: - schema_extra = { + json_schema_extra = { "example": get_sample_user_resource_template_in_response() } diff --git a/api_app/models/schemas/users.py b/api_app/models/schemas/users.py index 56c8025e56..86daa8f485 100644 --- a/api_app/models/schemas/users.py +++ b/api_app/models/schemas/users.py @@ -8,7 +8,7 @@ class UsersInResponse(BaseModel): users: List[User] = Field(..., title="Users", description="List of users assigned to the workspace") class Config: - schema_extra = { + json_schema_extra = { "example": { "users": [ { diff --git a/api_app/models/schemas/workspace.py b/api_app/models/schemas/workspace.py index 94c6bf861e..105ed7b63c 100644 --- a/api_app/models/schemas/workspace.py +++ b/api_app/models/schemas/workspace.py @@ -39,7 +39,7 @@ class WorkspaceInResponse(BaseModel): workspace: Workspace class Config: - schema_extra = { + json_schema_extra = { "example": { "workspace": get_sample_workspace("933ad738-7265-4b5f-9eae-a1a62928772e") } @@ -50,7 +50,7 @@ class WorkspaceAuthInResponse(BaseModel): workspaceAuth: WorkspaceAuth class Config: - schema_extra = { + json_schema_extra = { "example": { "scopeId": "api://mytre-ws-1233456" } @@ -61,7 +61,7 @@ class WorkspacesInList(BaseModel): workspaces: List[Workspace] class Config: - schema_extra = { + json_schema_extra = { "example": { "workspaces": [ get_sample_workspace("933ad738-7265-4b5f-9eae-a1a62928772e", "0001"), @@ -73,10 +73,10 @@ class Config: class WorkspaceInCreate(BaseModel): templateName: str = Field(title="Workspace type", description="Bundle name") - properties: dict = Field({}, title="Workspace parameters", description="Values for the parameters required by the workspace resource specification") + properties: dict = Field(None, title="Workspace parameters", description="Values for the parameters required by the workspace resource specification") class Config: - schema_extra = { + json_schema_extra = { "example": { "templateName": "tre-workspace-base", "properties": { diff --git a/api_app/models/schemas/workspace_service.py b/api_app/models/schemas/workspace_service.py index 069fa3a4f7..032b597a6b 100644 --- a/api_app/models/schemas/workspace_service.py +++ b/api_app/models/schemas/workspace_service.py @@ -24,7 +24,7 @@ class WorkspaceServiceInResponse(BaseModel): workspaceService: WorkspaceService class Config: - schema_extra = { + json_schema_extra = { "example": { "workspace_service": get_sample_workspace_service("933ad738-7265-4b5f-9eae-a1a62928772e", "2fdc9fba-726e-4db6-a1b8-9018a2165748") } @@ -35,7 +35,7 @@ class WorkspaceServicesInList(BaseModel): workspaceServices: List[WorkspaceService] = Field([], title="Workspace services") class Config: - schema_extra = { + json_schema_extra = { "example": { "workspaceServices": [ get_sample_workspace_service("933ad738-7265-4b5f-9eae-a1a62928772e", "2fdc9fba-726e-4db6-a1b8-9018a2165748"), @@ -50,7 +50,7 @@ class WorkspaceServiceInCreate(BaseModel): properties: dict = Field({}, title="Workspace service parameters", description="Values for the parameters required by the workspace service resource specification") class Config: - schema_extra = { + json_schema_extra = { "example": { "templateName": "tre-service-guacamole", "properties": { diff --git a/api_app/models/schemas/workspace_service_template.py b/api_app/models/schemas/workspace_service_template.py index c3f493169e..6ca3936046 100644 --- a/api_app/models/schemas/workspace_service_template.py +++ b/api_app/models/schemas/workspace_service_template.py @@ -39,7 +39,7 @@ def get_sample_workspace_service_template_in_response() -> dict: class WorkspaceServiceTemplateInCreate(ResourceTemplateInCreate): class Config: - schema_extra = { + json_schema_extra = { "example": { "name": "my-tre-workspace-service", "version": "0.0.1", @@ -67,6 +67,6 @@ class Config: class WorkspaceServiceTemplateInResponse(ResourceTemplateInResponse): class Config: - schema_extra = { + json_schema_extra = { "example": get_sample_workspace_service_template_in_response() } diff --git a/api_app/models/schemas/workspace_template.py b/api_app/models/schemas/workspace_template.py index bc20955217..c6a364a210 100644 --- a/api_app/models/schemas/workspace_template.py +++ b/api_app/models/schemas/workspace_template.py @@ -43,7 +43,7 @@ def get_sample_workspace_template_in_response() -> dict: class WorkspaceTemplateInCreate(ResourceTemplateInCreate): class Config: - schema_extra = { + json_schema_extra = { "example": { "name": "my-tre-workspace", "version": "0.0.1", @@ -95,6 +95,6 @@ class Config: class WorkspaceTemplateInResponse(ResourceTemplateInResponse): class Config: - schema_extra = { + json_schema_extra = { "example": get_sample_workspace_template_in_response() } From e36ca158c60893f62fdddeb8b50f1d5e0a0b25e9 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Wed, 13 Nov 2024 10:59:05 +0000 Subject: [PATCH 34/47] add in user, edit output.sh, revert schema_extra --- api_app/models/domain/operation.py | 3 ++- api_app/models/domain/resource.py | 5 +++-- api_app/models/domain/restricted_resource.py | 3 ++- api_app/models/schemas/airlock_request.py | 12 ++++++------ api_app/models/schemas/airlock_request_url.py | 2 +- api_app/models/schemas/operation.py | 4 ++-- api_app/models/schemas/resource.py | 4 ++-- api_app/models/schemas/resource_template.py | 2 +- api_app/models/schemas/shared_service.py | 10 +++++----- api_app/models/schemas/shared_service_template.py | 4 ++-- api_app/models/schemas/user_resource.py | 6 +++--- api_app/models/schemas/user_resource_template.py | 4 ++-- api_app/models/schemas/users.py | 2 +- api_app/models/schemas/workspace.py | 8 ++++---- api_app/models/schemas/workspace_service.py | 6 +++--- api_app/models/schemas/workspace_service_template.py | 4 ++-- api_app/models/schemas/workspace_template.py | 4 ++-- core/terraform/outputs.sh | 6 ++++++ 18 files changed, 49 insertions(+), 40 deletions(-) diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index f8c2dd36dc..2d11d40a55 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -6,6 +6,7 @@ from models.domain.azuretremodel import AzureTREModel from models.domain.resource import Output, ResourceType +from models.domain.authentication import User from resources import strings @@ -92,7 +93,7 @@ class Operation(AzureTREModel): message: str = Field("", title="Additional operation status information") createdWhen: float = Field("", title="POSIX Timestamp for when the operation was submitted") updatedWhen: float = Field("", title="POSIX Timestamp for When the operation was updated") - user: dict = {} + user: User steps: Optional[List[OperationStep]] = Field(None, title="Operation Steps") diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index b694af4009..64e299427e 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -2,6 +2,7 @@ from typing import Optional, Union, List from pydantic import BaseModel, Field, field_validator from models.domain.azuretremodel import AzureTREModel +from models.domain.authentication import User from models.domain.request_action import RequestAction from resources import strings @@ -26,7 +27,7 @@ class ResourceHistoryItem(AzureTREModel): isEnabled: bool = True resourceVersion: int = 0 updatedWhen: float = 0 - user: dict = {} + user: User templateVersion: Optional[str] = Field(title="Resource template version", description="The version of the resource template (bundle) to deploy") @@ -50,7 +51,7 @@ class Resource(AzureTREModel): etag: str = Field(None, title="_etag", description="eTag of the document", alias="_etag") resourcePath: str = "" resourceVersion: int = 0 - user: dict = {} + user: Optional[User] = {} updatedWhen: float = 0 def get_resource_request_message_payload(self, operation_id: str, step_id: str, action: RequestAction) -> dict: diff --git a/api_app/models/domain/restricted_resource.py b/api_app/models/domain/restricted_resource.py index 4f9c993f1e..583361a7ca 100644 --- a/api_app/models/domain/restricted_resource.py +++ b/api_app/models/domain/restricted_resource.py @@ -2,6 +2,7 @@ from pydantic import Field from models.domain.resource import AvailableUpgrade, ResourceType from models.domain.azuretremodel import AzureTREModel +from models.domain.authentication import User class RestrictedProperties(AzureTREModel): @@ -27,5 +28,5 @@ class RestrictedResource(AzureTREModel): etag: str = Field(title="_etag", description="eTag of the document", alias="_etag") resourcePath: str = "" resourceVersion: int = 0 - user: dict = {} + user: User updatedWhen: float = 0 diff --git a/api_app/models/schemas/airlock_request.py b/api_app/models/schemas/airlock_request.py index 1e9bed7269..2500b2462f 100644 --- a/api_app/models/schemas/airlock_request.py +++ b/api_app/models/schemas/airlock_request.py @@ -46,7 +46,7 @@ class AirlockRequestInResponse(BaseModel): airlockRequest: AirlockRequest class Config: - json_schema_extra = { + schema_extra = { "example": { "airlockRequest": get_sample_airlock_request("933ad738-7265-4b5f-9eae-a1a62928772e", "121e921f-a4aa-44b3-90a9-e8da030495ef") } @@ -58,7 +58,7 @@ class AirlockRequestAndOperationInResponse(BaseModel): operation: Operation class Config: - json_schema_extra = { + schema_extra = { "example": { "airlockRequest": get_sample_airlock_request("933ad738-7265-4b5f-9eae-a1a62928772e", "121e921f-a4aa-44b3-90a9-e8da030495ef"), "operation": get_sample_operation("121e921f-a4aa-44b3-90a9-e8da030495ef") @@ -71,7 +71,7 @@ class AirlockRequestWithAllowedUserActions(BaseModel): allowedUserActions: List[str] = Field([], title="actions that the requesting user can do on the request") class Config: - json_schema_extra = { + schema_extra = { "example": get_sample_airlock_request_with_allowed_user_actions("933ad738-7265-4b5f-9eae-a1a62928772e"), } @@ -80,7 +80,7 @@ class AirlockRequestWithAllowedUserActionsInList(BaseModel): airlockRequests: List[AirlockRequestWithAllowedUserActions] = Field([], title="Airlock Requests") class Config: - json_schema_extra = { + schema_extra = { "example": { "airlockRequests": [ get_sample_airlock_request_with_allowed_user_actions("933ad738-7265-4b5f-9eae-a1a62928772e"), @@ -97,7 +97,7 @@ class AirlockRequestInCreate(BaseModel): properties: dict = Field({}, title="Airlock request parameters", description="Values for the parameters required by the Airlock request specification") class Config: - json_schema_extra = { + schema_extra = { "example": { "type": "import", "title": "a request title", @@ -111,7 +111,7 @@ class AirlockReviewInCreate(BaseModel): decisionExplanation: str = Field("Decision Explanation", title="Explanation of the reviewer for the reviews decision") class Config: - json_schema_extra = { + schema_extra = { "example": { "approval": "True", "decisionExplanation": "the reason why this request was approved/rejected" diff --git a/api_app/models/schemas/airlock_request_url.py b/api_app/models/schemas/airlock_request_url.py index aed07f92c3..a83e3ccfdd 100644 --- a/api_app/models/schemas/airlock_request_url.py +++ b/api_app/models/schemas/airlock_request_url.py @@ -11,7 +11,7 @@ class AirlockRequestTokenInResponse(BaseModel): containerUrl: str class Config: - json_schema_extra = { + schema_extra = { "example": { "container_url": get_sample_airlock_request_container_url("container_url") } diff --git a/api_app/models/schemas/operation.py b/api_app/models/schemas/operation.py index e3a85a8887..4b4c833c14 100644 --- a/api_app/models/schemas/operation.py +++ b/api_app/models/schemas/operation.py @@ -33,7 +33,7 @@ class OperationInResponse(BaseModel): operation: Operation class Config: - json_schema_extra = { + schema_extra = { "example": { "operation": get_sample_operation("7ac667f0-fd3f-4a6c-815b-82d0cb7a2132") } @@ -44,7 +44,7 @@ class OperationInList(BaseModel): operations: List[Operation] = Field([], title="Operations") class Config: - json_schema_extra = { + schema_extra = { "example": { "operations": [ get_sample_operation("7ac667f0-fd3f-4a6c-815b-82d0cb7a2132"), diff --git a/api_app/models/schemas/resource.py b/api_app/models/schemas/resource.py index d6ae60ed19..34d84c9cb6 100644 --- a/api_app/models/schemas/resource.py +++ b/api_app/models/schemas/resource.py @@ -11,7 +11,7 @@ class ResourcePatch(BaseModel): class Config: extra = Extra.forbid - json_schema_extra = { + schema_extra = { "example": { "isEnabled": False, "templateVersion": "1.0.1", @@ -45,7 +45,7 @@ class ResourceHistoryInList(BaseModel): resource_history: List[ResourceHistoryItem] = Field([], title="Resource history") class Config: - json_schema_extra = { + schema_extra = { "example": { "resource_history": [ get_sample_resource_history("2fdc9fba-726e-4db6-a1b8-9018a2165748"), diff --git a/api_app/models/schemas/resource_template.py b/api_app/models/schemas/resource_template.py index 04e9ca3ee8..dd3b75722e 100644 --- a/api_app/models/schemas/resource_template.py +++ b/api_app/models/schemas/resource_template.py @@ -28,7 +28,7 @@ class ResourceTemplateInformationInList(BaseModel): templates: List[ResourceTemplateInformation] class Config: - json_schema_extra = { + schema_extra = { "example": { "templates": [ { diff --git a/api_app/models/schemas/shared_service.py b/api_app/models/schemas/shared_service.py index c1ed7d6654..6c194f3faf 100644 --- a/api_app/models/schemas/shared_service.py +++ b/api_app/models/schemas/shared_service.py @@ -25,7 +25,7 @@ class SharedServiceInResponse(BaseModel): sharedService: SharedService class Config: - json_schema_extra = { + schema_extra = { "example": { "shared_service": get_sample_shared_service("2fdc9fba-726e-4db6-a1b8-9018a2165748") } @@ -36,7 +36,7 @@ class RestrictedSharedServiceInResponse(BaseModel): sharedService: RestrictedResource class Config: - json_schema_extra = { + schema_extra = { "example": { "shared_service": get_sample_shared_service("2fdc9fba-726e-4db6-a1b8-9018a2165748") } @@ -47,7 +47,7 @@ class RestrictedSharedServicesInList(BaseModel): sharedServices: List[RestrictedResource] = Field([], title="shared services") class Config: - json_schema_extra = { + schema_extra = { "example": { "sharedServices": [ get_sample_shared_service("2fdc9fba-726e-4db6-a1b8-9018a2165748"), @@ -61,7 +61,7 @@ class SharedServicesInList(BaseModel): sharedServices: List[SharedService] = Field([], title="shared services") class Config: - json_schema_extra = { + schema_extra = { "example": { "sharedServices": [ get_sample_shared_service("2fdc9fba-726e-4db6-a1b8-9018a2165748"), @@ -76,7 +76,7 @@ class SharedServiceInCreate(BaseModel): properties: dict = Field({}, title="Shared service parameters", description="Values for the parameters required by the shared service resource specification") class Config: - json_schema_extra = { + schema_extra = { "example": { "templateName": "tre-shared-service-firewall", "properties": { diff --git a/api_app/models/schemas/shared_service_template.py b/api_app/models/schemas/shared_service_template.py index f76a1217a9..048973b374 100644 --- a/api_app/models/schemas/shared_service_template.py +++ b/api_app/models/schemas/shared_service_template.py @@ -38,7 +38,7 @@ def get_sample_shared_service_template_in_response() -> dict: class SharedServiceTemplateInCreate(ResourceTemplateInCreate): class Config: - json_schema_extra = { + schema_extra = { "example": { "name": "my-tre-shared-service", "version": "0.0.1", @@ -65,6 +65,6 @@ class Config: class SharedServiceTemplateInResponse(ResourceTemplateInResponse): class Config: - json_schema_extra = { + schema_extra = { "example": get_sample_shared_service_template_in_response() } diff --git a/api_app/models/schemas/user_resource.py b/api_app/models/schemas/user_resource.py index d324788451..a39f69759f 100644 --- a/api_app/models/schemas/user_resource.py +++ b/api_app/models/schemas/user_resource.py @@ -29,7 +29,7 @@ class UserResourceInResponse(BaseModel): userResource: UserResource class Config: - json_schema_extra = { + schema_extra = { "example": { "user_resource": get_sample_user_resource("933ad738-7265-4b5f-9eae-a1a62928772e") } @@ -40,7 +40,7 @@ class UserResourcesInList(BaseModel): userResources: List[UserResource] = Field([], title="User resources") class Config: - json_schema_extra = { + schema_extra = { "example": { "userResources": [ get_sample_user_resource("2fdc9fba-726e-4db6-a1b8-9018a2165748"), @@ -55,7 +55,7 @@ class UserResourceInCreate(BaseModel): properties: dict = Field({}, title="User resource parameters", description="Values for the parameters required by the user resource specification") class Config: - json_schema_extra = { + schema_extra = { "example": { "templateName": "user-resource-type", "properties": { diff --git a/api_app/models/schemas/user_resource_template.py b/api_app/models/schemas/user_resource_template.py index bcae61b26f..cc0013cd75 100644 --- a/api_app/models/schemas/user_resource_template.py +++ b/api_app/models/schemas/user_resource_template.py @@ -38,7 +38,7 @@ def get_sample_user_resource_template_in_response() -> dict: class UserResourceTemplateInCreate(ResourceTemplateInCreate): class Config: - json_schema_extra = { + schema_extra = { "example": { "name": "my-tre-user-resource", "version": "0.0.1", @@ -71,6 +71,6 @@ class UserResourceTemplateInResponse(ResourceTemplateInResponse): parentWorkspaceService: str = Field(title="Workspace type", description="Bundle name") class Config: - json_schema_extra = { + schema_extra = { "example": get_sample_user_resource_template_in_response() } diff --git a/api_app/models/schemas/users.py b/api_app/models/schemas/users.py index 86daa8f485..56c8025e56 100644 --- a/api_app/models/schemas/users.py +++ b/api_app/models/schemas/users.py @@ -8,7 +8,7 @@ class UsersInResponse(BaseModel): users: List[User] = Field(..., title="Users", description="List of users assigned to the workspace") class Config: - json_schema_extra = { + schema_extra = { "example": { "users": [ { diff --git a/api_app/models/schemas/workspace.py b/api_app/models/schemas/workspace.py index 105ed7b63c..dc85db999b 100644 --- a/api_app/models/schemas/workspace.py +++ b/api_app/models/schemas/workspace.py @@ -39,7 +39,7 @@ class WorkspaceInResponse(BaseModel): workspace: Workspace class Config: - json_schema_extra = { + schema_extra = { "example": { "workspace": get_sample_workspace("933ad738-7265-4b5f-9eae-a1a62928772e") } @@ -50,7 +50,7 @@ class WorkspaceAuthInResponse(BaseModel): workspaceAuth: WorkspaceAuth class Config: - json_schema_extra = { + schema_extra = { "example": { "scopeId": "api://mytre-ws-1233456" } @@ -61,7 +61,7 @@ class WorkspacesInList(BaseModel): workspaces: List[Workspace] class Config: - json_schema_extra = { + schema_extra = { "example": { "workspaces": [ get_sample_workspace("933ad738-7265-4b5f-9eae-a1a62928772e", "0001"), @@ -76,7 +76,7 @@ class WorkspaceInCreate(BaseModel): properties: dict = Field(None, title="Workspace parameters", description="Values for the parameters required by the workspace resource specification") class Config: - json_schema_extra = { + schema_extra = { "example": { "templateName": "tre-workspace-base", "properties": { diff --git a/api_app/models/schemas/workspace_service.py b/api_app/models/schemas/workspace_service.py index 032b597a6b..069fa3a4f7 100644 --- a/api_app/models/schemas/workspace_service.py +++ b/api_app/models/schemas/workspace_service.py @@ -24,7 +24,7 @@ class WorkspaceServiceInResponse(BaseModel): workspaceService: WorkspaceService class Config: - json_schema_extra = { + schema_extra = { "example": { "workspace_service": get_sample_workspace_service("933ad738-7265-4b5f-9eae-a1a62928772e", "2fdc9fba-726e-4db6-a1b8-9018a2165748") } @@ -35,7 +35,7 @@ class WorkspaceServicesInList(BaseModel): workspaceServices: List[WorkspaceService] = Field([], title="Workspace services") class Config: - json_schema_extra = { + schema_extra = { "example": { "workspaceServices": [ get_sample_workspace_service("933ad738-7265-4b5f-9eae-a1a62928772e", "2fdc9fba-726e-4db6-a1b8-9018a2165748"), @@ -50,7 +50,7 @@ class WorkspaceServiceInCreate(BaseModel): properties: dict = Field({}, title="Workspace service parameters", description="Values for the parameters required by the workspace service resource specification") class Config: - json_schema_extra = { + schema_extra = { "example": { "templateName": "tre-service-guacamole", "properties": { diff --git a/api_app/models/schemas/workspace_service_template.py b/api_app/models/schemas/workspace_service_template.py index 6ca3936046..c3f493169e 100644 --- a/api_app/models/schemas/workspace_service_template.py +++ b/api_app/models/schemas/workspace_service_template.py @@ -39,7 +39,7 @@ def get_sample_workspace_service_template_in_response() -> dict: class WorkspaceServiceTemplateInCreate(ResourceTemplateInCreate): class Config: - json_schema_extra = { + schema_extra = { "example": { "name": "my-tre-workspace-service", "version": "0.0.1", @@ -67,6 +67,6 @@ class Config: class WorkspaceServiceTemplateInResponse(ResourceTemplateInResponse): class Config: - json_schema_extra = { + schema_extra = { "example": get_sample_workspace_service_template_in_response() } diff --git a/api_app/models/schemas/workspace_template.py b/api_app/models/schemas/workspace_template.py index c6a364a210..bc20955217 100644 --- a/api_app/models/schemas/workspace_template.py +++ b/api_app/models/schemas/workspace_template.py @@ -43,7 +43,7 @@ def get_sample_workspace_template_in_response() -> dict: class WorkspaceTemplateInCreate(ResourceTemplateInCreate): class Config: - json_schema_extra = { + schema_extra = { "example": { "name": "my-tre-workspace", "version": "0.0.1", @@ -95,6 +95,6 @@ class Config: class WorkspaceTemplateInResponse(ResourceTemplateInResponse): class Config: - json_schema_extra = { + schema_extra = { "example": get_sample_workspace_template_in_response() } diff --git a/core/terraform/outputs.sh b/core/terraform/outputs.sh index e00090bbc1..02cc025de8 100755 --- a/core/terraform/outputs.sh +++ b/core/terraform/outputs.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +chmod g+x ../../core/ + if [ ! -f ../tre_output.json ]; then # Connect to the remote backend of Terraform export TF_LOG="" @@ -15,6 +17,8 @@ if [ ! -f ../tre_output.json ]; then terraform output -json > ../tre_output.json fi +touch ../private.env + # Now create an .env file ./json-to-env.sh < ../tre_output.json > ../private.env @@ -33,3 +37,5 @@ echo "TEST_WORKSPACE_APP_SECRET='${WORKSPACE_API_CLIENT_SECRET}'" >> ../private. echo "SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env echo "AZURE_SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env echo "AZURE_TENANT_ID='${TENANT_ID}'" >> ../private.env + +ls -al ../ From 71d05f52d6ba01809680f16c6b88eefd9c9de2b4 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Thu, 28 Nov 2024 22:03:46 +0000 Subject: [PATCH 35/47] updates --- api_app/db/repositories/resource_templates.py | 14 +++++++++----- api_app/models/domain/operation.py | 2 +- api_app/models/domain/resource.py | 2 +- api_app/models/domain/resource_template.py | 16 ++++++++-------- api_app/requirements.txt | 4 ++-- resource_processor/vmss_porter/requirements.txt | 4 ++-- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index 66674e7cf0..5cd0ed6e56 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -1,7 +1,7 @@ import uuid from typing import List, Optional, Union -from pydantic import parse_obj_as +from pydantic import parse_obj_as, TypeAdapter from core import config from db.errors import DuplicateEntity, EntityDoesNotExist, EntityVersionExist, InvalidInput @@ -90,9 +90,11 @@ async def get_template_by_name_and_version(self, name: str, version: str, resour if len(templates) != 1: raise EntityDoesNotExist if resource_type == ResourceType.UserResource: - return parse_obj_as(UserResourceTemplate, templates[0]) + ta = TypeAdapter(UserResourceTemplate) + return ta.validate_python(templates[0]) else: - return parse_obj_as(ResourceTemplate, templates[0]) + ta = TypeAdapter(ResourceTemplate) + return ta.validate_python(templates[0]) async def get_all_template_versions(self, template_name: str) -> List[str]: query = 'SELECT VALUE c.version FROM c where c.name = @template_name' @@ -131,9 +133,11 @@ async def create_template(self, template_input: ResourceTemplateInCreate, resour if resource_type == ResourceType.UserResource: template["parentWorkspaceService"] = parent_service_name - template = parse_obj_as(UserResourceTemplate, template) + ta = TypeAdapter(UserResourceTemplate) + template = ta.validate_python(template) else: - template = parse_obj_as(ResourceTemplate, template) + ta = TypeAdapter(ResourceTemplate) + template = ta.validate_python(template) await self.save_item(template) return template diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 2d11d40a55..40b6a27ff2 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -93,7 +93,7 @@ class Operation(AzureTREModel): message: str = Field("", title="Additional operation status information") createdWhen: float = Field("", title="POSIX Timestamp for when the operation was submitted") updatedWhen: float = Field("", title="POSIX Timestamp for When the operation was updated") - user: User + user: Optional[User] = {} steps: Optional[List[OperationStep]] = Field(None, title="Operation Steps") diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index 64e299427e..95530e612e 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -27,7 +27,7 @@ class ResourceHistoryItem(AzureTREModel): isEnabled: bool = True resourceVersion: int = 0 updatedWhen: float = 0 - user: User + user: Optional[User] = {} templateVersion: Optional[str] = Field(title="Resource template version", description="The version of the resource template (bundle) to deploy") diff --git a/api_app/models/domain/resource_template.py b/api_app/models/domain/resource_template.py index aa213dedef..d4506218b7 100644 --- a/api_app/models/domain/resource_template.py +++ b/api_app/models/domain/resource_template.py @@ -34,20 +34,20 @@ class CustomAction(AzureTREModel): class PipelineStepProperty(AzureTREModel): - name: str = Field(title="name", description="name of the property to update") - type: str = Field(title="type", description="data type of the property to update") + name: str = Field(None, title="name", description="name of the property to update") + type: str = Field(None, title="type", description="data type of the property to update") value: Union[dict, str] = Field(None, title="value", description="value to use in substitution for the property to update") arraySubstitutionAction: Optional[str] = Field("", title="Array Substitution Action", description="How to treat existing values of this property in an array [overwrite | append | replace | remove]") arrayMatchField: Optional[str] = Field("", title="Array match field", description="Name of the field to use for finding an item in an array - to replace/remove it") class PipelineStep(AzureTREModel): - stepId: Optional[str] = Field(title="stepId", description="Unique id identifying the step") - stepTitle: Optional[str] = Field(title="stepTitle", description="Human readable title of what the step is for") - resourceTemplateName: Optional[str] = Field(title="resourceTemplateName", description="Name of the template for the resource under change") - resourceType: Optional[ResourceType] = Field(title="resourceType", description="Type of resource under change") - resourceAction: Optional[str] = Field(title="resourceAction", description="Action - install / upgrade / uninstall etc") - properties: Optional[List[PipelineStepProperty]] + stepId: Optional[str] = Field(None, title="stepId", description="Unique id identifying the step") + stepTitle: Optional[str] = Field(None, title="stepTitle", description="Human readable title of what the step is for") + resourceTemplateName: Optional[str] = Field(None, title="resourceTemplateName", description="Name of the template for the resource under change") + resourceType: Optional[ResourceType] = Field(None, title="resourceType", description="Type of resource under change") + resourceAction: Optional[str] = Field(None, title="resourceAction", description="Action - install / upgrade / uninstall etc") + properties: Optional[List[PipelineStepProperty]] = Field(None, title="properties", description="List of properties to update") class Pipeline(AzureTREModel): diff --git a/api_app/requirements.txt b/api_app/requirements.txt index af38ff9ba6..4f750d4a3f 100644 --- a/api_app/requirements.txt +++ b/api_app/requirements.txt @@ -8,7 +8,7 @@ azure-mgmt-cosmosdb==9.6.0 azure-mgmt-costmanagement==4.0.1 azure-mgmt-resource==23.2.0 azure-mgmt-storage==21.2.1 -azure-monitor-opentelemetry~=1.6 +azure-monitor-opentelemetry==1.6.4 azure-servicebus==7.12.3 azure-storage-blob==12.23.1 fastapi==0.115.3 @@ -16,7 +16,7 @@ fastapi-utils==0.7.0 gunicorn==23.0.0 jsonschema[format_nongpl]==4.19.1 msal==1.31.0 -opentelemetry.instrumentation.logging>=0.44b0 +opentelemetry-instrumentation-logging==0.49b2 pandas==2.2.2 PyJWT==2.9.0 pytz==2024.2 diff --git a/resource_processor/vmss_porter/requirements.txt b/resource_processor/vmss_porter/requirements.txt index b53e65aba8..ed12509d1c 100644 --- a/resource_processor/vmss_porter/requirements.txt +++ b/resource_processor/vmss_porter/requirements.txt @@ -1,7 +1,7 @@ aiohttp==3.10.10 azure-cli-core==2.65.0 azure-identity==1.19.0 -azure-monitor-opentelemetry~=1.6 +azure-monitor-opentelemetry==1.6.4 azure-servicebus==7.12.3 -opentelemetry.instrumentation.logging>=0.44b0 +opentelemetry-instrumentation-logging==0.49b2 setuptools==75.3.0 From 87dad1f6d4d410cf363c771160d00d8e36a5a545 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Fri, 29 Nov 2024 16:49:37 +0000 Subject: [PATCH 36/47] update oydantic version --- airlock_processor/requirements.txt | 2 +- api_app/db/repositories/resource_templates.py | 14 +++++--------- api_app/models/domain/airlock_request.py | 4 ++-- api_app/models/domain/resource.py | 4 ++-- api_app/requirements.txt | 2 +- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/airlock_processor/requirements.txt b/airlock_processor/requirements.txt index 249dcde1a3..ad98e6ac64 100644 --- a/airlock_processor/requirements.txt +++ b/airlock_processor/requirements.txt @@ -5,5 +5,5 @@ azure-storage-blob==12.23.1 azure-identity==1.19.0 azure-mgmt-storage==21.2.1 azure-mgmt-resource==23.2.0 -pydantic==2.9.2 +pydantic==1.10.19 setuptools==75.3.0 diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index 5cd0ed6e56..66674e7cf0 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -1,7 +1,7 @@ import uuid from typing import List, Optional, Union -from pydantic import parse_obj_as, TypeAdapter +from pydantic import parse_obj_as from core import config from db.errors import DuplicateEntity, EntityDoesNotExist, EntityVersionExist, InvalidInput @@ -90,11 +90,9 @@ async def get_template_by_name_and_version(self, name: str, version: str, resour if len(templates) != 1: raise EntityDoesNotExist if resource_type == ResourceType.UserResource: - ta = TypeAdapter(UserResourceTemplate) - return ta.validate_python(templates[0]) + return parse_obj_as(UserResourceTemplate, templates[0]) else: - ta = TypeAdapter(ResourceTemplate) - return ta.validate_python(templates[0]) + return parse_obj_as(ResourceTemplate, templates[0]) async def get_all_template_versions(self, template_name: str) -> List[str]: query = 'SELECT VALUE c.version FROM c where c.name = @template_name' @@ -133,11 +131,9 @@ async def create_template(self, template_input: ResourceTemplateInCreate, resour if resource_type == ResourceType.UserResource: template["parentWorkspaceService"] = parent_service_name - ta = TypeAdapter(UserResourceTemplate) - template = ta.validate_python(template) + template = parse_obj_as(UserResourceTemplate, template) else: - ta = TypeAdapter(ResourceTemplate) - template = ta.validate_python(template) + template = parse_obj_as(ResourceTemplate, template) await self.save_item(template) return template diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py index 9cef0792f5..d568a316c7 100644 --- a/api_app/models/domain/airlock_request.py +++ b/api_app/models/domain/airlock_request.py @@ -2,7 +2,7 @@ from typing import List, Dict, Optional from models.domain.azuretremodel import AzureTREModel -from pydantic import Field, field_validator +from pydantic import Field, validator from resources import strings @@ -98,7 +98,7 @@ class AirlockRequest(AzureTREModel): reviewUserResources: Dict[str, AirlockReviewUserResource] = Field({}, title="User resources created for Airlock Reviews") # SQL API CosmosDB saves ETag as an escaped string: https://github.com/microsoft/AzureTRE/issues/1931 - @field_validator("etag") + @validator("etag", pre=True) def parse_etag_to_remove_escaped_quotes(cls, value): if value: return value.replace('\"', '') diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index 95530e612e..f877f8cb98 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -1,6 +1,6 @@ from enum import StrEnum from typing import Optional, Union, List -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, validator from models.domain.azuretremodel import AzureTREModel from models.domain.authentication import User from models.domain.request_action import RequestAction @@ -77,7 +77,7 @@ def get_resource_request_message_payload(self, operation_id: str, step_id: str, # SQL API CosmosDB saves etag as an escaped string by default, with no apparent way to change it. # Removing escaped quotes on pydantic deserialization. https://github.com/microsoft/AzureTRE/issues/1931 - @field_validator("etag") + @validator("etag", pre=True) def parse_etag_to_remove_escaped_quotes(cls, value): return value.replace('\"', '') diff --git a/api_app/requirements.txt b/api_app/requirements.txt index 4f750d4a3f..745d921bbe 100644 --- a/api_app/requirements.txt +++ b/api_app/requirements.txt @@ -12,7 +12,6 @@ azure-monitor-opentelemetry==1.6.4 azure-servicebus==7.12.3 azure-storage-blob==12.23.1 fastapi==0.115.3 -fastapi-utils==0.7.0 gunicorn==23.0.0 jsonschema[format_nongpl]==4.19.1 msal==1.31.0 @@ -24,3 +23,4 @@ python-dateutil==2.9.0 semantic-version==2.10.0 setuptools==75.3.0 uvicorn[standard]==0.31.0 +pydantic==1.10.19 From d05ffb512bd68063402efe8a16a4bdf5b115864e Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Fri, 29 Nov 2024 21:51:52 +0000 Subject: [PATCH 37/47] update --- .../mlflow/mlflow-server/docker/Dockerfile | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile diff --git a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile b/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile deleted file mode 100644 index 53595fb1e3..0000000000 --- a/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM python:3.12-slim-bullseye - -# Install MLflow Python Packages -RUN pip install --no-cache-dir psycopg2-binary==2.9.10 mlflow==2.17.2 azure-storage-blob==12.23.1 - -RUN apt-get update \ - && apt-get install openssh-server -y --no-install-recommends \ - && apt-get clean -y && rm -rf /var/lib/apt/lists/* - -# define default server env variables -ENV MLFLOW_SERVER_HOST 0.0.0.0 -ENV MLFLOW_SERVER_PORT 5000 -ENV MLFLOW_SERVER_WORKERS 1 - -COPY ./docker/sshd_config /etc/ssh/ -COPY ./docker/startup.sh /usr/local/bin/ - -EXPOSE 5000 2222 -ENTRYPOINT ["sh", "/usr/local/bin/startup.sh"] From 4ee2bdb3560094d0dc4a745fc02299e3c2f7906a Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 2 Dec 2024 11:05:14 +0000 Subject: [PATCH 38/47] update --- api_app/models/domain/azuretremodel.py | 6 +++--- api_app/models/domain/operation.py | 3 +-- api_app/models/domain/resource.py | 12 ++++++------ api_app/models/domain/restricted_resource.py | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/api_app/models/domain/azuretremodel.py b/api_app/models/domain/azuretremodel.py index d84ed78827..dd7dde690c 100644 --- a/api_app/models/domain/azuretremodel.py +++ b/api_app/models/domain/azuretremodel.py @@ -1,7 +1,7 @@ -from pydantic import ConfigDict, BaseModel +from pydantic import BaseConfig, BaseModel class AzureTREModel(BaseModel): - class Config(ConfigDict): - populate_by_name = True + class Config(BaseConfig): + allow_population_by_field_name = True arbitrary_types_allowed = True diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 40b6a27ff2..f8c2dd36dc 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -6,7 +6,6 @@ from models.domain.azuretremodel import AzureTREModel from models.domain.resource import Output, ResourceType -from models.domain.authentication import User from resources import strings @@ -93,7 +92,7 @@ class Operation(AzureTREModel): message: str = Field("", title="Additional operation status information") createdWhen: float = Field("", title="POSIX Timestamp for when the operation was submitted") updatedWhen: float = Field("", title="POSIX Timestamp for When the operation was updated") - user: Optional[User] = {} + user: dict = {} steps: Optional[List[OperationStep]] = Field(None, title="Operation Steps") diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index f877f8cb98..b957073086 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -23,11 +23,11 @@ class ResourceHistoryItem(AzureTREModel): """ id: str = Field(title="Id", description="GUID identifying the resource request") resourceId: str = Field(title="Id", description="GUID identifying the resource request") - properties: dict = Field(None, title="Resource template parameters", description="Parameters for the deployment") + properties: dict = Field({}, title="Resource template parameters", description="Parameters for the deployment") isEnabled: bool = True resourceVersion: int = 0 updatedWhen: float = 0 - user: Optional[User] = {} + user: dict = {} templateVersion: Optional[str] = Field(title="Resource template version", description="The version of the resource template (bundle) to deploy") @@ -43,12 +43,12 @@ class Resource(AzureTREModel): id: str = Field(title="Id", description="GUID identifying the resource request") templateName: str = Field(title="Resource template name", description="The resource template (bundle) to deploy") templateVersion: str = Field(title="Resource template version", description="The version of the resource template (bundle) to deploy") - properties: dict = Field(None, title="Resource template parameters", description="Parameters for the deployment") - availableUpgrades: Optional[List[AvailableUpgrade]] = Field([], title="Available template upgrades", description="Versions of the template that are available for upgrade") + properties: dict = Field({}, title="Resource template parameters", description="Parameters for the deployment") + availableUpgrades: Optional[List[AvailableUpgrade]] = Field(title="Available template upgrades", description="Versions of the template that are available for upgrade") isEnabled: bool = True # Must be set before a resource can be deleted resourceType: ResourceType - deploymentStatus: Optional[str] = Field(None, title="Deployment Status", description="Overall deployment status of the resource") - etag: str = Field(None, title="_etag", description="eTag of the document", alias="_etag") + deploymentStatus: Optional[str] = Field(title="Deployment Status", description="Overall deployment status of the resource") + etag: str = Field(title="_etag", description="eTag of the document", alias="_etag") resourcePath: str = "" resourceVersion: int = 0 user: Optional[User] = {} diff --git a/api_app/models/domain/restricted_resource.py b/api_app/models/domain/restricted_resource.py index 583361a7ca..6a39dac65c 100644 --- a/api_app/models/domain/restricted_resource.py +++ b/api_app/models/domain/restricted_resource.py @@ -28,5 +28,5 @@ class RestrictedResource(AzureTREModel): etag: str = Field(title="_etag", description="eTag of the document", alias="_etag") resourcePath: str = "" resourceVersion: int = 0 - user: User + user: dict = {} updatedWhen: float = 0 From d99e1959bc5dbbc90a17556b57e6203865c0f274 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 2 Dec 2024 12:10:33 +0000 Subject: [PATCH 39/47] update --- api_app/models/domain/airlock_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py index d568a316c7..154dbc78d3 100644 --- a/api_app/models/domain/airlock_request.py +++ b/api_app/models/domain/airlock_request.py @@ -91,7 +91,7 @@ class AirlockRequest(AzureTREModel): files: List[AirlockFile] = Field([], title="Files of the request") title: str = Field("Airlock Request", title="Brief title for the request") businessJustification: str = Field("Business Justification", title="Explanation that will be provided to the request reviewer") - status: str = AirlockRequestStatus.Draft + status = AirlockRequestStatus.Draft statusMessage: Optional[str] = Field(title="Optional - contains additional information about the current status.") reviews: Optional[List[AirlockReview]] etag: Optional[str] = Field(title="_etag", alias="_etag") From aaf94085e4f41bbff0ef28d5d44a681a4d25b737 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 2 Dec 2024 12:35:09 +0000 Subject: [PATCH 40/47] linting --- api_app/models/domain/restricted_resource.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api_app/models/domain/restricted_resource.py b/api_app/models/domain/restricted_resource.py index 6a39dac65c..4f9c993f1e 100644 --- a/api_app/models/domain/restricted_resource.py +++ b/api_app/models/domain/restricted_resource.py @@ -2,7 +2,6 @@ from pydantic import Field from models.domain.resource import AvailableUpgrade, ResourceType from models.domain.azuretremodel import AzureTREModel -from models.domain.authentication import User class RestrictedProperties(AzureTREModel): From 178010fad6d0bb725f08ae2b0490a91de1f97c18 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 2 Dec 2024 15:39:25 +0000 Subject: [PATCH 41/47] update --- core/terraform/outputs.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/terraform/outputs.sh b/core/terraform/outputs.sh index 02cc025de8..e00090bbc1 100755 --- a/core/terraform/outputs.sh +++ b/core/terraform/outputs.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e -chmod g+x ../../core/ - if [ ! -f ../tre_output.json ]; then # Connect to the remote backend of Terraform export TF_LOG="" @@ -17,8 +15,6 @@ if [ ! -f ../tre_output.json ]; then terraform output -json > ../tre_output.json fi -touch ../private.env - # Now create an .env file ./json-to-env.sh < ../tre_output.json > ../private.env @@ -37,5 +33,3 @@ echo "TEST_WORKSPACE_APP_SECRET='${WORKSPACE_API_CLIENT_SECRET}'" >> ../private. echo "SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env echo "AZURE_SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env echo "AZURE_TENANT_ID='${TENANT_ID}'" >> ../private.env - -ls -al ../ From b7323ea9f76d26f5137b0b4bc57d21954e8afe25 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 2 Dec 2024 15:42:55 +0000 Subject: [PATCH 42/47] update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a59d49c7e5..63d1d03c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ **BREAKING CHANGES & MIGRATIONS**: * InnerEye and MLFlow bundles depreciated and removed from main. If you wish to update and deploy these worksapce services they can be retrieved from release 0.19.1. ([#4127](https://github.com/microsoft/AzureTRE/issues/4127)) +* Upgrade Python version from 3.8 to 3.12 ([#3949](https://github.com/microsoft/AzureTRE/issues/3949)) FEATURES: @@ -17,9 +18,10 @@ ENHANCEMENTS: * Update Guacamole version and dependencies ([#4140](https://github.com/microsoft/AzureTRE/issues/4140)) * Add partial (core resources only) support for customer managed keys ([#4141](https://github.com/microsoft/AzureTRE/issues/4142), [#4144](https://github.com/microsoft/AzureTRE/issues/4144)) * Update the Azure CLI version to 2.67.0 in dev container and vmss ([#4157](https://github.com/microsoft/AzureTRE/pull/4157)) +* Upgrade Python version from 3.8 to 3.12 ([#3949](https://github.com/microsoft/AzureTRE/issues/3949)) BUG FIXES: -- Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) +* Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) COMPONENTS: From dd779867d1762c5c32e81faa08053b8c3ed11b01 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 2 Dec 2024 15:46:04 +0000 Subject: [PATCH 43/47] remove pydantic chnages --- api_app/models/domain/resource.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index b957073086..1e660059ba 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -2,7 +2,6 @@ from typing import Optional, Union, List from pydantic import BaseModel, Field, validator from models.domain.azuretremodel import AzureTREModel -from models.domain.authentication import User from models.domain.request_action import RequestAction from resources import strings @@ -51,7 +50,7 @@ class Resource(AzureTREModel): etag: str = Field(title="_etag", description="eTag of the document", alias="_etag") resourcePath: str = "" resourceVersion: int = 0 - user: Optional[User] = {} + user: dict = {} updatedWhen: float = 0 def get_resource_request_message_payload(self, operation_id: str, step_id: str, action: RequestAction) -> dict: From 56d66d88d7b0e6cc0df86345b2930debda7cada8 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 2 Dec 2024 15:49:04 +0000 Subject: [PATCH 44/47] update --- api_app/models/domain/resource_template.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api_app/models/domain/resource_template.py b/api_app/models/domain/resource_template.py index d4506218b7..aa213dedef 100644 --- a/api_app/models/domain/resource_template.py +++ b/api_app/models/domain/resource_template.py @@ -34,20 +34,20 @@ class CustomAction(AzureTREModel): class PipelineStepProperty(AzureTREModel): - name: str = Field(None, title="name", description="name of the property to update") - type: str = Field(None, title="type", description="data type of the property to update") + name: str = Field(title="name", description="name of the property to update") + type: str = Field(title="type", description="data type of the property to update") value: Union[dict, str] = Field(None, title="value", description="value to use in substitution for the property to update") arraySubstitutionAction: Optional[str] = Field("", title="Array Substitution Action", description="How to treat existing values of this property in an array [overwrite | append | replace | remove]") arrayMatchField: Optional[str] = Field("", title="Array match field", description="Name of the field to use for finding an item in an array - to replace/remove it") class PipelineStep(AzureTREModel): - stepId: Optional[str] = Field(None, title="stepId", description="Unique id identifying the step") - stepTitle: Optional[str] = Field(None, title="stepTitle", description="Human readable title of what the step is for") - resourceTemplateName: Optional[str] = Field(None, title="resourceTemplateName", description="Name of the template for the resource under change") - resourceType: Optional[ResourceType] = Field(None, title="resourceType", description="Type of resource under change") - resourceAction: Optional[str] = Field(None, title="resourceAction", description="Action - install / upgrade / uninstall etc") - properties: Optional[List[PipelineStepProperty]] = Field(None, title="properties", description="List of properties to update") + stepId: Optional[str] = Field(title="stepId", description="Unique id identifying the step") + stepTitle: Optional[str] = Field(title="stepTitle", description="Human readable title of what the step is for") + resourceTemplateName: Optional[str] = Field(title="resourceTemplateName", description="Name of the template for the resource under change") + resourceType: Optional[ResourceType] = Field(title="resourceType", description="Type of resource under change") + resourceAction: Optional[str] = Field(title="resourceAction", description="Action - install / upgrade / uninstall etc") + properties: Optional[List[PipelineStepProperty]] class Pipeline(AzureTREModel): From 5d9c549bec466461467e507c14d984ecb7cd6000 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Mon, 2 Dec 2024 15:50:50 +0000 Subject: [PATCH 45/47] update --- api_app/models/schemas/workspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/models/schemas/workspace.py b/api_app/models/schemas/workspace.py index dc85db999b..94c6bf861e 100644 --- a/api_app/models/schemas/workspace.py +++ b/api_app/models/schemas/workspace.py @@ -73,7 +73,7 @@ class Config: class WorkspaceInCreate(BaseModel): templateName: str = Field(title="Workspace type", description="Bundle name") - properties: dict = Field(None, title="Workspace parameters", description="Values for the parameters required by the workspace resource specification") + properties: dict = Field({}, title="Workspace parameters", description="Values for the parameters required by the workspace resource specification") class Config: schema_extra = { From 6d581be048568a2d6a09cc1fdec609959a04d7c0 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Wed, 4 Dec 2024 23:38:25 +0000 Subject: [PATCH 46/47] update cli dependencies --- cli/requirements.txt | 12 ++++++------ cli/setup.py | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cli/requirements.txt b/cli/requirements.txt index 57c1f59079..5c398ac38f 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -1,11 +1,11 @@ # if you update this file, update the install_requires in setup.py as well click==8.1.3 -httpx~=0.23.0 -msal==1.26.0 +httpx==0.25.0 +msal==1.31.0 jmespath==1.0.1 tabulate==0.9.0 pygments==2.16.1 -PyJWT==2.8.0 -azure-cli-core==2.57.0 -azure-identity==1.16.1 -aiohttp==3.9.4 +PyJWT==2.9.0 +azure-cli-core==2.65.0 +azure-identity==1.19.0 +aiohttp==3.10.10 diff --git a/cli/setup.py b/cli/setup.py index 04e260921f..2418b4d0fa 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -46,10 +46,10 @@ "jmespath==1.0.1", "tabulate==0.9.0", "pygments==2.16.1", - "PyJWT==2.8.0", - "azure-cli-core==2.57.0", - "azure-identity==1.14.1", - "aiohttp==3.9.4" + "PyJWT==2.9.0", + "azure-cli-core==2.65.0", + "azure-identity==1.19.0", + "aiohttp==3.10.10" ], namespace_packages=[], From 084af4eeab2337af90a5bbaa3004df99c60242b5 Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Wed, 4 Dec 2024 23:39:39 +0000 Subject: [PATCH 47/47] update version --- cli/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/setup.py b/cli/setup.py index 2418b4d0fa..b1313b5542 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -4,7 +4,7 @@ from setuptools import setup PROJECT = 'azure-tre-cli' -VERSION = '0.2.3' +VERSION = '0.2.4' try: long_description = open('README.md', 'rt').read()