From f0090630ab6d50bbdcf27cf0f9709d62a93b25db Mon Sep 17 00:00:00 2001 From: Bryn Noel Ubald <55503826+bnubald@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:43:49 +0100 Subject: [PATCH 1/7] Fix #54: Update azurerm provider to newer version --- terraform/backend.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/backend.tf b/terraform/backend.tf index c1c87e2..1a868a1 100644 --- a/terraform/backend.tf +++ b/terraform/backend.tf @@ -6,7 +6,7 @@ terraform { } azurerm = { source = "hashicorp/azurerm" - version = "=3.59.0" + version = "=3.90.0" } null = { source = "hashicorp/null" From c6c6f34a47ffee467783986ce57a5e44a805f39c Mon Sep 17 00:00:00 2001 From: Bryn Noel Ubald <55503826+bnubald@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:50:02 +0100 Subject: [PATCH 2/7] Fix #55: Enable flag to build dependencies during deployment - fix pandas --- terraform/application/app.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/application/app.tf b/terraform/application/app.tf index cb9844f..b14caf4 100644 --- a/terraform/application/app.tf +++ b/terraform/application/app.tf @@ -32,7 +32,7 @@ resource "azurerm_linux_web_app" "this" { "ICENET_AUTH_LIST" = "/data/auth_list.json" "ICENET_DATA_LOCATION" = "/data" # "ENABLE_ORYX_BUILD" = "true" -# "SCM_DO_BUILD_DURING_DEPLOYMENT" = "true" + "SCM_DO_BUILD_DURING_DEPLOYMENT" = "true" } storage_account { From dfd8571cacebc4d4ef065b67b466c3744cadf819 Mon Sep 17 00:00:00 2001 From: Bryn Noel Ubald <55503826+bnubald@users.noreply.github.com> Date: Sun, 3 Nov 2024 15:26:57 +0000 Subject: [PATCH 3/7] Resolves #60: Allows specifying an SAS token for Blob read access Service Principal option also included, but commented out --- terraform/forecast_processor/functions.tf | 35 ++++++++--------------- terraform/forecast_processor/variables.tf | 26 +++++++++++++++++ terraform/main.tf | 5 ++++ terraform/variables.tf | 17 +++++++++++ 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/terraform/forecast_processor/functions.tf b/terraform/forecast_processor/functions.tf index b3f3251..6c03d30 100644 --- a/terraform/forecast_processor/functions.tf +++ b/terraform/forecast_processor/functions.tf @@ -121,6 +121,13 @@ resource "azurerm_linux_function_app" "this" { # enabled which mounts over the contents of the container. # https://github.com/Azure/azure-functions-docker/issues/642 "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = "false" + # Use an SAS Token with Blob read/list access + "AZURE_STORAGE_SAS_TOKEN" = var.storage_sas_token + # Use a service principal with blob read/list access (use either SAS token or service principal) + #"AZURE_CLIENT_ID" = var.service_principal_client_id + #"AZURE_CLIENT_SECRET" = var.service_principal_secret_val + #"AZURE_TENANT_ID" = var.tenant_id + #"AZURE_SUBSCRIPTION_ID" = var.subscription_id } identity { type = "SystemAssigned" @@ -140,29 +147,11 @@ resource "azurerm_linux_function_app" "this" { } } -resource "azurerm_role_definition" "app_data_read" { - description = "Allows for read access to Azure Storage blob containers and data" - name = "${local.app_name}-role-read-forecast-data" - scope = var.data_storage_account.id - - permissions { - actions = [ - "Microsoft.Storage/storageAccounts/blobServices/containers/read", - "Microsoft.Storage/storageAccounts/blobServices/generateUserDelegationKey/action", - ] - data_actions = [ - "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read", - ] - not_actions = [] - not_data_actions = [] - } -} - -resource "azurerm_role_assignment" "app_data_read_assoc" { - scope = var.data_storage_account.id - role_definition_id = azurerm_role_definition.app_data_read.role_definition_resource_id - principal_id = azurerm_linux_function_app.this.identity.0.principal_id -} +#resource "azurerm_role_assignment" "storage_blob_data_reader_assoc" { +# scope = var.data_storage_account.id +# role_definition_name = "Storage Blob Data Reader" +# principal_id = azurerm_linux_function_app.this.identity.0.principal_id +#} #resource "azurerm_private_endpoint" "event_proc_endpoint" { # name = "pvt-${var.project_name}-event-processing" diff --git a/terraform/forecast_processor/variables.tf b/terraform/forecast_processor/variables.tf index e010966..e13cef8 100644 --- a/terraform/forecast_processor/variables.tf +++ b/terraform/forecast_processor/variables.tf @@ -53,6 +53,32 @@ variable "default_tags" { default = {} } +variable "subscription_id" { + description = "Which Azure subscription to build in" + type = string +} +variable "tenant_id" { + description = "Which Azure tenant to build in" + type = string +} +variable "storage_sas_token" { + description = "Blob storage SAS token" + type = string + sensitive = true +} +variable "service_principal_client_id" { + description = "The special client/app ID, generated service principal for read/list blob storage access" + type = string + default = null + sensitive = true +} +variable "service_principal_secret_val" { + description = "Secret ID of above service principal" + type = string + default = null + sensitive = true +} + # Local variables locals { tags = merge( diff --git a/terraform/main.tf b/terraform/main.tf index 6dcc5ac..ddb1930 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -101,6 +101,11 @@ module "forecast_processor" { notification_email = var.notification_email sendfrom_email = var.sendfrom_email dns_zone = module.network.dns_zone + storage_sas_token = var.storage_sas_token + service_principal_client_id = var.service_principal_client_id + service_principal_secret_val = var.service_principal_secret_val + tenant_id = var.tenant_id + subscription_id = var.subscription_id } module "web" { diff --git a/terraform/variables.tf b/terraform/variables.tf index 8710a9d..18ecb26 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -15,6 +15,23 @@ variable "tenant_id" { description = "Which Azure tenant to build in" type = string } +variable "storage_sas_token" { + description = "Blob storage SAS token" + type = string + sensitive = true +} +variable "service_principal_client_id" { + description = "The special client/app ID, generated service principal for read/list blob storage access" + type = string + default = null + sensitive = true +} +variable "service_principal_secret_val" { + description = "Secret ID of above service principal" + type = string + default = null + sensitive = true +} # These have sensible defaults variable "domain_name" { From 27957cad0a5894976001006256d6e42786deda60 Mon Sep 17 00:00:00 2001 From: Bryn Noel Ubald <55503826+bnubald@users.noreply.github.com> Date: Sun, 3 Nov 2024 16:13:30 +0000 Subject: [PATCH 4/7] Resolves #60: Fix secret value variable description --- terraform/forecast_processor/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/forecast_processor/variables.tf b/terraform/forecast_processor/variables.tf index e13cef8..033b699 100644 --- a/terraform/forecast_processor/variables.tf +++ b/terraform/forecast_processor/variables.tf @@ -73,7 +73,7 @@ variable "service_principal_client_id" { sensitive = true } variable "service_principal_secret_val" { - description = "Secret ID of above service principal" + description = "Secret value of above service principal" type = string default = null sensitive = true From b2c7095ee5132c3faf991ff198050cb88e67e8f7 Mon Sep 17 00:00:00 2001 From: Bryn Noel Ubald <55503826+bnubald@users.noreply.github.com> Date: Sun, 3 Nov 2024 16:25:58 +0000 Subject: [PATCH 5/7] Fixes #48: Updates python version for dashboard flask app to 3.9 --- terraform/application/app.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/application/app.tf b/terraform/application/app.tf index b14caf4..794a444 100644 --- a/terraform/application/app.tf +++ b/terraform/application/app.tf @@ -23,7 +23,7 @@ resource "azurerm_linux_web_app" "this" { use_32_bit_worker = false always_on = true application_stack { - python_version = "3.8" + python_version = "3.9" } app_command_line = "gunicorn icenet_app.app:app" } From 53dfbe7fa8cd926065325b46e0bfe047be5677ab Mon Sep 17 00:00:00 2001 From: Bryn Noel Ubald <55503826+bnubald@users.noreply.github.com> Date: Sun, 3 Nov 2024 16:27:02 +0000 Subject: [PATCH 6/7] Potentially add logging for flask dashboard app --- terraform/application/app.tf | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/terraform/application/app.tf b/terraform/application/app.tf index 794a444..a7e0ccf 100644 --- a/terraform/application/app.tf +++ b/terraform/application/app.tf @@ -33,6 +33,7 @@ resource "azurerm_linux_web_app" "this" { "ICENET_DATA_LOCATION" = "/data" # "ENABLE_ORYX_BUILD" = "true" "SCM_DO_BUILD_DURING_DEPLOYMENT" = "true" + "DOCKER_ENABLE_CI" = "true" } storage_account { @@ -44,6 +45,18 @@ resource "azurerm_linux_web_app" "this" { mount_path = "/data" } + logs { + application_logs { + file_system_level = "Information" + } + http_logs { + file_system { + retention_in_days = 7 + retention_in_mb = 100 + } + } + } + tags = local.tags } From 09eadc8724f942e55152b8c35861dde41c6a9da5 Mon Sep 17 00:00:00 2001 From: Bryn Noel Ubald <55503826+bnubald@users.noreply.github.com> Date: Sun, 3 Nov 2024 16:27:29 +0000 Subject: [PATCH 7/7] Fix secret value being described as secret ID --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 18ecb26..b689a0b 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -27,7 +27,7 @@ variable "service_principal_client_id" { sensitive = true } variable "service_principal_secret_val" { - description = "Secret ID of above service principal" + description = "Secret value of above service principal" type = string default = null sensitive = true