diff --git a/terraform/modules/services/gsp/README.md b/terraform/modules/services/gsp/README.md deleted file mode 100644 index 5e00d715..00000000 --- a/terraform/modules/services/gsp/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Modules/Services/GSP - -This module makes -- AWS task definition, one for in-day, one for day-after -- IAM role to setup application -- IAM role for running task -- temp: scheduled aws task, and iam roles diff --git a/terraform/modules/services/gsp/cloudwatch.tf b/terraform/modules/services/gsp/cloudwatch.tf deleted file mode 100644 index b2912704..00000000 --- a/terraform/modules/services/gsp/cloudwatch.tf +++ /dev/null @@ -1,39 +0,0 @@ -# set up cloudwatch log group - -# Define log group -resource "aws_cloudwatch_log_group" "gsp" { - name = var.log-group-name - - retention_in_days = 7 - - tags = { - Environment = var.environment - Application = "nowcasting" - } -} - -# Define log group policy for writing to log group -data "aws_iam_policy_document" "cloudwatch_policy_document" { - statement { - actions = [ - "logs:CreateLogStream", - "logs:PutLogEvents", - "logs:CreateLogGroup", - "logs:DescribeLogStreams", - "logs:DescribeLogGroups", - "logs:DeleteLogGroup", - "logs:PutRetentionPolicy" - ] - - resources = ["arn:aws:logs:*:*:log-group:${var.log-group-name}*"] - } -} - -# Create log group policy -resource "aws_iam_policy" "cloudwatch-gsp" { - name = "cloudwatch-read-and-write-gsp" - path = "/consumer/gsp/" - description = "Policy to allow read and write to cloudwatch logs" - - policy = data.aws_iam_policy_document.cloudwatch_policy_document.json -} diff --git a/terraform/modules/services/gsp/ecs.tf b/terraform/modules/services/gsp/ecs.tf deleted file mode 100644 index 9185829f..00000000 --- a/terraform/modules/services/gsp/ecs.tf +++ /dev/null @@ -1,147 +0,0 @@ -# define aws ecs task definition -# needs access to the internet - -resource "aws_ecs_task_definition" "gsp-task-definition" { - family = "gsp" - requires_compatibilities = ["FARGATE"] - network_mode = "awsvpc" - - tags = { - name = "gsp-consumer" - type = "ecs" - } - - # specific values are needed - - # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html - cpu = 256 - memory = 512 - - task_role_arn = aws_iam_role.consumer-gsp-iam-role.arn - execution_role_arn = var.ecs_task_execution_role_arn - container_definitions = jsonencode([ - { - name = "gsp-consumer" - image = "openclimatefix/gspconsumer:${var.docker_version}" - # cpu = 128 - # memory = 128 - essential = true - - environment : [ - { "name" : "LOGLEVEL", "value" : "DEBUG"}, - { "name" :"REGIME", "value" : "in-day"}, - { "name" :"N_GSPS", "value" : "317"} - ] - - secrets : [ - { - "name" : "DB_URL", - "valueFrom" : "${var.database_secret.arn}:url::", - } - ] - - logConfiguration : { - "logDriver" : "awslogs", - "options" : { - "awslogs-group" : var.log-group-name, - "awslogs-region" : var.region, - "awslogs-stream-prefix" : "streaming" - } - } - } - ]) - - # add volume? So we dont have to keep downloading same docker image -} - -resource "aws_ecs_task_definition" "gsp-day-after-task-definition" { - family = "gsp-day-after" - requires_compatibilities = ["FARGATE"] - network_mode = "awsvpc" - - # specific values are needed - - # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html - cpu = 256 - memory = 512 - - task_role_arn = aws_iam_role.consumer-gsp-iam-role.arn - execution_role_arn = var.ecs_task_execution_role_arn - container_definitions = jsonencode([ - { - name = "gsp-consumer-day-after" - image = "openclimatefix/gspconsumer:${var.docker_version}" - # cpu = 128 - # memory = 128 - essential = true - - environment : [ - { "name" : "LOGLEVEL", "value" : "DEBUG"}, - { "name" :"REGIME", "value" : "day-after"}, - { "name" :"N_GSPS", "value" : "317"}, - ] - - secrets : [ - { - "name" : "DB_URL", - "valueFrom" : "${var.database_secret.arn}:url::", - } - ] - - logConfiguration : { - "logDriver" : "awslogs", - "options" : { - "awslogs-group" : var.log-group-name, - "awslogs-region" : var.region, - "awslogs-stream-prefix" : "streaming" - } - } - } - ]) - - # add volume? So we dont have to keep downloading same docker image -} - -resource "aws_ecs_task_definition" "national-day-after-task-definition" { - family = "national-day-after" - requires_compatibilities = ["FARGATE"] - network_mode = "awsvpc" - - # specific values are needed - - # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html - cpu = 256 - memory = 512 - - task_role_arn = aws_iam_role.consumer-gsp-iam-role.arn - execution_role_arn = var.ecs_task_execution_role_arn - container_definitions = jsonencode([ - { - name = "national-day-after" - image = "openclimatefix/gspconsumer:${var.docker_version}" - # cpu = 128 - # memory = 128 - essential = true - - environment : [ - { "name" : "LOGLEVEL", "value" : "DEBUG"}, - { "name" :"REGIME", "value" : "day-after"}, - { "name" :"N_GSPS", "value" : "0"}, - { "name" :"INCLUDE_NATIONAL", "value" : "True"}, - ] - - secrets : [ - { - "name" : "DB_URL", - "valueFrom" : "${var.database_secret.arn}:url::", - } - ] - - logConfiguration : { - "logDriver" : "awslogs", - "options" : { - "awslogs-group" : var.log-group-name, - "awslogs-region" : var.region, - "awslogs-stream-prefix" : "streaming" - } - } - } - ]) -} diff --git a/terraform/modules/services/gsp/iam.tf b/terraform/modules/services/gsp/iam.tf deleted file mode 100644 index cd78b4ae..00000000 --- a/terraform/modules/services/gsp/iam.tf +++ /dev/null @@ -1,29 +0,0 @@ -# Define the IAM task Instance role, used to run the task - -data "aws_iam_policy_document" "ecs_assumerole_policy_document" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["ecs-tasks.amazonaws.com"] - } - } -} - -# Create the role -resource "aws_iam_role" "consumer-gsp-iam-role" { - name = "consumer-gsp-iam-role" - path = "/consumer/" - assume_role_policy = data.aws_iam_policy_document.ecs_assumerole_policy_document.json -} - -# Attach policies -resource "aws_iam_role_policy_attachment" "attach-logs" { - role = aws_iam_role.consumer-gsp-iam-role.name - policy_arn = aws_iam_policy.cloudwatch-gsp.arn -} -resource "aws_iam_role_policy_attachment" "read-db-secret" { - role = aws_iam_role.consumer-gsp-iam-role.name - policy_arn = var.iam-policy-rds-read-secret.arn -} diff --git a/terraform/modules/services/gsp/variables.tf b/terraform/modules/services/gsp/variables.tf deleted file mode 100644 index 1e8600d3..00000000 --- a/terraform/modules/services/gsp/variables.tf +++ /dev/null @@ -1,36 +0,0 @@ -variable "environment" { - description = "The Deployment environment" -} - - -variable "region" { - description = "The AWS region" -} - -variable "log-group-name" { - description = "The log group name where log streams are saved" - default = "/aws/ecs/consumer/gsp/" -} - -variable "public_subnet_ids" { - type = list(string) - description = "Public subnet ids" -} - -variable "database_secret" { - description = "AWS secret that gives connection details to the database" -} - -variable "iam-policy-rds-read-secret" { - description = "IAM policy to be able to read the RDS secret" -} - -variable "ecs_task_execution_role_arn" { - description = "The arn of the ECS cluster task execution role" - type = string -} - -variable "docker_version" { - description = "The version of the docker that should be used" - default = "0.0.2" -} diff --git a/terraform/nowcasting/development/main.tf b/terraform/nowcasting/development/main.tf index 8522ca0e..06a7e12b 100644 --- a/terraform/nowcasting/development/main.tf +++ b/terraform/nowcasting/development/main.tf @@ -18,7 +18,9 @@ The componentes ares: 3.4 - Satellite Consumer 3.5 - Satellite Data Tailor Clean up 3.6 - PV Consumer -3.7 - GSP Consumer (from PVLive) +3.7 - GSP Consumer (From PVLive) +3.8 - GSP Consumer - GSP Day After +3.9 - GSP Consumer - National Day After 4.1 - Metrics 4.2 - Forecast PVnet 1 4.3 - Forecast National XG @@ -333,16 +335,105 @@ module "pv" { } # 3.7 -module "gsp" { - source = "../../modules/services/gsp" +module "gsp-consumer" { + source = "../../modules/services/ecs_task" - region = var.region - environment = local.environment - public_subnet_ids = module.networking.public_subnet_ids - database_secret = module.database.forecast-database-secret - docker_version = var.gsp_version - iam-policy-rds-read-secret = module.database.iam-policy-forecast-db-read - ecs_task_execution_role_arn = module.ecs.ecs_task_execution_role_arn + ecs-task_name = "gsp" + ecs-task_type = "consumer" + ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn + ecs-task_size = { + cpu = 256 + memory = 512 + storage = 21 + } + + aws-region = var.region + aws-environment = local.environment + + s3-buckets = [] + + container-env_vars = [ + { "name" : "SENTRY_DSN", "value" : var.sentry_dsn }, + { "name" : "ENVIRONMENT", "value" : local.environment }, + { "name" : "LOGLEVEL", "value" : "DEBUG"}, + { "name" :"REGIME", "value" : "in-day"}, + { "name" :"N_GSPS", "value" : "317"} + ] + container-secret_vars = [ + {secret_policy_arn: module.database.forecast-database-secret.arn, + values: ["DB_URL"]} + ] + container-tag = var.gsp_version + container-name = "openclimatefix/gspconsumer" + container-command = [] +} + +# 3.8 +module "gsp-consumer-day-after-gsp" { + source = "../../modules/services/ecs_task" + + ecs-task_name = "gsp-day-after" + ecs-task_type = "consumer" + ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn + ecs-task_size = { + cpu = 256 + memory = 512 + storage = 21 + } + + aws-region = var.region + aws-environment = local.environment + + s3-buckets = [] + + container-env_vars = [ + { "name" : "SENTRY_DSN", "value" : var.sentry_dsn }, + { "name" : "ENVIRONMENT", "value" : local.environment }, + { "name" : "LOGLEVEL", "value" : "DEBUG"}, + { "name" :"REGIME", "value" : "day-after"}, + { "name" :"N_GSPS", "value" : "317"} + ] + container-secret_vars = [ + {secret_policy_arn: module.database.forecast-database-secret.arn, + values: ["DB_URL"]} + ] + container-tag = var.gsp_version + container-name = "openclimatefix/gspconsumer" + container-command = [] +} + +# 3.9 +module "gsp-consumer-day-after-national" { + source = "../../modules/services/ecs_task" + + ecs-task_name = "national-day-after" + ecs-task_type = "consumer" + ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn + ecs-task_size = { + cpu = 256 + memory = 512 + storage = 21 + } + + aws-region = var.region + aws-environment = local.environment + + s3-buckets = [] + + container-env_vars = [ + { "name" : "SENTRY_DSN", "value" : var.sentry_dsn }, + { "name" : "ENVIRONMENT", "value" : local.environment }, + { "name" :"REGIME", "value" : "day-after"}, + { "name" :"N_GSPS", "value" : "0"}, + { "name" :"INCLUDE_NATIONAL", "value" : "True"}, + ] + container-secret_vars = [ + {secret_policy_arn: module.database.forecast-database-secret.arn, + values: ["DB_URL"]} + ] + container-tag = var.gsp_version + container-name = "openclimatefix/gspconsumer" + container-command = [] } # 4.1