From db8128b27c3a95c270c711411b74f242805c712c Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb Date: Fri, 22 Nov 2024 11:05:45 -0500 Subject: [PATCH 01/11] change aws_provider version back to 3.37.0, revert deprecation warning changes from feature/batch to be in accordance with old 3.37.0 implementation. --- .../api-configuration/environment.tpl | 5 +- infrastructure/api.tf | 2 +- infrastructure/database.tf | 2 +- infrastructure/init_terraform.py | 1 + infrastructure/networking.tf | 2 +- infrastructure/provider.tf | 2 +- infrastructure/s3.tf | 56 +++++-------------- infrastructure/variables.tf | 2 +- 8 files changed, 22 insertions(+), 50 deletions(-) diff --git a/infrastructure/api-configuration/environment.tpl b/infrastructure/api-configuration/environment.tpl index 59c8fbcfa..5bcb36411 100644 --- a/infrastructure/api-configuration/environment.tpl +++ b/infrastructure/api-configuration/environment.tpl @@ -7,8 +7,9 @@ DATABASE_PORT=${database_port} DATABASE_USER=${database_user} DATABASE_NAME=${database_name} DATABASE_PASSWORD=${database_password} -AWS_BATCH_JOB_QUEUE_NAME=${aws_batch_job_queue_name} -AWS_BATCH_JOB_DEFINITION_NAME=${aws_batch_job_definition_name} +# TODO: enable batch +# AWS_BATCH_JOB_QUEUE_NAME +# AWS_BATCH_JOB_DEFINITION_NAME AWS_REGION=${aws_region} AWS_S3_BUCKET_NAME=${aws_s3_bucket_name} SENTRY_DSN=${sentry_dsn} diff --git a/infrastructure/api.tf b/infrastructure/api.tf index 3399cc92b..802a1f1bc 100644 --- a/infrastructure/api.tf +++ b/infrastructure/api.tf @@ -50,7 +50,7 @@ resource "aws_instance" "api_server_1" { database_host = aws_db_instance.postgres_db.address database_port = aws_db_instance.postgres_db.port database_user = aws_db_instance.postgres_db.username - database_name = aws_db_instance.postgres_db.db_name + database_name = aws_db_instance.postgres_db.name database_password = var.database_password # TODO: enable batch # aws_batch_job_queue_name = module.batch.job_queue_name diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 0b0ee50a9..f2c924ed5 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -37,7 +37,7 @@ resource "aws_db_instance" "postgres_db" { engine_version = "12.19" auto_minor_version_upgrade = false instance_class = var.database_instance_type - db_name = "scpca_portal" + name = "scpca_portal" port = "5432" username = "scpcapostgresuser" password = var.database_password diff --git a/infrastructure/init_terraform.py b/infrastructure/init_terraform.py index feeeeb007..e2ce81717 100644 --- a/infrastructure/init_terraform.py +++ b/infrastructure/init_terraform.py @@ -15,6 +15,7 @@ def init_terraform(env, user): command = [ "terraform", "init", + "-upgrade", init_bucket, init_key, "-backend-config=dynamodb_table=scpca-portal-terraform-lock", diff --git a/infrastructure/networking.tf b/infrastructure/networking.tf index 3fae8e0ae..2aba34018 100644 --- a/infrastructure/networking.tf +++ b/infrastructure/networking.tf @@ -86,7 +86,7 @@ resource "aws_db_subnet_group" "scpca_portal" { # Get the API a static IP address. resource "aws_eip" "scpca_portal_api_ip" { - domain = "vpc" + vpc = true tags = merge( var.default_tags, diff --git a/infrastructure/provider.tf b/infrastructure/provider.tf index 19bde5775..f08070ba0 100644 --- a/infrastructure/provider.tf +++ b/infrastructure/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "4.0.0" + version = "3.37.0" } } diff --git a/infrastructure/s3.tf b/infrastructure/s3.tf index 39da305dc..939a6a013 100644 --- a/infrastructure/s3.tf +++ b/infrastructure/s3.tf @@ -1,5 +1,6 @@ resource "aws_s3_bucket" "scpca_portal_bucket" { bucket = "scpca-portal-${var.user}-${var.stage}" + acl = "private" force_destroy = var.stage == "prod" ? false : true tags = merge( @@ -11,20 +12,6 @@ resource "aws_s3_bucket" "scpca_portal_bucket" { ) } -resource "aws_s3_bucket_ownership_controls" "scpca_portal_bucket" { - bucket = aws_s3_bucket.scpca_portal_bucket.id - rule { - object_ownership = "BucketOwnerPreferred" - } -} - -resource "aws_s3_bucket_acl" "scpca_portal_bucket" { - depends_on = [aws_s3_bucket_ownership_controls.scpca_portal_bucket] - - bucket = aws_s3_bucket.scpca_portal_bucket.id - acl = "private" -} - resource "aws_s3_bucket_public_access_block" "scpca_portal_bucket" { bucket = aws_s3_bucket.scpca_portal_bucket.id @@ -34,44 +21,27 @@ resource "aws_s3_bucket_public_access_block" "scpca_portal_bucket" { resource "aws_s3_bucket" "scpca_portal_cert_bucket" { bucket = "scpca-portal-cert-${var.user}-${var.stage}" - force_destroy = var.stage == "prod" ? false : true - - tags = merge( - var.default_tags, - { - Name = "scpca-portal-cert-${var.user}-${var.stage}" - Environment = var.stage - } - ) -} - -resource "aws_s3_bucket_ownership_controls" "scpca_portal_cert_bucket" { - bucket = aws_s3_bucket.scpca_portal_cert_bucket.id - rule { - object_ownership = "BucketOwnerPreferred" - } -} - -resource "aws_s3_bucket_acl" "scpca_portal_cert_bucket" { - depends_on = [aws_s3_bucket_ownership_controls.scpca_portal_cert_bucket] - bucket = aws_s3_bucket.scpca_portal_cert_bucket.id acl = "private" -} + force_destroy = var.stage == "prod" ? false : true -resource "aws_s3_bucket_lifecycle_configuration" "scpca_portal_cert_bucket" { - bucket = aws_s3_bucket.scpca_portal_cert_bucket.id - rule { + lifecycle_rule { id = "auto-delete-after-30-days-${var.user}-${var.stage}" - status = "Enabled" - abort_incomplete_multipart_upload { - days_after_initiation = 1 - } + prefix = "" + enabled = true + abort_incomplete_multipart_upload_days = 1 expiration { days = 30 } } + tags = merge( + var.default_tags, + { + Name = "scpca-portal-cert-${var.user}-${var.stage}" + Environment = var.stage + } + ) } resource "aws_s3_bucket_public_access_block" "scpca_portal_cert_bucket" { diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index 5b1dc4475..193da0b95 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -71,7 +71,7 @@ variable "ssh_public_key" { output "environment_variables" { value = [ {name = "DATABASE_NAME" - value = aws_db_instance.postgres_db.db_name}, + value = aws_db_instance.postgres_db.name}, {name = "DATABASE_HOST" value = aws_db_instance.postgres_db.address}, {name = "DATABASE_USER" From 56c64322c0a7095f19ab5365215525abd4f9f00d Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb Date: Fri, 22 Nov 2024 11:30:17 -0500 Subject: [PATCH 02/11] comment out newer 5.12.0 version of aws resources --- infrastructure/api.tf | 3 ++- infrastructure/database.tf | 1 + infrastructure/init_terraform.py | 1 - infrastructure/networking.tf | 1 + infrastructure/provider.tf | 2 +- infrastructure/s3.tf | 45 ++++++++++++++++++++++++++++++++ infrastructure/variables.tf | 1 + 7 files changed, 51 insertions(+), 3 deletions(-) diff --git a/infrastructure/api.tf b/infrastructure/api.tf index 802a1f1bc..381d95a80 100644 --- a/infrastructure/api.tf +++ b/infrastructure/api.tf @@ -25,7 +25,7 @@ data "aws_ami" "ubuntu" { resource "aws_instance" "api_server_1" { ami = data.aws_ami.ubuntu.id instance_type = var.api_instance_type - availability_zone = "${var.region}a" + availability_zone = "${var.region}a"api.tf vpc_security_group_ids = [aws_security_group.scpca_portal_api.id] iam_instance_profile = aws_iam_instance_profile.scpca_portal_instance_profile.name subnet_id = aws_subnet.scpca_portal_1a.id @@ -51,6 +51,7 @@ resource "aws_instance" "api_server_1" { database_port = aws_db_instance.postgres_db.port database_user = aws_db_instance.postgres_db.username database_name = aws_db_instance.postgres_db.name + # database_name = aws_db_instance.postgres_db.db_name database_password = var.database_password # TODO: enable batch # aws_batch_job_queue_name = module.batch.job_queue_name diff --git a/infrastructure/database.tf b/infrastructure/database.tf index f2c924ed5..e0f925ed1 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -38,6 +38,7 @@ resource "aws_db_instance" "postgres_db" { auto_minor_version_upgrade = false instance_class = var.database_instance_type name = "scpca_portal" + # db_name = "scpca_portal" port = "5432" username = "scpcapostgresuser" password = var.database_password diff --git a/infrastructure/init_terraform.py b/infrastructure/init_terraform.py index e2ce81717..feeeeb007 100644 --- a/infrastructure/init_terraform.py +++ b/infrastructure/init_terraform.py @@ -15,7 +15,6 @@ def init_terraform(env, user): command = [ "terraform", "init", - "-upgrade", init_bucket, init_key, "-backend-config=dynamodb_table=scpca-portal-terraform-lock", diff --git a/infrastructure/networking.tf b/infrastructure/networking.tf index 2aba34018..24932ed4e 100644 --- a/infrastructure/networking.tf +++ b/infrastructure/networking.tf @@ -87,6 +87,7 @@ resource "aws_db_subnet_group" "scpca_portal" { # Get the API a static IP address. resource "aws_eip" "scpca_portal_api_ip" { vpc = true + # domain = "vpc" tags = merge( var.default_tags, diff --git a/infrastructure/provider.tf b/infrastructure/provider.tf index bef938fad..1d638ba78 100644 --- a/infrastructure/provider.tf +++ b/infrastructure/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "3.76.1" + version = "3.37.0" } } } diff --git a/infrastructure/s3.tf b/infrastructure/s3.tf index 939a6a013..b13f289f6 100644 --- a/infrastructure/s3.tf +++ b/infrastructure/s3.tf @@ -1,5 +1,6 @@ resource "aws_s3_bucket" "scpca_portal_bucket" { bucket = "scpca-portal-${var.user}-${var.stage}" + # TODO: remove this when upgrading aws_provider version acl = "private" force_destroy = var.stage == "prod" ? false : true @@ -12,6 +13,20 @@ resource "aws_s3_bucket" "scpca_portal_bucket" { ) } +# resource "aws_s3_bucket_ownership_controls" "scpca_portal_bucket" { +# bucket = aws_s3_bucket.scpca_portal_bucket.id +# rule { +# object_ownership = "BucketOwnerPreferred" +# } +#} + +# resource "aws_s3_bucket_acl" "scpca_portal_bucket" { +# depends_on = [aws_s3_bucket_ownership_controls.scpca_portal_bucket] +# +# bucket = aws_s3_bucket.scpca_portal_bucket.id +# acl = "private" +#} + resource "aws_s3_bucket_public_access_block" "scpca_portal_bucket" { bucket = aws_s3_bucket.scpca_portal_bucket.id @@ -21,9 +36,11 @@ resource "aws_s3_bucket_public_access_block" "scpca_portal_bucket" { resource "aws_s3_bucket" "scpca_portal_cert_bucket" { bucket = "scpca-portal-cert-${var.user}-${var.stage}" + # TODO: remove this when upgrading aws_provider version acl = "private" force_destroy = var.stage == "prod" ? false : true + # TODO: remove lifecycle rule when we upgrade aws_provider version lifecycle_rule { id = "auto-delete-after-30-days-${var.user}-${var.stage}" prefix = "" @@ -44,6 +61,34 @@ resource "aws_s3_bucket" "scpca_portal_cert_bucket" { ) } +# resource "aws_s3_bucket_ownership_controls" "scpca_portal_cert_bucket" { +# bucket = aws_s3_bucket.scpca_portal_cert_bucket.id +# rule { +# object_ownership = "BucketOwnerPreferred" +# } +#} + +# resource "aws_s3_bucket_acl" "scpca_portal_cert_bucket" { +# depends_on = [aws_s3_bucket_ownership_controls.scpca_portal_cert_bucket] +# bucket = aws_s3_bucket.scpca_portal_cert_bucket.id +# acl = "private" +#} + +# resource "aws_s3_bucket_lifecycle_configuration" "scpca_portal_cert_bucket" { +# bucket = aws_s3_bucket.scpca_portal_cert_bucket.id +# rule { +# id = "auto-delete-after-30-days-${var.user}-${var.stage}" +# status = "Enabled" +# abort_incomplete_multipart_upload { +# days_after_initiation = 1 +# } +# +# expiration { +# days = 30 +# } +# } +#} + resource "aws_s3_bucket_public_access_block" "scpca_portal_cert_bucket" { bucket = aws_s3_bucket.scpca_portal_cert_bucket.id diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index 193da0b95..a0ff7ab96 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -72,6 +72,7 @@ output "environment_variables" { value = [ {name = "DATABASE_NAME" value = aws_db_instance.postgres_db.name}, + # value = aws_db_instance.postgres_db.db_name}, {name = "DATABASE_HOST" value = aws_db_instance.postgres_db.address}, {name = "DATABASE_USER" From ca9fe3d69f5fa56c8db7439e7d55087a06d897ef Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb Date: Fri, 22 Nov 2024 11:32:24 -0500 Subject: [PATCH 03/11] fix typo --- infrastructure/api.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/api.tf b/infrastructure/api.tf index 381d95a80..e3ff9ec6f 100644 --- a/infrastructure/api.tf +++ b/infrastructure/api.tf @@ -25,7 +25,7 @@ data "aws_ami" "ubuntu" { resource "aws_instance" "api_server_1" { ami = data.aws_ami.ubuntu.id instance_type = var.api_instance_type - availability_zone = "${var.region}a"api.tf + availability_zone = "${var.region}a" vpc_security_group_ids = [aws_security_group.scpca_portal_api.id] iam_instance_profile = aws_iam_instance_profile.scpca_portal_instance_profile.name subnet_id = aws_subnet.scpca_portal_1a.id From 1c2d7848ee505ea5049f9c0db09163cb32b0f7dd Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb Date: Fri, 22 Nov 2024 11:35:44 -0500 Subject: [PATCH 04/11] add upgrade flag to init_terraform script --- infrastructure/init_terraform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/init_terraform.py b/infrastructure/init_terraform.py index feeeeb007..e2ce81717 100644 --- a/infrastructure/init_terraform.py +++ b/infrastructure/init_terraform.py @@ -15,6 +15,7 @@ def init_terraform(env, user): command = [ "terraform", "init", + "-upgrade", init_bucket, init_key, "-backend-config=dynamodb_table=scpca-portal-terraform-lock", From e53873041c385dfb8eefd4180ce70bf8a58c79d2 Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb <108290446+avrohomgottlieb@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:42:06 -0500 Subject: [PATCH 05/11] Update infrastructure/variables.tf Co-authored-by: David --- infrastructure/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index a0ff7ab96..d25dfa3fb 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -72,6 +72,7 @@ output "environment_variables" { value = [ {name = "DATABASE_NAME" value = aws_db_instance.postgres_db.name}, + # TODO: replace db_name with name after upgrade # value = aws_db_instance.postgres_db.db_name}, {name = "DATABASE_HOST" value = aws_db_instance.postgres_db.address}, From 8d8a88fafba3001bdac493940d17a93b6c9d49be Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb <108290446+avrohomgottlieb@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:42:32 -0500 Subject: [PATCH 06/11] Update infrastructure/api.tf Co-authored-by: David --- infrastructure/api.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/api.tf b/infrastructure/api.tf index e3ff9ec6f..a7dddb1c2 100644 --- a/infrastructure/api.tf +++ b/infrastructure/api.tf @@ -51,6 +51,7 @@ resource "aws_instance" "api_server_1" { database_port = aws_db_instance.postgres_db.port database_user = aws_db_instance.postgres_db.username database_name = aws_db_instance.postgres_db.name + # TODO: enable after upgrade # database_name = aws_db_instance.postgres_db.db_name database_password = var.database_password # TODO: enable batch From 11105d63fdf77c19e8ade33da3615e703c77e3af Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb <108290446+avrohomgottlieb@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:42:44 -0500 Subject: [PATCH 07/11] Update infrastructure/s3.tf Co-authored-by: David --- infrastructure/s3.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/s3.tf b/infrastructure/s3.tf index b13f289f6..98d93f9d3 100644 --- a/infrastructure/s3.tf +++ b/infrastructure/s3.tf @@ -61,6 +61,7 @@ resource "aws_s3_bucket" "scpca_portal_cert_bucket" { ) } +# TODO: enable after upgrade # resource "aws_s3_bucket_ownership_controls" "scpca_portal_cert_bucket" { # bucket = aws_s3_bucket.scpca_portal_cert_bucket.id # rule { From d049c27c602b37a83c026e9675326dbc2cc8cd1b Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb <108290446+avrohomgottlieb@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:42:55 -0500 Subject: [PATCH 08/11] Update infrastructure/s3.tf Co-authored-by: David --- infrastructure/s3.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/s3.tf b/infrastructure/s3.tf index 98d93f9d3..efdfdfd84 100644 --- a/infrastructure/s3.tf +++ b/infrastructure/s3.tf @@ -20,6 +20,7 @@ resource "aws_s3_bucket" "scpca_portal_bucket" { # } #} +# TODO: enable after upgrade # resource "aws_s3_bucket_acl" "scpca_portal_bucket" { # depends_on = [aws_s3_bucket_ownership_controls.scpca_portal_bucket] # From 3a0fc2f88733f8d54a7c466a6840c0cfafc0e378 Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb <108290446+avrohomgottlieb@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:43:06 -0500 Subject: [PATCH 09/11] Update infrastructure/s3.tf Co-authored-by: David --- infrastructure/s3.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/s3.tf b/infrastructure/s3.tf index efdfdfd84..c89e8ee1d 100644 --- a/infrastructure/s3.tf +++ b/infrastructure/s3.tf @@ -13,6 +13,7 @@ resource "aws_s3_bucket" "scpca_portal_bucket" { ) } +# TODO: enable after upgrade # resource "aws_s3_bucket_ownership_controls" "scpca_portal_bucket" { # bucket = aws_s3_bucket.scpca_portal_bucket.id # rule { From 127f3721c32797e67bdb0452b35bac1b24e01035 Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb <108290446+avrohomgottlieb@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:43:19 -0500 Subject: [PATCH 10/11] Update infrastructure/networking.tf Co-authored-by: David --- infrastructure/networking.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/networking.tf b/infrastructure/networking.tf index 24932ed4e..6640ebd07 100644 --- a/infrastructure/networking.tf +++ b/infrastructure/networking.tf @@ -87,6 +87,7 @@ resource "aws_db_subnet_group" "scpca_portal" { # Get the API a static IP address. resource "aws_eip" "scpca_portal_api_ip" { vpc = true + # TODO: replace vpc with domain after upgrade # domain = "vpc" tags = merge( From 923d24f929f41a9fcb494f12ea739312c6361621 Mon Sep 17 00:00:00 2001 From: Avrohom Gottlieb <108290446+avrohomgottlieb@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:43:32 -0500 Subject: [PATCH 11/11] Update infrastructure/database.tf Co-authored-by: David --- infrastructure/database.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index e0f925ed1..cbf331b07 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -38,6 +38,7 @@ resource "aws_db_instance" "postgres_db" { auto_minor_version_upgrade = false instance_class = var.database_instance_type name = "scpca_portal" + # TODO: replace db_name with name after upgrade # db_name = "scpca_portal" port = "5432" username = "scpcapostgresuser"