Skip to content

Commit

Permalink
Merge pull request #87 from dxw/conditionally-create-transfer-s3-bucket
Browse files Browse the repository at this point in the history
Conditionally create transfer S3 bucket
  • Loading branch information
Stretch96 authored May 31, 2024
2 parents c7d75eb + d4f76b5 commit fdc6c28
Show file tree
Hide file tree
Showing 10 changed files with 230 additions and 1 deletion.
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_s3_bucket.infrastructure_ecs_cluster_service_build_pipeline_buildspec_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
| [aws_s3_bucket.infrastructure_ecs_cluster_service_environment_files](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
| [aws_s3_bucket.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
| [aws_s3_bucket.infrastructure_vpc_transfer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
| [aws_s3_bucket_acl.infrastructure_logs_log_delivery_write](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource |
| [aws_s3_bucket_lifecycle_configuration.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_lifecycle_configuration) | resource |
| [aws_s3_bucket_lifecycle_configuration.infrastructure_ecs_cluster_service_alb_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_lifecycle_configuration) | resource |
Expand All @@ -230,6 +231,7 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_s3_bucket_logging.infrastructure_ecs_cluster_service_build_pipeline_artifact_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_logging) | resource |
| [aws_s3_bucket_logging.infrastructure_ecs_cluster_service_build_pipeline_buildspec_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_logging) | resource |
| [aws_s3_bucket_logging.infrastructure_ecs_cluster_service_environment_files](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_logging) | resource |
| [aws_s3_bucket_logging.infrastructure_vpc_transfer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_logging) | resource |
| [aws_s3_bucket_ownership_controls.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_ownership_controls) | resource |
| [aws_s3_bucket_ownership_controls.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_ownership_controls) | resource |
| [aws_s3_bucket_policy.cloudformation_custom_stack_template_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
Expand All @@ -239,27 +241,31 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_s3_bucket_policy.infrastructure_ecs_cluster_service_build_pipeline_buildspec_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
| [aws_s3_bucket_policy.infrastructure_ecs_cluster_service_environment_files](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
| [aws_s3_bucket_policy.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
| [aws_s3_bucket_policy.infrastructure_vpc_transfer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
| [aws_s3_bucket_public_access_block.cloudformation_custom_stack_template_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_public_access_block.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_public_access_block.infrastructure_ecs_cluster_service_alb_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_public_access_block.infrastructure_ecs_cluster_service_build_pipeline_artifact_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_public_access_block.infrastructure_ecs_cluster_service_build_pipeline_buildspec_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_public_access_block.infrastructure_ecs_cluster_service_environment_files](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_public_access_block.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_public_access_block.infrastructure_vpc_transfer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.cloudformation_custom_stack_template_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.infrastructure_ecs_cluster_service_alb_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.infrastructure_ecs_cluster_service_build_pipeline_artifact_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.infrastructure_ecs_cluster_service_build_pipeline_buildspec_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.infrastructure_ecs_cluster_service_environment_files](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.infrastructure_vpc_transfer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
| [aws_s3_bucket_versioning.cloudformation_custom_stack_template_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
| [aws_s3_bucket_versioning.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
| [aws_s3_bucket_versioning.infrastructure_ecs_cluster_service_alb_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
| [aws_s3_bucket_versioning.infrastructure_ecs_cluster_service_build_pipeline_artifact_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
| [aws_s3_bucket_versioning.infrastructure_ecs_cluster_service_build_pipeline_buildspec_store](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
| [aws_s3_bucket_versioning.infrastructure_ecs_cluster_service_environment_files](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
| [aws_s3_bucket_versioning.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
| [aws_s3_bucket_versioning.infrastructure_vpc_transfer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
| [aws_s3_object.infrastructure_ecs_cluster_service_build_pipeline_buildspec_store_files](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
| [aws_secretsmanager_secret.infrastructure_rds_root_password](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource |
| [aws_secretsmanager_secret_version.infrastructure_rds_root_password](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource |
Expand All @@ -284,6 +290,8 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_security_group_rule.infrastructure_rds_ingress_tcp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
| [aws_sns_topic.infrastructure_ecs_cluster_autoscaling_lifecycle_termination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource |
| [aws_sns_topic_subscription.ecs_cluster_infrastructure_draining_autoscaling_lifecycle_termination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource |
| [aws_ssm_document.infrastructure_vpc_transfer_s3_download](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_document) | resource |
| [aws_ssm_document.infrastructure_vpc_transfer_s3_upload](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_document) | resource |
| [aws_subnet.infrastructure_private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource |
| [aws_subnet.infrastructure_public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource |
| [aws_vpc.infrastructure](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc) | resource |
Expand Down Expand Up @@ -320,6 +328,7 @@ This project creates and manages resources within an AWS account for infrastruct
| <a name="input_enable_infrastructure_ecs_cluster_efs"></a> [enable\_infrastructure\_ecs\_cluster\_efs](#input\_enable\_infrastructure\_ecs\_cluster\_efs) | Conditionally create and mount EFS to the ECS cluster instances | `bool` | n/a | yes |
| <a name="input_enable_infrastructure_ecs_cluster_services_alb_logs"></a> [enable\_infrastructure\_ecs\_cluster\_services\_alb\_logs](#input\_enable\_infrastructure\_ecs\_cluster\_services\_alb\_logs) | Enable Infrastructure ECS cluster services ALB logs | `bool` | n/a | yes |
| <a name="input_enable_infrastructure_route53_hosted_zone"></a> [enable\_infrastructure\_route53\_hosted\_zone](#input\_enable\_infrastructure\_route53\_hosted\_zone) | Creates a Route53 hosted zone, where DNS records will be created for resources launched within this module. | `bool` | n/a | yes |
| <a name="input_enable_infrastructure_vpc_transfer_s3_bucket"></a> [enable\_infrastructure\_vpc\_transfer\_s3\_bucket](#input\_enable\_infrastructure\_vpc\_transfer\_s3\_bucket) | Enable VPC transfer S3 bucket. This allows uploading/downloading files from resources within the infrastructure VPC | `bool` | n/a | yes |
| <a name="input_environment"></a> [environment](#input\_environment) | The environment name to be used as part of the resource prefix | `string` | n/a | yes |
| <a name="input_infrastructure_dockerhub_email"></a> [infrastructure\_dockerhub\_email](#input\_infrastructure\_dockerhub\_email) | Dockerhub email | `string` | n/a | yes |
| <a name="input_infrastructure_dockerhub_token"></a> [infrastructure\_dockerhub\_token](#input\_infrastructure\_dockerhub\_token) | Dockerhub token which has permissions to pull images | `string` | n/a | yes |
Expand Down Expand Up @@ -373,6 +382,7 @@ This project creates and manages resources within an AWS account for infrastruct
| <a name="input_infrastructure_vpc_network_availability_zones"></a> [infrastructure\_vpc\_network\_availability\_zones](#input\_infrastructure\_vpc\_network\_availability\_zones) | A list of availability zone characters (eg. ["a", "b", "c"]) | `list(string)` | n/a | yes |
| <a name="input_infrastructure_vpc_network_enable_private"></a> [infrastructure\_vpc\_network\_enable\_private](#input\_infrastructure\_vpc\_network\_enable\_private) | Enable private networking on Infrastructure VPC. This will create subnets with a route to a NAT Gateway (If Public networking has been enabled) | `bool` | n/a | yes |
| <a name="input_infrastructure_vpc_network_enable_public"></a> [infrastructure\_vpc\_network\_enable\_public](#input\_infrastructure\_vpc\_network\_enable\_public) | Enable public networking on Infrastructure VPC. This will create subnets with a route to an Internet Gateway | `bool` | n/a | yes |
| <a name="input_infrastructure_vpc_transfer_s3_bucket_access_vpc_ids"></a> [infrastructure\_vpc\_transfer\_s3\_bucket\_access\_vpc\_ids](#input\_infrastructure\_vpc\_transfer\_s3\_bucket\_access\_vpc\_ids) | Additional VPC ids which are allowed to access the transfer S3 bucket | `list(string)` | n/a | yes |
| <a name="input_project_name"></a> [project\_name](#input\_project\_name) | Project name to be used as a prefix for all resources | `string` | n/a | yes |
| <a name="input_route53_root_hosted_zone_domain_name"></a> [route53\_root\_hosted\_zone\_domain\_name](#input\_route53\_root\_hosted\_zone\_domain\_name) | Route53 Hosted Zone in which to delegate Infrastructure Route53 Hosted Zones. | `string` | n/a | yes |

Expand Down
7 changes: 7 additions & 0 deletions kms-infrastructure.tf
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,15 @@ resource "aws_kms_key" "infrastructure" {
{
account_id = (local.infrastructure_vpc_flow_logs_s3_with_athena || local.enable_cloudformatian_s3_template_store || contains([for service in local.infrastructure_ecs_cluster_services : service["cloudfront_access_logging_enabled"]], true)) || length(local.custom_s3_buckets) > 0 && local.infrastructure_kms_encryption ? local.aws_account_id : ""
region = local.aws_region
})}${local.enable_infrastructure_vpc_transfer_s3_bucket ? "," : ""}
${templatefile("${path.root}/policies/kms-key-policy-statements/vpc-id-and-s3-bucket-allow.json.tpl",
{
vpc_ids = jsonencode(local.infrastructure_vpc_transfer_s3_bucket_access_vpc_ids)
region = local.aws_region
bucket_arn = aws_s3_bucket.infrastructure_vpc_transfer[0].arn
}
)}
]
EOT
}
Expand Down
11 changes: 10 additions & 1 deletion locals.tf
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ locals {
local.infrastructure_vpc_flow_logs_s3_with_athena ||
length(local.infrastructure_ecs_cluster_services) != 0 ||
length(local.custom_s3_buckets) != 0 ||
local.enable_cloudformatian_s3_template_store
local.enable_cloudformatian_s3_template_store ||
local.enable_infrastructure_vpc_transfer_s3_bucket
)
logs_bucket_s3_source_arns = concat(
length(local.infrastructure_ecs_cluster_services) != 0 ? [aws_s3_bucket.infrastructure_ecs_cluster_service_build_pipeline_artifact_store[0].arn] : [],
local.enable_infrastructure_vpc_transfer_s3_bucket ? [aws_s3_bucket.infrastructure_vpc_transfer[0].arn] : [],
[for k, v in local.custom_s3_buckets : aws_s3_bucket.custom[k].arn]
)
logs_bucket_logs_source_arns = concat(
Expand Down Expand Up @@ -97,6 +99,13 @@ locals {
date = "string",
hour = "string"
}
enable_infrastructure_vpc_transfer_s3_bucket = var.enable_infrastructure_vpc_transfer_s3_bucket
infrastructure_vpc_transfer_s3_bucket_access_vpc_ids = concat(
local.infrastructure_vpc ? [aws_vpc.infrastructure[0].id] : [],
var.infrastructure_vpc_transfer_s3_bucket_access_vpc_ids
)
infrastructure_vpc_transfer_ssm_download_command = "aws s3 cp {{ Source }} {{ HostTarget }} {{ Recursive }}; if [ -n \\\"{{ TargetUID }}\\\" ] && [ -n \\\"{{ TargetGID }}\\\" ]; then chown {{ TargetUID }}:{{ TargetGID }} -R {{ HostTarget }}; fi"
infrastructure_vpc_transfer_ssm_upload_command = "aws s3 cp {{ Source }} {{ S3Target }} {{ Recursive }}"

infrastructure_dockerhub_email = var.infrastructure_dockerhub_email
infrastructure_dockerhub_username = var.infrastructure_dockerhub_username
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
%{if vpc_ids != "[]"}{
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": [
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*"
],
"Resource": "*",
"Condition": {
"StringEquals": {
"aws:sourceVpc": ${vpc_ids},
"kms:ViaService": "s3.${region}.amazonaws.com",
"kms:EncryptionContext:SourceARN": "${bucket_arn}"
}
}
}%{endif}
11 changes: 11 additions & 0 deletions policies/s3-bucket-policy-statements/vpc-rw.json.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"Effect": "Allow",
"Principal": "*",
"Action": "s3:*",
"Resource": "${bucket_arn}/*",
"Condition": {
"StringEquals": {
"aws:sourceVpc": ${vpc_ids}
}
}
}
40 changes: 40 additions & 0 deletions ssm-documents/s3-download.json.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
{
"schemaVersion": "1.0",
"description": "Document to download from S3",
"sessionType": "InteractiveCommands",
"parameters": {
"TargetUID": {
"type": "String",
"description": "The default unix UID of the target system",
"default": ""
},
"TargetGID": {
"type": "String",
"description": "The default unix GID of the target system",
"default": ""
},
"Source": {
"type": "String",
"description": "S3 Bucket source path"
},
"HostTarget": {
"type": "String",
"description": "Target path on instance"
},
"Recursive": {
"type": "String",
"description": "Recursive copy",
"default": "--ignore-glacier-warnings",
"allowedValues": [
"--ignore-glacier-warnings",
"--recursive"
]
}
},
"properties": {
"linux": {
"commands": "${command}",
"runAsElevated": true
}
}
}
30 changes: 30 additions & 0 deletions ssm-documents/s3-upload.json.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{
"schemaVersion": "1.0",
"description": "Document to uplaod to S3",
"sessionType": "InteractiveCommands",
"parameters": {
"Source": {
"type": "String",
"description": "file source path"
},
"S3Target": {
"type": "String",
"description": "s3 bucket path"
},
"Recursive": {
"type": "String",
"description": "Recursive copy",
"default": "--ignore-glacier-warnings",
"allowedValues": [
"--ignore-glacier-warnings",
"--recursive"
]
}
},
"properties": {
"linux": {
"commands": "${command}",
"runAsElevated": true
}
}
}
10 changes: 10 additions & 0 deletions variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,16 @@ variable "infrastructure_vpc_network_acl_ingress_custom_rules_public" {
}))
}

variable "enable_infrastructure_vpc_transfer_s3_bucket" {
description = "Enable VPC transfer S3 bucket. This allows uploading/downloading files from resources within the infrastructure VPC"
type = bool
}

variable "infrastructure_vpc_transfer_s3_bucket_access_vpc_ids" {
description = "Additional VPC ids which are allowed to access the transfer S3 bucket"
type = list(string)
}

variable "route53_root_hosted_zone_domain_name" {
description = "Route53 Hosted Zone in which to delegate Infrastructure Route53 Hosted Zones."
type = string
Expand Down
19 changes: 19 additions & 0 deletions vpc-infrastructure-s3-transfer-ssm-documents.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
resource "aws_ssm_document" "infrastructure_vpc_transfer_s3_download" {
count = local.enable_infrastructure_vpc_transfer_s3_bucket ? 1 : 0

name = "${local.resource_prefix_hash}-infrastructure-vpc-transfer-s3-download"
document_type = "Session"
content = templatefile("${path.root}/ssm-documents/s3-download.json.tpl", {
command = local.infrastructure_vpc_transfer_ssm_download_command
})
}

resource "aws_ssm_document" "infrastructure_vpc_transfer_s3_upload" {
count = local.enable_infrastructure_vpc_transfer_s3_bucket ? 1 : 0

name = "${local.resource_prefix_hash}-infrastructure-vpc-transfer-s3-upload"
document_type = "Session"
content = templatefile("${path.root}/ssm-documents/s3-upload.json.tpl", {
command = local.infrastructure_vpc_transfer_ssm_upload_command
})
}
Loading

0 comments on commit fdc6c28

Please sign in to comment.