From 5388f6610d875406e99f2c595e2fcebd61a4e0f6 Mon Sep 17 00:00:00 2001 From: Chris Wright Date: Fri, 17 Nov 2023 15:01:19 +0000 Subject: [PATCH 1/3] Create an S3 bucket to store logs * This bucket will be used to store all logs within the infrastructure. Storing them centrally in a single bucket reduces the complexity of permissions across multiple S3 buckets, and any future maintenance that is required on them. If we feel that they are critical logs, we can decide in future wether to keep a backup of this bucket. * Each logging service will be isolated using a unqique prefix. The first service that will use this will be the VPC flow logs, so this is needed before we can create them. --- README.md | 7 ++ locals.tf | 15 +-- .../enforce-tls.json.tpl | 17 ++++ .../log-delivery-access.json.tpl | 37 +++++++ policies/s3-bucket-policy.json.tpl | 4 + s3-infrastructure-logs.tf | 97 +++++++++++++++++++ variables.tf | 5 + 7 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 policies/s3-bucket-policy-statements/enforce-tls.json.tpl create mode 100644 policies/s3-bucket-policy-statements/log-delivery-access.json.tpl create mode 100644 policies/s3-bucket-policy.json.tpl create mode 100644 s3-infrastructure-logs.tf diff --git a/README.md b/README.md index 35efd2c..ffd7400 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,12 @@ This project creates and manages resources within an AWS account for infrastruct |------|------| | [aws_kms_alias.infrastructure](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource | | [aws_kms_key.infrastructure](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_s3_bucket.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket_lifecycle_configuration.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_lifecycle_configuration) | resource | +| [aws_s3_bucket_policy.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource | +| [aws_s3_bucket_public_access_block.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | +| [aws_s3_bucket_server_side_encryption_configuration.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | +| [aws_s3_bucket_versioning.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | ## Inputs @@ -34,6 +40,7 @@ This project creates and manages resources within an AWS account for infrastruct | [aws\_region](#input\_aws\_region) | AWS region in which to launch resources | `string` | n/a | yes | | [environment](#input\_environment) | The environment name to be used as part of the resource prefix | `string` | n/a | yes | | [infrastructure\_kms\_encryption](#input\_infrastructure\_kms\_encryption) | Enable infrastructure KMS encryption. This will create a single KMS key to be used across all resources that support KMS encryption. | `bool` | n/a | yes | +| [infrastructure\_logging\_bucket\_retention](#input\_infrastructure\_logging\_bucket\_retention) | Retention in days for the infrasrtucture S3 logs. This is for the default S3 logs bucket, where all AWS service logs will be delivered | `number` | n/a | yes | | [infrastructure\_name](#input\_infrastructure\_name) | The infrastructure name to be used as part of the resource prefix | `string` | n/a | yes | | [project\_name](#input\_project\_name) | Project name to be used as a prefix for all resources | `string` | n/a | yes | diff --git a/locals.tf b/locals.tf index 52ab827..a9d8793 100644 --- a/locals.tf +++ b/locals.tf @@ -1,13 +1,16 @@ locals { - project_name = var.project_name - infrastructure_name = var.infrastructure_name - environment = var.environment - aws_region = var.aws_region - aws_account_id = data.aws_caller_identity.current.account_id - resource_prefix = "${var.project_name}-${var.infrastructure_name}-${var.environment}" + project_name = var.project_name + infrastructure_name = var.infrastructure_name + environment = var.environment + aws_region = var.aws_region + aws_account_id = data.aws_caller_identity.current.account_id + resource_prefix = "${var.project_name}-${var.infrastructure_name}-${var.environment}" + resource_prefix_hash = format("%.8s", sha512(local.resource_prefix)) infrastructure_kms_encryption = var.infrastructure_kms_encryption + infrastructure_logging_bucket_retention = var.infrastructure_logging_bucket_retention + default_tags = { Project = local.project_name, Infrastructure = local.infrastructure_name, diff --git a/policies/s3-bucket-policy-statements/enforce-tls.json.tpl b/policies/s3-bucket-policy-statements/enforce-tls.json.tpl new file mode 100644 index 0000000..406eea2 --- /dev/null +++ b/policies/s3-bucket-policy-statements/enforce-tls.json.tpl @@ -0,0 +1,17 @@ +{ + "Principal": "*", + "Action": "s3:*", + "Effect": "Deny", + "Resource": [ + "${bucket_arn}", + "${bucket_arn}/*" + ], + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + }, + "NumericLessThan": { + "s3:TlsVersion": "1.2" + } + } +} diff --git a/policies/s3-bucket-policy-statements/log-delivery-access.json.tpl b/policies/s3-bucket-policy-statements/log-delivery-access.json.tpl new file mode 100644 index 0000000..a706829 --- /dev/null +++ b/policies/s3-bucket-policy-statements/log-delivery-access.json.tpl @@ -0,0 +1,37 @@ +{ + "Principal": { + "Service": "logging.s3.amazonaws.com" + }, + "Action": [ + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "${log_bucket_arn}/*", + "Condition": { + "ArnLike": { + "aws:SourceArn": ${source_arns} + }, + "StringEquals": { + "aws:SourceAccount": "${account_id}" + } + } +}, +{ + "Effect": "Allow", + "Principal": { + "Service": "delivery.logs.amazonaws.com" + }, + "Action": [ + "s3:GetBucketAcl", + "s3:ListBucket" + ], + "Resource": "${log_bucket_arn}", + "Condition": { + "StringEquals": { + "aws:SourceAccount": "${account_id}" + }, + "ArnLike": { + "aws:SourceArn": ${source_arns} + } + } +} diff --git a/policies/s3-bucket-policy.json.tpl b/policies/s3-bucket-policy.json.tpl new file mode 100644 index 0000000..11e4ab9 --- /dev/null +++ b/policies/s3-bucket-policy.json.tpl @@ -0,0 +1,4 @@ +{ + "Version": "2012-10-17", + "Statement": ${statement} +} diff --git a/s3-infrastructure-logs.tf b/s3-infrastructure-logs.tf new file mode 100644 index 0000000..f261531 --- /dev/null +++ b/s3-infrastructure-logs.tf @@ -0,0 +1,97 @@ +# https://github.com/aquasecurity/tfsec/issues/2081 +# tfsec:ignore:aws-s3-enable-bucket-logging +resource "aws_s3_bucket" "infrastructure_logs" { + count = local.enable_infrastructure_logs_bucket ? 1 : 0 + + bucket = "${local.resource_prefix_hash}-logs" +} + +resource "aws_s3_bucket_policy" "infrastructure_logs" { + count = local.enable_infrastructure_logs_bucket ? 1 : 0 + + bucket = aws_s3_bucket.infrastructure_logs[0].id + policy = templatefile( + "${path.module}/policies/s3-bucket-policy.json.tpl", + { + statement = < Date: Fri, 17 Nov 2023 15:07:15 +0000 Subject: [PATCH 2/3] Create infrastructure VPC * This will be the default VPC for where all resources will be deployed into. All options for the VPC are configurable. --- README.md | 8 ++++++++ variables.tf | 35 +++++++++++++++++++++++++++++++++++ vpc-infrastructure.tf | 10 ++++++++++ 3 files changed, 53 insertions(+) create mode 100644 vpc-infrastructure.tf diff --git a/README.md b/README.md index ffd7400..9216ae7 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ This project creates and manages resources within an AWS account for infrastruct | [aws_s3_bucket_public_access_block.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | | [aws_s3_bucket_server_side_encryption_configuration.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | | [aws_s3_bucket_versioning.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource | +| [aws_vpc.infrastructure](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | ## Inputs @@ -42,6 +43,13 @@ This project creates and manages resources within an AWS account for infrastruct | [infrastructure\_kms\_encryption](#input\_infrastructure\_kms\_encryption) | Enable infrastructure KMS encryption. This will create a single KMS key to be used across all resources that support KMS encryption. | `bool` | n/a | yes | | [infrastructure\_logging\_bucket\_retention](#input\_infrastructure\_logging\_bucket\_retention) | Retention in days for the infrasrtucture S3 logs. This is for the default S3 logs bucket, where all AWS service logs will be delivered | `number` | n/a | yes | | [infrastructure\_name](#input\_infrastructure\_name) | The infrastructure name to be used as part of the resource prefix | `string` | n/a | yes | +| [infrastructure\_vpc](#input\_infrastructure\_vpc) | Enable infrastructure VPC | `bool` | n/a | yes | +| [infrastructure\_vpc\_assign\_generated\_ipv6\_cidr\_block](#input\_infrastructure\_vpc\_assign\_generated\_ipv6\_cidr\_block) | Assign generated IPv6 CIDR block on infrastructure VPC | `bool` | n/a | yes | +| [infrastructure\_vpc\_cidr\_block](#input\_infrastructure\_vpc\_cidr\_block) | Infrastructure VPC CIDR block | `string` | n/a | yes | +| [infrastructure\_vpc\_enable\_dns\_hostnames](#input\_infrastructure\_vpc\_enable\_dns\_hostnames) | Enable DNS hostnames on infrastructure VPC | `bool` | n/a | yes | +| [infrastructure\_vpc\_enable\_dns\_support](#input\_infrastructure\_vpc\_enable\_dns\_support) | Enable DNS support on infrastructure VPC | `bool` | n/a | yes | +| [infrastructure\_vpc\_enable\_network\_address\_usage\_metrics](#input\_infrastructure\_vpc\_enable\_network\_address\_usage\_metrics) | Enable network address usage metrics on infrastructure VPC | `bool` | n/a | yes | +| [infrastructure\_vpc\_instance\_tenancy](#input\_infrastructure\_vpc\_instance\_tenancy) | Infrastructure VPC instance tenancy | `string` | n/a | yes | | [project\_name](#input\_project\_name) | Project name to be used as a prefix for all resources | `string` | n/a | yes | ## Outputs diff --git a/variables.tf b/variables.tf index fff88ac..ca6abff 100644 --- a/variables.tf +++ b/variables.tf @@ -27,3 +27,38 @@ variable "infrastructure_logging_bucket_retention" { description = "Retention in days for the infrasrtucture S3 logs. This is for the default S3 logs bucket, where all AWS service logs will be delivered" type = number } + +variable "infrastructure_vpc" { + description = "Enable infrastructure VPC" + type = bool +} + +variable "infrastructure_vpc_cidr_block" { + description = "Infrastructure VPC CIDR block" + type = string +} + +variable "infrastructure_vpc_enable_dns_support" { + description = "Enable DNS support on infrastructure VPC" + type = bool +} + +variable "infrastructure_vpc_enable_dns_hostnames" { + description = "Enable DNS hostnames on infrastructure VPC" + type = bool +} + +variable "infrastructure_vpc_instance_tenancy" { + description = "Infrastructure VPC instance tenancy" + type = string +} + +variable "infrastructure_vpc_enable_network_address_usage_metrics" { + description = "Enable network address usage metrics on infrastructure VPC" + type = bool +} + +variable "infrastructure_vpc_assign_generated_ipv6_cidr_block" { + description = "Assign generated IPv6 CIDR block on infrastructure VPC" + type = bool +} diff --git a/vpc-infrastructure.tf b/vpc-infrastructure.tf new file mode 100644 index 0000000..ef819e7 --- /dev/null +++ b/vpc-infrastructure.tf @@ -0,0 +1,10 @@ +resource "aws_vpc" "infrastructure" { + count = local.infrastructure_vpc ? 1 : 0 + + cidr_block = local.infrastructure_vpc_cidr_block + enable_dns_support = local.infrastructure_vpc_enable_dns_support + enable_dns_hostnames = local.infrastructure_vpc_enable_dns_hostnames + instance_tenancy = local.infrastructure_vpc_instance_tenancy + enable_network_address_usage_metrics = local.infrastructure_vpc_enable_network_address_usage_metrics + assign_generated_ipv6_cidr_block = local.infrastructure_vpc_assign_generated_ipv6_cidr_block +} From 9c43f856a2929ca69eec968b89e9908f23d35ba9 Mon Sep 17 00:00:00 2001 From: Chris Wright Date: Fri, 17 Nov 2023 15:08:51 +0000 Subject: [PATCH 3/3] Infrastructure VPC Flow Logs * This allows setting up Flow Logs on the Infrastructure VPC, with either/both S3 and CloudWatch destinations. If the S3 destination is deployed, it will also deploy the compatible Glue table/database for AThena, so that we can easily query the logs. An Athena workgroup has also been created, where we can create useful stored queries (The athena output goes to the logging bucket). --- README.md | 13 +++++ kms-infrastructure.tf | 11 ++++ locals.tf | 55 ++++++++++++++++++ .../service-principle-standard.json.tpl | 13 +++++ policies/cloudwatch-logs-rw.json.tpl | 16 +++++ .../cloudwatch-logs-allow.json.tpl | 19 ++++++ .../log-delivery-allow.json.tpl | 22 +++++++ variables.tf | 25 ++++++++ vpc-infrastructure-flow-logs-athena.tf | 19 ++++++ vpc-infrastructure-flow-logs-cloudwatch.tf | 34 +++++++++++ vpc-infrastructure-flow-logs-glue-tables.tf | 58 +++++++++++++++++++ vpc-infrastructure-flow-logs-s3.tf | 13 +++++ 12 files changed, 298 insertions(+) create mode 100644 policies/assume-roles/service-principle-standard.json.tpl create mode 100644 policies/cloudwatch-logs-rw.json.tpl create mode 100644 policies/kms-key-policy-statements/cloudwatch-logs-allow.json.tpl create mode 100644 policies/kms-key-policy-statements/log-delivery-allow.json.tpl create mode 100644 vpc-infrastructure-flow-logs-athena.tf create mode 100644 vpc-infrastructure-flow-logs-cloudwatch.tf create mode 100644 vpc-infrastructure-flow-logs-glue-tables.tf create mode 100644 vpc-infrastructure-flow-logs-s3.tf diff --git a/README.md b/README.md index 9216ae7..350fdcd 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,14 @@ This project creates and manages resources within an AWS account for infrastruct | Name | Type | |------|------| +| [aws_athena_workgroup.infrastructure_vpc_flow_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_workgroup) | resource | +| [aws_cloudwatch_log_group.infrastructure_vpc_flow_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_flow_log.infrastructure_vpc_flow_logs_cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | +| [aws_flow_log.infrastructure_vpc_flow_logs_s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | +| [aws_glue_catalog_database.infrastructure_vpc_flow_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_catalog_database) | resource | +| [aws_glue_catalog_table.infrastructure_vpc_flow_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_catalog_table) | resource | +| [aws_iam_role.infrastructure_vpc_flow_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.infrastructure_vpc_flow_logs_allow_cloudwatch_rw](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_kms_alias.infrastructure](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource | | [aws_kms_key.infrastructure](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_s3_bucket.infrastructure_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | @@ -49,6 +57,11 @@ This project creates and manages resources within an AWS account for infrastruct | [infrastructure\_vpc\_enable\_dns\_hostnames](#input\_infrastructure\_vpc\_enable\_dns\_hostnames) | Enable DNS hostnames on infrastructure VPC | `bool` | n/a | yes | | [infrastructure\_vpc\_enable\_dns\_support](#input\_infrastructure\_vpc\_enable\_dns\_support) | Enable DNS support on infrastructure VPC | `bool` | n/a | yes | | [infrastructure\_vpc\_enable\_network\_address\_usage\_metrics](#input\_infrastructure\_vpc\_enable\_network\_address\_usage\_metrics) | Enable network address usage metrics on infrastructure VPC | `bool` | n/a | yes | +| [infrastructure\_vpc\_flow\_logs\_cloudwatch\_logs](#input\_infrastructure\_vpc\_flow\_logs\_cloudwatch\_logs) | Enable VPC logs on infrastructure VPC to CloudWatch Logs | `bool` | n/a | yes | +| [infrastructure\_vpc\_flow\_logs\_retention](#input\_infrastructure\_vpc\_flow\_logs\_retention) | VPC flow logs retention in days | `number` | n/a | yes | +| [infrastructure\_vpc\_flow\_logs\_s3\_key\_prefix](#input\_infrastructure\_vpc\_flow\_logs\_s3\_key\_prefix) | Flow Logs by default will go into the infrastructure S3 logs bucket. This is the key prefix used to isolate them from other logs | `string` | n/a | yes | +| [infrastructure\_vpc\_flow\_logs\_s3\_with\_athena](#input\_infrastructure\_vpc\_flow\_logs\_s3\_with\_athena) | Enable VPC flow logs in infrastructure VPC to the S3 logs bucket. A compatible Glue table/database and Athena workgroup will also be created to allow querying the logs. | `bool` | n/a | yes | +| [infrastructure\_vpc\_flow\_logs\_traffic\_type](#input\_infrastructure\_vpc\_flow\_logs\_traffic\_type) | Infrastructure VPC flow logs traffic type | `string` | n/a | yes | | [infrastructure\_vpc\_instance\_tenancy](#input\_infrastructure\_vpc\_instance\_tenancy) | Infrastructure VPC instance tenancy | `string` | n/a | yes | | [project\_name](#input\_project\_name) | Project name to be used as a prefix for all resources | `string` | n/a | yes | diff --git a/kms-infrastructure.tf b/kms-infrastructure.tf index c57f9d8..5461369 100644 --- a/kms-infrastructure.tf +++ b/kms-infrastructure.tf @@ -14,6 +14,17 @@ resource "aws_kms_key" "infrastructure" { { aws_account_id = local.aws_account_id } + )}${local.infrastructure_vpc_flow_logs_cloudwatch_logs && local.infrastructure_kms_encryption ? "," : ""} + ${templatefile("${path.root}/policies/kms-key-policy-statements/cloudwatch-logs-allow.json.tpl", + { + log_group_arn = local.infrastructure_vpc_flow_logs_cloudwatch_logs && local.infrastructure_kms_encryption ? "arn:aws:logs:${local.aws_region}:${local.aws_account_id}:log-group:${local.resource_prefix}-infrastructure-vpc-flow-logs" : "" + } + )}${local.infrastructure_vpc_flow_logs_s3_with_athena && local.infrastructure_kms_encryption ? "," : ""} + ${templatefile("${path.root}/policies/kms-key-policy-statements/log-delivery-allow.json.tpl", + { + account_id = local.infrastructure_vpc_flow_logs_s3_with_athena && local.infrastructure_kms_encryption ? local.aws_account_id : "" + region = local.aws_region + } )} ] EOT diff --git a/locals.tf b/locals.tf index a9d8793..41d4f13 100644 --- a/locals.tf +++ b/locals.tf @@ -11,6 +11,61 @@ locals { infrastructure_logging_bucket_retention = var.infrastructure_logging_bucket_retention + enable_infrastructure_logs_bucket = ( + local.infrastructure_vpc_flow_logs_s3_with_athena + ) + logs_bucket_source_arns = concat( + local.infrastructure_vpc_flow_logs_s3_with_athena ? ["arn:aws:logs:${local.aws_region}:${local.aws_account_id}:*"] : [], + ) + + infrastructure_vpc = var.infrastructure_vpc + infrastructure_vpc_cidr_block = var.infrastructure_vpc_cidr_block + infrastructure_vpc_enable_dns_support = var.infrastructure_vpc_enable_dns_support + infrastructure_vpc_enable_dns_hostnames = var.infrastructure_vpc_enable_dns_hostnames + infrastructure_vpc_instance_tenancy = var.infrastructure_vpc_instance_tenancy + infrastructure_vpc_enable_network_address_usage_metrics = var.infrastructure_vpc_enable_network_address_usage_metrics + infrastructure_vpc_assign_generated_ipv6_cidr_block = var.infrastructure_vpc_assign_generated_ipv6_cidr_block + infrastructure_vpc_flow_logs_cloudwatch_logs = var.infrastructure_vpc_flow_logs_cloudwatch_logs && local.infrastructure_vpc + infrastructure_vpc_flow_logs_s3_with_athena = var.infrastructure_vpc_flow_logs_s3_with_athena && local.infrastructure_vpc + infrastructure_vpc_flow_logs_s3_key_prefix = trim(var.infrastructure_vpc_flow_logs_s3_key_prefix, "/") + infrastructure_vpc_flow_logs_retention = var.infrastructure_vpc_flow_logs_retention + infrastructure_vpc_flow_logs_traffic_type = var.infrastructure_vpc_flow_logs_traffic_type + infrastructure_vpc_flow_logs_glue_table_columns = { + version = "int", + account_id = "string", + interface_id = "string", + srcaddr = "string", + dstaddr = "string", + srcport = "int", + dstport = "int", + protocol = "bigint", + packets = "bigint", + bytes = "bigint", + start = "bigint", + "`end`" = "bigint", + action = "string", + log_status = "string", + vpc_id = "string", + subnet_id = "string", + instance_id = "string", + tcp_flags = "int", + type = "string", + pkt_srcaddr = "string", + pkt_dstaddr = "string", + az_id = "string", + sublocation_type = "string", + sublocation_id = "string", + pkt_src_aws_service = "string", + pkt_dst_aws_service = "string", + flow_direction = "string", + traffic_path = "int", + } + infrastructure_vpc_flow_logs_glue_table_partition_keys = { + region = "string", + date = "string", + hour = "string" + } + default_tags = { Project = local.project_name, Infrastructure = local.infrastructure_name, diff --git a/policies/assume-roles/service-principle-standard.json.tpl b/policies/assume-roles/service-principle-standard.json.tpl new file mode 100644 index 0000000..0801c45 --- /dev/null +++ b/policies/assume-roles/service-principle-standard.json.tpl @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": ${services} + }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/policies/cloudwatch-logs-rw.json.tpl b/policies/cloudwatch-logs-rw.json.tpl new file mode 100644 index 0000000..143a709 --- /dev/null +++ b/policies/cloudwatch-logs-rw.json.tpl @@ -0,0 +1,16 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams" + ], + "Effect": "Allow", + "Resource": "*" + } + ] +} diff --git a/policies/kms-key-policy-statements/cloudwatch-logs-allow.json.tpl b/policies/kms-key-policy-statements/cloudwatch-logs-allow.json.tpl new file mode 100644 index 0000000..a88ef9b --- /dev/null +++ b/policies/kms-key-policy-statements/cloudwatch-logs-allow.json.tpl @@ -0,0 +1,19 @@ +%{if log_group_arn != ""}{ + "Effect": "Allow", + "Principal": { + "Service": "logs.amazonaws.com" + }, + "Action": [ + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ], + "Resource": "*", + "Condition": { + "ArnEquals": { + "kms:EncryptionContext:aws:logs:arn": "${log_group_arn}" + } + } +}%{endif} diff --git a/policies/kms-key-policy-statements/log-delivery-allow.json.tpl b/policies/kms-key-policy-statements/log-delivery-allow.json.tpl new file mode 100644 index 0000000..38a6d49 --- /dev/null +++ b/policies/kms-key-policy-statements/log-delivery-allow.json.tpl @@ -0,0 +1,22 @@ +%{if account_id != ""}{ + "Effect": "Allow", + "Principal": { + "Service": [ "delivery.logs.amazonaws.com" ] + }, + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "aws:SourceAccount": ["${account_id}"] + }, + "ArnLike": { + "aws:SourceArn": ["arn:aws:logs:${region}:${account_id}:*"] + } + } +}%{endif} diff --git a/variables.tf b/variables.tf index ca6abff..cd214fc 100644 --- a/variables.tf +++ b/variables.tf @@ -62,3 +62,28 @@ variable "infrastructure_vpc_assign_generated_ipv6_cidr_block" { description = "Assign generated IPv6 CIDR block on infrastructure VPC" type = bool } + +variable "infrastructure_vpc_flow_logs_cloudwatch_logs" { + description = "Enable VPC logs on infrastructure VPC to CloudWatch Logs" + type = bool +} + +variable "infrastructure_vpc_flow_logs_s3_with_athena" { + description = "Enable VPC flow logs in infrastructure VPC to the S3 logs bucket. A compatible Glue table/database and Athena workgroup will also be created to allow querying the logs." + type = bool +} + +variable "infrastructure_vpc_flow_logs_retention" { + description = "VPC flow logs retention in days" + type = number +} + +variable "infrastructure_vpc_flow_logs_traffic_type" { + description = "Infrastructure VPC flow logs traffic type" + type = string +} + +variable "infrastructure_vpc_flow_logs_s3_key_prefix" { + description = "Flow Logs by default will go into the infrastructure S3 logs bucket. This is the key prefix used to isolate them from other logs" + type = string +} diff --git a/vpc-infrastructure-flow-logs-athena.tf b/vpc-infrastructure-flow-logs-athena.tf new file mode 100644 index 0000000..c8bf89e --- /dev/null +++ b/vpc-infrastructure-flow-logs-athena.tf @@ -0,0 +1,19 @@ +resource "aws_athena_workgroup" "infrastructure_vpc_flow_logs" { + count = local.infrastructure_vpc_flow_logs_s3_with_athena ? 1 : 0 + + name = "${local.resource_prefix}-infrastructure-vpc-flow-logs" + + configuration { + enforce_workgroup_configuration = true + publish_cloudwatch_metrics_enabled = true + + result_configuration { + output_location = "s3://${aws_s3_bucket.infrastructure_logs[0].bucket}/${local.infrastructure_vpc_flow_logs_s3_key_prefix}-athena-output" + + encryption_configuration { + encryption_option = local.infrastructure_kms_encryption ? "SSE_KMS" : "SSE_S3" + kms_key_arn = local.infrastructure_kms_encryption ? aws_kms_key.infrastructure[0].arn : null + } + } + } +} diff --git a/vpc-infrastructure-flow-logs-cloudwatch.tf b/vpc-infrastructure-flow-logs-cloudwatch.tf new file mode 100644 index 0000000..db2764d --- /dev/null +++ b/vpc-infrastructure-flow-logs-cloudwatch.tf @@ -0,0 +1,34 @@ +resource "aws_cloudwatch_log_group" "infrastructure_vpc_flow_logs" { + count = local.infrastructure_vpc_flow_logs_cloudwatch_logs ? 1 : 0 + + name = "${local.resource_prefix}-infrastructure-vpc-flow-logs" + retention_in_days = local.infrastructure_vpc_flow_logs_retention + kms_key_id = local.infrastructure_kms_encryption ? aws_kms_key.infrastructure[0].arn : null + skip_destroy = true +} + +resource "aws_iam_role" "infrastructure_vpc_flow_logs" { + count = local.infrastructure_vpc_flow_logs_cloudwatch_logs ? 1 : 0 + + name = "${local.resource_prefix}-infrastructure-vpc-flow-logs" + assume_role_policy = templatefile("${path.root}/policies/assume-roles/service-principle-standard.json.tpl", { + services = jsonencode(["vpc-flow-logs.amazonaws.com"]) + }) +} + +resource "aws_iam_role_policy" "infrastructure_vpc_flow_logs_allow_cloudwatch_rw" { + count = local.infrastructure_vpc_flow_logs_cloudwatch_logs ? 1 : 0 + + name = "${local.resource_prefix}-ecs-vpc-flow-logs-cloudwatch-logs-rw" + role = aws_iam_role.infrastructure_vpc_flow_logs[0].id + policy = templatefile("${path.root}/policies/cloudwatch-logs-rw.json.tpl", {}) +} + +resource "aws_flow_log" "infrastructure_vpc_flow_logs_cloudwatch" { + count = local.infrastructure_vpc_flow_logs_cloudwatch_logs ? 1 : 0 + + iam_role_arn = aws_iam_role.infrastructure_vpc_flow_logs[0].arn + log_destination = aws_cloudwatch_log_group.infrastructure_vpc_flow_logs[0].arn + traffic_type = local.infrastructure_vpc_flow_logs_traffic_type + vpc_id = aws_vpc.infrastructure[0].id +} diff --git a/vpc-infrastructure-flow-logs-glue-tables.tf b/vpc-infrastructure-flow-logs-glue-tables.tf new file mode 100644 index 0000000..f40709c --- /dev/null +++ b/vpc-infrastructure-flow-logs-glue-tables.tf @@ -0,0 +1,58 @@ +resource "aws_glue_catalog_database" "infrastructure_vpc_flow_logs" { + count = local.infrastructure_vpc_flow_logs_s3_with_athena ? 1 : 0 + + name = "${replace(local.resource_prefix, "-", "_")}_infrastructure_vpc_logs" + description = "Database for ${local.resource_prefix} VPC flow log tables to be queried with Athena" +} + +resource "aws_glue_catalog_table" "infrastructure_vpc_flow_logs" { + count = local.infrastructure_vpc_flow_logs_s3_with_athena ? 1 : 0 + + name = "${replace(local.resource_prefix, "-", "_")}_infrastructure_vpc_logs" + database_name = aws_glue_catalog_database.infrastructure_vpc_flow_logs[0].name + + dynamic "partition_keys" { + for_each = local.infrastructure_vpc_flow_logs_glue_table_partition_keys + content { + name = partition_keys.key + type = partition_keys.value + } + } + + parameters = { + comment = "VPC Flow logs table for ${local.resource_prefix} infrastructure VPC" + EXTERNAL = "TRUE" + "skip.header.line.count" = "1" + "projection.enabled" = "true" + "projection.region.type" = "enum" + "projection.region.values" = local.aws_region + "projection.day.type" = "date" + "projection.day.range" = "2023/01/01,NOW" + "projection.day.format" = "yyyy/MM/dd" + "projection.hour.type" = "integer" + "projection.hour.range" = "00,23" + "projection.hour.digits" = "2" + "storage.location.template" = "s3://${aws_s3_bucket.infrastructure_logs[0].id}/${local.infrastructure_vpc_flow_logs_s3_key_prefix}/AWSLogs/${local.aws_account_id}/vpcflowlogs/$${region}/$${day}/$${hour}" + } + + storage_descriptor { + input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" + location = "s3://${aws_s3_bucket.infrastructure_logs[0].id}/${local.infrastructure_vpc_flow_logs_s3_key_prefix}/AWSLogs/${local.aws_account_id}/vpcflowlogs" + + ser_de_info { + parameters = { + "serialization.format" = "1" + } + serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" + } + + dynamic "columns" { + for_each = local.infrastructure_vpc_flow_logs_glue_table_columns + content { + name = columns.key + type = columns.value + } + } + } +} diff --git a/vpc-infrastructure-flow-logs-s3.tf b/vpc-infrastructure-flow-logs-s3.tf new file mode 100644 index 0000000..32a10f3 --- /dev/null +++ b/vpc-infrastructure-flow-logs-s3.tf @@ -0,0 +1,13 @@ +resource "aws_flow_log" "infrastructure_vpc_flow_logs_s3" { + count = local.infrastructure_vpc_flow_logs_s3_with_athena ? 1 : 0 + + log_destination_type = "s3" + log_destination = "${aws_s3_bucket.infrastructure_logs[0].arn}/${local.infrastructure_vpc_flow_logs_s3_key_prefix}" + traffic_type = local.infrastructure_vpc_flow_logs_traffic_type + vpc_id = aws_vpc.infrastructure[0].id + + destination_options { + file_format = "parquet" + per_hour_partition = true + } +}