From 2fdb1da1b72ad3f8dcb89c59e50a38d35339ec8a Mon Sep 17 00:00:00 2001 From: Chris Wright Date: Fri, 30 Aug 2024 14:27:22 +0100 Subject: [PATCH] Fix VPC flow logs glue table * Ensures the column names are kept in order * Corrects the partition keys and projections --- locals.tf | 74 +++++++++++---------- vpc-infrastructure-flow-logs-glue-tables.tf | 21 +++--- 2 files changed, 52 insertions(+), 43 deletions(-) diff --git a/locals.tf b/locals.tf index 6871134..166520c 100644 --- a/locals.tf +++ b/locals.tf @@ -80,41 +80,45 @@ locals { infrastructure_vpc_flow_logs_s3_key_prefix = trim(var.infrastructure_vpc_flow_logs_s3_key_prefix, "/") infrastructure_vpc_flow_logs_retention = var.infrastructure_vpc_flow_logs_retention infrastructure_vpc_flow_logs_traffic_type = var.infrastructure_vpc_flow_logs_traffic_type - infrastructure_vpc_flow_logs_glue_table_columns = { - version = "int", - account_id = "string", - interface_id = "string", - srcaddr = "string", - dstaddr = "string", - srcport = "int", - dstport = "int", - protocol = "bigint", - packets = "bigint", - bytes = "bigint", - start = "bigint", - "`end`" = "bigint", - action = "string", - log_status = "string", - vpc_id = "string", - subnet_id = "string", - instance_id = "string", - tcp_flags = "int", - type = "string", - pkt_srcaddr = "string", - pkt_dstaddr = "string", - az_id = "string", - sublocation_type = "string", - sublocation_id = "string", - pkt_src_aws_service = "string", - pkt_dst_aws_service = "string", - flow_direction = "string", - traffic_path = "int", - } - infrastructure_vpc_flow_logs_glue_table_partition_keys = { - region = "string", - date = "string", - hour = "string" - } + infrastructure_vpc_flow_logs_glue_table_columns = [ + { name = "version", type = "int" }, + { name = "account_id", type = "string" }, + { name = "interface_id", type = "string" }, + { name = "srcaddr", type = "string" }, + { name = "dstaddr", type = "string" }, + { name = "srcport", type = "int" }, + { name = "dstport", type = "int" }, + { name = "protocol", type = "bigint" }, + { name = "packets", type = "bigint" }, + { name = "bytes", type = "bigint" }, + { name = "start", type = "bigint" }, + { name = "`end`", type = "bigint" }, + { name = "action", type = "string" }, + { name = "log_status", type = "string" }, + { name = "vpc_id", type = "string" }, + { name = "subnet_id", type = "string" }, + { name = "instance_id", type = "string" }, + { name = "tcp_flags", type = "int" }, + { name = "type", type = "string" }, + { name = "pkt_srcaddr", type = "string" }, + { name = "pkt_dstaddr", type = "string" }, + { name = "az_id", type = "string" }, + { name = "sublocation_type", type = "string" }, + { name = "sublocation_id", type = "string" }, + { name = "pkt_src_aws_service", type = "string" }, + { name = "pkt_dst_aws_service", type = "string" }, + { name = "flow_direction", type = "string" }, + { name = "traffic_path", type = "int" }, + ] + infrastructure_vpc_flow_logs_glue_table_partition_keys = [ + { name = "aws-account-id", type = "string" }, + { name = "aws-service", type = "string" }, + { name = "aws-region", type = "string" }, + { name = "year", type = "string" }, + { name = "month", type = "string" }, + { name = "day", type = "string" }, + { name = "hour", type = "string" }, + ] enable_infrastructure_vpc_transfer_s3_bucket = var.enable_infrastructure_vpc_transfer_s3_bucket infrastructure_vpc_transfer_s3_bucket_access_vpc_ids = concat( local.infrastructure_vpc ? [aws_vpc.infrastructure[0].id] : [], diff --git a/vpc-infrastructure-flow-logs-glue-tables.tf b/vpc-infrastructure-flow-logs-glue-tables.tf index f40709c..78aaf77 100644 --- a/vpc-infrastructure-flow-logs-glue-tables.tf +++ b/vpc-infrastructure-flow-logs-glue-tables.tf @@ -14,8 +14,8 @@ resource "aws_glue_catalog_table" "infrastructure_vpc_flow_logs" { dynamic "partition_keys" { for_each = local.infrastructure_vpc_flow_logs_glue_table_partition_keys content { - name = partition_keys.key - type = partition_keys.value + name = partition_keys.value["name"] + type = partition_keys.value["type"] } } @@ -26,13 +26,18 @@ resource "aws_glue_catalog_table" "infrastructure_vpc_flow_logs" { "projection.enabled" = "true" "projection.region.type" = "enum" "projection.region.values" = local.aws_region - "projection.day.type" = "date" - "projection.day.range" = "2023/01/01,NOW" - "projection.day.format" = "yyyy/MM/dd" + "projection.year.type" = "integer" + "projection.year.digits" = "4" + "projection.month.type" = "integer" + "projection.month.range" = "00,12" + "projection.month.digits" = "2" + "projection.day.type" = "integer" + "projection.day.range" = "00,31" + "projection.day.digits" = "2" "projection.hour.type" = "integer" "projection.hour.range" = "00,23" "projection.hour.digits" = "2" - "storage.location.template" = "s3://${aws_s3_bucket.infrastructure_logs[0].id}/${local.infrastructure_vpc_flow_logs_s3_key_prefix}/AWSLogs/${local.aws_account_id}/vpcflowlogs/$${region}/$${day}/$${hour}" + "storage.location.template" = "s3://${aws_s3_bucket.infrastructure_logs[0].id}/${local.infrastructure_vpc_flow_logs_s3_key_prefix}/AWSLogs/$${aws-account-id}/vpcflowlogs/$${region}/$${year}/$${month}/$${day}/$${hour}" } storage_descriptor { @@ -50,8 +55,8 @@ resource "aws_glue_catalog_table" "infrastructure_vpc_flow_logs" { dynamic "columns" { for_each = local.infrastructure_vpc_flow_logs_glue_table_columns content { - name = columns.key - type = columns.value + name = columns.value["name"] + type = columns.value["type"] } } }