Fix VPC flow logs glue table

* Ensures the column names are kept in order
* Corrects the partition keys and projections

locals.tf

@@ -80,41 +80,45 @@ locals {
   infrastructure_vpc_flow_logs_s3_key_prefix                  = trim(var.infrastructure_vpc_flow_logs_s3_key_prefix, "/")
   infrastructure_vpc_flow_logs_retention                      = var.infrastructure_vpc_flow_logs_retention
   infrastructure_vpc_flow_logs_traffic_type                   = var.infrastructure_vpc_flow_logs_traffic_type
-  infrastructure_vpc_flow_logs_glue_table_columns = {
-    version             = "int",
-    account_id          = "string",
-    interface_id        = "string",
-    srcaddr             = "string",
-    dstaddr             = "string",
-    srcport             = "int",
-    dstport             = "int",
-    protocol            = "bigint",
-    packets             = "bigint",
-    bytes               = "bigint",
-    start               = "bigint",
-    "`end`"             = "bigint",
-    action              = "string",
-    log_status          = "string",
-    vpc_id              = "string",
-    subnet_id           = "string",
-    instance_id         = "string",
-    tcp_flags           = "int",
-    type                = "string",
-    pkt_srcaddr         = "string",
-    pkt_dstaddr         = "string",
-    az_id               = "string",
-    sublocation_type    = "string",
-    sublocation_id      = "string",
-    pkt_src_aws_service = "string",
-    pkt_dst_aws_service = "string",
-    flow_direction      = "string",
-    traffic_path        = "int",
-  }
-  infrastructure_vpc_flow_logs_glue_table_partition_keys = {
-    region = "string",
-    date   = "string",
-    hour   = "string"
-  }
+  infrastructure_vpc_flow_logs_glue_table_columns = [
+    { name = "version", type = "int" },
+    { name = "account_id", type = "string" },
+    { name = "interface_id", type = "string" },
+    { name = "srcaddr", type = "string" },
+    { name = "dstaddr", type = "string" },
+    { name = "srcport", type = "int" },
+    { name = "dstport", type = "int" },
+    { name = "protocol", type = "bigint" },
+    { name = "packets", type = "bigint" },
+    { name = "bytes", type = "bigint" },
+    { name = "start", type = "bigint" },
+    { name = "`end`", type = "bigint" },
+    { name = "action", type = "string" },
+    { name = "log_status", type = "string" },
+    { name = "vpc_id", type = "string" },
+    { name = "subnet_id", type = "string" },
+    { name = "instance_id", type = "string" },
+    { name = "tcp_flags", type = "int" },
+    { name = "type", type = "string" },
+    { name = "pkt_srcaddr", type = "string" },
+    { name = "pkt_dstaddr", type = "string" },
+    { name = "az_id", type = "string" },
+    { name = "sublocation_type", type = "string" },
+    { name = "sublocation_id", type = "string" },
+    { name = "pkt_src_aws_service", type = "string" },
+    { name = "pkt_dst_aws_service", type = "string" },
+    { name = "flow_direction", type = "string" },
+    { name = "traffic_path", type = "int" },
+  ]
+  infrastructure_vpc_flow_logs_glue_table_partition_keys = [
+    { name = "aws-account-id", type = "string" },
+    { name = "aws-service", type = "string" },
+    { name = "aws-region", type = "string" },
+    { name = "year", type = "string" },
+    { name = "month", type = "string" },
+    { name = "day", type = "string" },
+    { name = "hour", type = "string" },
+  ]
   enable_infrastructure_vpc_transfer_s3_bucket = var.enable_infrastructure_vpc_transfer_s3_bucket
   infrastructure_vpc_transfer_s3_bucket_access_vpc_ids = concat(
     local.infrastructure_vpc ? [aws_vpc.infrastructure[0].id] : [],

vpc-infrastructure-flow-logs-glue-tables.tf

@@ -14,8 +14,8 @@ resource "aws_glue_catalog_table" "infrastructure_vpc_flow_logs" {
   dynamic "partition_keys" {
     for_each = local.infrastructure_vpc_flow_logs_glue_table_partition_keys
     content {
-      name = partition_keys.key
-      type = partition_keys.value
+      name = partition_keys.value["name"]
+      type = partition_keys.value["type"]
     }
   }
 
@@ -26,13 +26,18 @@ resource "aws_glue_catalog_table" "infrastructure_vpc_flow_logs" {
     "projection.enabled"        = "true"
     "projection.region.type"    = "enum"
     "projection.region.values"  = local.aws_region
-    "projection.day.type"       = "date"
-    "projection.day.range"      = "2023/01/01,NOW"
-    "projection.day.format"     = "yyyy/MM/dd"
+    "projection.year.type"      = "integer"
+    "projection.year.digits"    = "4"
+    "projection.month.type"     = "integer"
+    "projection.month.range"    = "00,12"
+    "projection.month.digits"   = "2"
+    "projection.day.type"       = "integer"
+    "projection.day.range"      = "00,31"
+    "projection.day.digits"     = "2"
     "projection.hour.type"      = "integer"
     "projection.hour.range"     = "00,23"
     "projection.hour.digits"    = "2"
-    "storage.location.template" = "s3://${aws_s3_bucket.infrastructure_logs[0].id}/${local.infrastructure_vpc_flow_logs_s3_key_prefix}/AWSLogs/${local.aws_account_id}/vpcflowlogs/$${region}/$${day}/$${hour}"
+    "storage.location.template" = "s3://${aws_s3_bucket.infrastructure_logs[0].id}/${local.infrastructure_vpc_flow_logs_s3_key_prefix}/AWSLogs/$${aws-account-id}/vpcflowlogs/$${region}/$${year}/$${month}/$${day}/$${hour}"
   }
 
   storage_descriptor {
@@ -50,8 +55,8 @@ resource "aws_glue_catalog_table" "infrastructure_vpc_flow_logs" {
     dynamic "columns" {
       for_each = local.infrastructure_vpc_flow_logs_glue_table_columns
       content {
-        name = columns.key
-        type = columns.value
+        name = columns.value["name"]
+        type = columns.value["type"]
       }
     }
   }