feat: refactor lambda service metadata

src/infra/api/oi.tpl.json

@@ -927,21 +927,35 @@
           }
         ],
         "x-amazon-apigateway-integration": {
-          "credentials": "${lambda_apigateway_proxy_role}",
-          "uri": "arn:aws:apigateway:${aws_region}:lambda:path/2015-03-31/functions/${metadata_lambda_arn}/invocations",
-          "passthroughBehavior": "when_no_match",
-          "httpMethod": "POST",
-          "cacheKeyParameters": [],
-          "type": "aws_proxy",
+          "credentials": "${s3_apigateway_proxy_role}",
+          "httpMethod": "GET",
+          "uri": "${metadata_bucket_uri}/{id_type}.xml",
+          "type": "aws",
+          "passthroughBehavior": "WHEN_NO_TEMPLATES",
           "requestParameters": {
             "integration.request.path.id_type": "method.request.path.id_type"
           },
-          "responses": {}
+          "responseParameters": {},
+          "responses": {
+            "200": {
+              "statusCode": "200",
+              "responseParameters": {
+                "method.response.header.content-type": "integration.response.header.Content-Type"
+              }
+            },
+            "404": {
+              "statusCode": "404",
+              "responseParameters": {}
+            }
+          }
         },
         "responses": {
           "200": {
             "$ref": "#/components/responses/responseOkXml"
           },
+          "404": {
+            "$ref": "#/components/responses/notFound"
+          },
           "405": {
             "$ref": "#/components/responses/methodNotAllowed"
           },

src/infra/dev/eu-south-1/main.tf

@@ -64,8 +64,11 @@ module "frontend" {
   api_method_settings            = var.api_method_settings
 
 
-  assets_bucket_arn  = module.storage.assets_bucket_arn
-  assets_bucket_name = module.storage.assets_bucket_name
+  assets_bucket_arn    = module.storage.assets_bucket_arn
+  assets_bucket_name   = module.storage.assets_bucket_name
+  metadata_bucket_arn  = module.storage.metadata_bucket_arn
+  metadata_bucket_name = module.storage.metadata_bucket_name
+
 
   xray_tracing_enabled = var.xray_tracing_enabled
   api_alarms           = local.cloudwatch__api_alarms_with_sns
@@ -88,6 +91,7 @@ module "storage" {
   assertions_crawler_schedule     = var.assertions_crawler_schedule
   idp_metadata_bucket_prefix      = "idp-metadata"
   assets_bucket_prefix            = "assets"
+  metadata_bucket_prefix          = "metadata"
   github_repository               = "pagopa/oneidentity"
   account_id                      = data.aws_caller_identity.current.account_id
   assertion_accesslogs_expiration = 2
@@ -228,6 +232,7 @@ module "backend" {
   metadata_lambda = {
     name                           = format("%s-metadata", local.project)
     filename                       = "${path.module}/../../hello-java/build/libs/hello-java-1.0-SNAPSHOT.jar"
+    metadata_bucket_arn            = module.storage.metadata_bucket_arn
     table_client_registrations_arn = module.database.table_client_registrations_arn
     environment_variables = {
       "ORGANIZATION_URL"                = "https://www.pagopa.it"
@@ -241,16 +246,19 @@ module "backend" {
       "CONTACT_PERSON_COMPANY"          = "PagoPA S.p.A."
       "CLIENT_REGISTRATIONS_TABLE_NAME" = "ClientRegistrations"
       "LOG_LEVEL"                       = var.app_log_level
+      "SERVICE_METADATA_BUCKET_NAME"    = module.storage.metadata_bucket_name
     }
     vpc_id                            = module.network.vpc_id
     vpc_subnet_ids                    = module.network.intra_subnets_ids
     vpc_endpoint_dynamodb_prefix_id   = module.network.vpc_endpoints["dynamodb"]["prefix_list_id"]
+    vpc_s3_prefix_id                  = module.network.vpc_endpoints["s3"]["prefix_list_id"]
     vpc_endpoint_ssm_nsg_ids          = tolist(module.network.vpc_endpoints["ssm"].security_group_ids)
     cloudwatch_logs_retention_in_days = var.lambda_cloudwatch_logs_retention_in_days
   }
 
 
-  dynamodb_table_stream_arn = module.database.dynamodb_table_stream_arn
+  dynamodb_clients_table_stream_arn = module.database.dynamodb_clients_table_stream_arn
+  dynamodb_table_stream_arn         = module.database.dynamodb_table_stream_arn
   eventbridge_pipe_sessions = {
     pipe_name                     = format("%s-sessions-pipe", local.project)
     kms_sessions_table_alias      = module.database.kms_sessions_table_alias_arn

src/infra/modules/backend/README.md

@@ -175,6 +175,7 @@
 | [aws_iam_role_policy_attachment.deploy_ecs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.deploy_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.switch_region](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_lambda_event_source_mapping.trigger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource |
 | [aws_pipes_pipe.sessions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/pipes_pipe) | resource |
 | [aws_s3_bucket_notification.bucket_notification](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource |
 | [aws_security_group_rule.metadata_vpc_tls](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
@@ -202,6 +203,7 @@
 | <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS Region. | `string` | n/a | yes |
 | <a name="input_client_registration_lambda"></a> [client\_registration\_lambda](#input\_client\_registration\_lambda) | n/a | <pre>object({<br>    name                              = string<br>    filename                          = string<br>    table_client_registrations_arn    = string<br>    cloudwatch_logs_retention_in_days = number<br>    vpc_id                            = string<br>    vpc_endpoint_dynamodb_prefix_id   = string<br>    vpc_subnet_ids                    = list(string)<br>    environment_variables             = map(string)<br>  })</pre> | n/a | yes |
 | <a name="input_dlq_alarms"></a> [dlq\_alarms](#input\_dlq\_alarms) | n/a | <pre>object({<br>    metric_name         = string<br>    namespace           = string<br>    threshold           = number<br>    evaluation_periods  = number<br>    period              = number<br>    statistic           = string<br>    comparison_operator = string<br>    sns_topic_alarm_arn = string<br>  })</pre> | n/a | yes |
+| <a name="input_dynamodb_clients_table_stream_arn"></a> [dynamodb\_clients\_table\_stream\_arn](#input\_dynamodb\_clients\_table\_stream\_arn) | n/a | `string` | `null` | no |
 | <a name="input_dynamodb_table_idpMetadata"></a> [dynamodb\_table\_idpMetadata](#input\_dynamodb\_table\_idpMetadata) | Dynamodb table idpMetadata anrs | <pre>object({<br>    table_arn       = string<br>    gsi_pointer_arn = string<br>  })</pre> | n/a | yes |
 | <a name="input_dynamodb_table_sessions"></a> [dynamodb\_table\_sessions](#input\_dynamodb\_table\_sessions) | Dynamodb table sessions anrs | <pre>object({<br>    table_arn    = string<br>    gsi_code_arn = string<br>  })</pre> | n/a | yes |
 | <a name="input_dynamodb_table_stream_arn"></a> [dynamodb\_table\_stream\_arn](#input\_dynamodb\_table\_stream\_arn) | n/a | `string` | `null` | no |
@@ -219,7 +221,8 @@
 | <a name="input_kms_sessions_table_alias_arn"></a> [kms\_sessions\_table\_alias\_arn](#input\_kms\_sessions\_table\_alias\_arn) | Kms key used to encrypt and dectypt session table. | `string` | n/a | yes |
 | <a name="input_kms_ssm_enable_rotation"></a> [kms\_ssm\_enable\_rotation](#input\_kms\_ssm\_enable\_rotation) | n/a | `bool` | `true` | no |
 | <a name="input_lambda_alarms"></a> [lambda\_alarms](#input\_lambda\_alarms) | n/a | <pre>map(object({<br>    metric_name         = string<br>    namespace           = string<br>    threshold           = number<br>    evaluation_periods  = number<br>    period              = number<br>    statistic           = string<br>    comparison_operator = string<br>    sns_topic_alarm_arn = string<br>    treat_missing_data  = string<br>  }))</pre> | n/a | yes |
-| <a name="input_metadata_lambda"></a> [metadata\_lambda](#input\_metadata\_lambda) | n/a | <pre>object({<br>    name                              = string<br>    filename                          = string<br>    table_client_registrations_arn    = string<br>    environment_variables             = map(string)<br>    vpc_id                            = string<br>    vpc_subnet_ids                    = list(string)<br>    vpc_endpoint_dynamodb_prefix_id   = string<br>    vpc_endpoint_ssm_nsg_ids          = list(string)<br>    cloudwatch_logs_retention_in_days = number<br>  })</pre> | n/a | yes |
+| <a name="input_lambda_client_registration_trigger_enabled"></a> [lambda\_client\_registration\_trigger\_enabled](#input\_lambda\_client\_registration\_trigger\_enabled) | n/a | `bool` | `true` | no |
+| <a name="input_metadata_lambda"></a> [metadata\_lambda](#input\_metadata\_lambda) | n/a | <pre>object({<br>    name                              = string<br>    filename                          = string<br>    table_client_registrations_arn    = string<br>    environment_variables             = map(string)<br>    metadata_bucket_arn               = string<br>    vpc_id                            = string<br>    vpc_subnet_ids                    = list(string)<br>    vpc_endpoint_dynamodb_prefix_id   = string<br>    vpc_s3_prefix_id                  = string<br>    vpc_endpoint_ssm_nsg_ids          = list(string)<br>    cloudwatch_logs_retention_in_days = number<br>  })</pre> | n/a | yes |
 | <a name="input_nlb_name"></a> [nlb\_name](#input\_nlb\_name) | Network load balancer name | `string` | n/a | yes |
 | <a name="input_private_subnets"></a> [private\_subnets](#input\_private\_subnets) | Private subnets ids. | `list(string)` | n/a | yes |
 | <a name="input_role_prefix"></a> [role\_prefix](#input\_role\_prefix) | IAM Role prefix. | `string` | n/a | yes |

src/infra/modules/backend/lambda.tf

@@ -143,17 +143,29 @@ module "client_registration_lambda" {
 
 data "aws_iam_policy_document" "metadata_lambda" {
   statement {
-    effect    = "Allow"
-    actions   = ["dynamodb:Scan"]
-    resources = ["${var.table_client_registrations_arn}"]
+    effect = "Allow"
+    actions = [
+      "dynamodb:DescribeStream",
+      "dynamodb:GetRecords",
+      "dynamodb:GetShardIterator",
+      "dynamodb:ListStreams",
+    ]
+    resources = [var.dynamodb_clients_table_stream_arn]
+  }
+  statement {
+    effect = "Allow"
+    actions = [
+      "dynamodb:Scan",
+    ]
+    resources = [var.table_client_registrations_arn]
   }
   statement {
     effect = "Allow"
     actions = [
       "kms:Decrypt",
       "kms:Encrypt",
     ]
-    resources = ["${module.kms_key_pem.aliases["keyPem/SSM"].target_key_arn}"]
+    resources = [module.kms_key_pem.aliases["keyPem/SSM"].target_key_arn]
   }
   statement {
     effect = "Allow"
@@ -162,9 +174,16 @@ data "aws_iam_policy_document" "metadata_lambda" {
       "ssm:Get*",
       "ssm:List*"
     ]
-    resources = ["${data.aws_ssm_parameter.certificate.arn}", "${aws_ssm_parameter.key_pem.arn}"]
+    resources = [data.aws_ssm_parameter.certificate.arn, aws_ssm_parameter.key_pem.arn]
+  }
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject"
+    ]
+    resources = ["${var.metadata_lambda.metadata_bucket_arn}/*"]
   }
-
 }
 
 module "security_group_lambda_metadata" {
@@ -176,12 +195,15 @@ module "security_group_lambda_metadata" {
 
   vpc_id = var.metadata_lambda.vpc_id
 
+  egress_cidr_blocks      = []
   egress_ipv6_cidr_blocks = []
 
   # Prefix list ids to use in all egress rules in this module
-  egress_prefix_list_ids = [var.metadata_lambda.vpc_endpoint_dynamodb_prefix_id]
+  egress_prefix_list_ids = [
+    var.metadata_lambda.vpc_endpoint_dynamodb_prefix_id
+  ]
 
-  // egress_rules = ["https-443-tcp"]
+  # egress_rules = ["https-443-tcp"]
 }
 
 resource "aws_security_group_rule" "metadata_vpc_tls" {
@@ -191,6 +213,8 @@ resource "aws_security_group_rule" "metadata_vpc_tls" {
   protocol                 = "tcp"
   security_group_id        = module.security_group_lambda_metadata.security_group_id
   source_security_group_id = var.metadata_lambda.vpc_endpoint_ssm_nsg_ids[1]
+  prefix_list_ids          = [var.metadata_lambda.vpc_s3_prefix_id]
+
 }
 
 module "metadata_lambda" {
@@ -211,6 +235,13 @@ module "metadata_lambda" {
   policy_json           = data.aws_iam_policy_document.metadata_lambda.json
   attach_network_policy = true
 
+  allowed_triggers = {
+    dynamodb = {
+      principal  = "dynamodb.amazonaws.com"
+      source_arn = var.dynamodb_clients_table_stream_arn
+    }
+  }
+
   environment_variables  = var.metadata_lambda.environment_variables
   vpc_subnet_ids         = var.metadata_lambda.vpc_subnet_ids
   vpc_security_group_ids = [module.security_group_lambda_metadata.security_group_id]
@@ -223,6 +254,18 @@ module "metadata_lambda" {
 
 }
 
+resource "aws_lambda_event_source_mapping" "trigger" {
+  count = var.lambda_client_registration_trigger_enabled != false ? 1 : 0
+  depends_on = [
+    module.metadata_lambda.lambda_function_name,
+    var.table_client_registrations_arn
+  ]
+  event_source_arn  = var.dynamodb_clients_table_stream_arn
+  function_name     = module.metadata_lambda.lambda_function_arn
+  starting_position = "LATEST"
+  enabled           = true
+}
+
 ## Lambda idp_metadata
 
 data "aws_iam_policy_document" "idp_metadata_lambda" {
@@ -241,8 +284,8 @@ data "aws_iam_policy_document" "idp_metadata_lambda" {
       "dynamodb:DeleteItem",
     ]
     resources = [
-      "${var.dynamodb_table_idpMetadata.table_arn}",
-      "${var.dynamodb_table_idpMetadata.gsi_pointer_arn}"
+      var.dynamodb_table_idpMetadata.table_arn,
+      var.dynamodb_table_idpMetadata.gsi_pointer_arn
     ]
   }
 
@@ -330,7 +373,7 @@ data "aws_iam_policy_document" "is_gh_integration_lambda" {
       "ssm:Get*",
       "ssm:List*"
     ]
-    resources = ["${data.aws_ssm_parameter.is_gh_integration_lambda.arn}"]
+    resources = [data.aws_ssm_parameter.is_gh_integration_lambda.arn]
   }
 }
 

src/infra/modules/backend/variables.tf

@@ -121,6 +121,11 @@ variable "table_client_registrations_arn" {
   description = "Dynamodb table client registrations arn."
 }
 
+variable "lambda_client_registration_trigger_enabled" {
+  type    = bool
+  default = true
+}
+
 variable "kms_sessions_table_alias_arn" {
   type        = string
   description = "Kms key used to encrypt and dectypt session table."
@@ -156,9 +161,11 @@ variable "metadata_lambda" {
     filename                          = string
     table_client_registrations_arn    = string
     environment_variables             = map(string)
+    metadata_bucket_arn               = string
     vpc_id                            = string
     vpc_subnet_ids                    = list(string)
     vpc_endpoint_dynamodb_prefix_id   = string
+    vpc_s3_prefix_id                  = string
     vpc_endpoint_ssm_nsg_ids          = list(string)
     cloudwatch_logs_retention_in_days = number
   })
@@ -192,6 +199,11 @@ variable "dynamodb_table_stream_arn" {
   default = null
 }
 
+variable "dynamodb_clients_table_stream_arn" {
+  type    = string
+  default = null
+}
+
 variable "assertion_lambda" {
   type = object({
     name                              = string

src/infra/modules/database/README.md

@@ -69,7 +69,7 @@ No resources.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_client_registrations_table"></a> [client\_registrations\_table](#input\_client\_registrations\_table) | Client registrations table configurations. | <pre>object({<br>    point_in_time_recovery_enabled = optional(bool, false)<br>    stream_enabled                 = optional(bool, false)<br>    stream_view_type               = optional(string, null)<br>    deletion_protection_enabled    = optional(bool, false)<br>    replication_regions = optional(list(object({<br>      region_name            = string<br>      propagate_tags         = optional(bool, true)<br>      point_in_time_recovery = optional(bool, true)<br>    })), [])<br>  })</pre> | n/a | yes |
+| <a name="input_client_registrations_table"></a> [client\_registrations\_table](#input\_client\_registrations\_table) | Client registrations table configurations. | <pre>object({<br>    point_in_time_recovery_enabled = optional(bool, false)<br>    stream_enabled                 = optional(bool, true)<br>    stream_view_type               = optional(string, "NEW_AND_OLD_IMAGES")<br>    deletion_protection_enabled    = optional(bool, false)<br>    replication_regions = optional(list(object({<br>      region_name            = string<br>      propagate_tags         = optional(bool, true)<br>      point_in_time_recovery = optional(bool, true)<br>    })), [])<br>  })</pre> | n/a | yes |
 | <a name="input_idp_metadata_table"></a> [idp\_metadata\_table](#input\_idp\_metadata\_table) | IDP Metadata table configurations. | <pre>object({<br>    point_in_time_recovery_enabled = optional(bool, false)<br>    stream_enabled                 = optional(bool, false)<br>    stream_view_type               = optional(string, null)<br>    deletion_protection_enabled    = optional(bool, false)<br>    replication_regions = optional(list(object({<br>      region_name            = string<br>      propagate_tags         = optional(bool, true)<br>      point_in_time_recovery = optional(bool, true)<br>    })), [])<br>  })</pre> | n/a | yes |
 | <a name="input_kms_rotation_period_in_days"></a> [kms\_rotation\_period\_in\_days](#input\_kms\_rotation\_period\_in\_days) | n/a | `number` | `365` | no |
 | <a name="input_kms_ssm_enable_rotation"></a> [kms\_ssm\_enable\_rotation](#input\_kms\_ssm\_enable\_rotation) | n/a | `bool` | `true` | no |
@@ -79,6 +79,7 @@ No resources.
 
 | Name | Description |
 |------|-------------|
+| <a name="output_dynamodb_clients_table_stream_arn"></a> [dynamodb\_clients\_table\_stream\_arn](#output\_dynamodb\_clients\_table\_stream\_arn) | n/a |
 | <a name="output_dynamodb_table_stream_arn"></a> [dynamodb\_table\_stream\_arn](#output\_dynamodb\_table\_stream\_arn) | n/a |
 | <a name="output_kms_sessions_table_alias_arn"></a> [kms\_sessions\_table\_alias\_arn](#output\_kms\_sessions\_table\_alias\_arn) | n/a |
 | <a name="output_table_client_registrations_arn"></a> [table\_client\_registrations\_arn](#output\_table\_client\_registrations\_arn) | n/a |

src/infra/modules/database/outputs.tf

@@ -26,6 +26,10 @@ output "dynamodb_table_stream_arn" {
   value = module.dynamodb_sessions_table.dynamodb_table_stream_arn
 }
 
+output "dynamodb_clients_table_stream_arn" {
+  value = try(module.dynamodb_table_client_registrations[0].dynamodb_table_stream_arn, null)
+}
+
 output "table_idp_metadata_name" {
   value = try(module.dynamodb_table_idpMetadata[0].dynamodb_table_id, null)
 }

src/infra/modules/database/variables.tf

@@ -23,8 +23,8 @@ variable "kms_rotation_period_in_days" {
 variable "client_registrations_table" {
   type = object({
     point_in_time_recovery_enabled = optional(bool, false)
-    stream_enabled                 = optional(bool, false)
-    stream_view_type               = optional(string, null)
+    stream_enabled                 = optional(bool, true)
+    stream_view_type               = optional(string, "NEW_AND_OLD_IMAGES")
     deletion_protection_enabled    = optional(bool, false)
     replication_regions = optional(list(object({
       region_name            = string