Merge pull request #2373 from cisagov/2372-api-gateway-alert-threshold

2372 alert for excessive 5xx responses; descriptions for alerts
cisagov · Dec 29, 2023 · a32ee59 · a32ee59
2 parents fe11a65 + 550385c
commit a32ee59
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 0 deletions.
diff --git a/infrastructure/api_gateway_alarms.tf b/infrastructure/api_gateway_alarms.tf
@@ -0,0 +1,20 @@
+resource "aws_cloudwatch_metric_alarm" "api_error_rate" {
+  alarm_name          = "${var.log_metric_api_error_rate}-alarm"
+  alarm_description   = "The percentage of API calls returning a 5xx error exceeds 5%"
+  metric_name         = "5XXError"
+  alarm_actions       = [aws_sns_topic.alarms.arn]
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  period              = 60
+  evaluation_periods  = 2
+  threshold           = 0.05
+  statistic           = "Average"
+  unit                = "Count"
+  treat_missing_data  = "notBreaching"
+
+  tags = {
+    Project  = var.project
+    Stage    = var.stage
+    Severity = var.severity_medium
+  }
+}
+
diff --git a/infrastructure/log_alarms.tf b/infrastructure/log_alarms.tf
@@ -1,5 +1,6 @@
 resource "aws_cloudwatch_metric_alarm" "root_user" {
   alarm_name          = "${var.log_metric_root_user}-alarm"
+  alarm_description   = "The root user account signed into AWS"
   metric_name         = var.log_metric_root_user
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -18,6 +19,7 @@ resource "aws_cloudwatch_metric_alarm" "root_user" {
 
 resource "aws_cloudwatch_metric_alarm" "unauthorized_api_call" {
   alarm_name          = "${var.log_metric_unauthorized_api_call}-alarm"
+  alarm_description   = "An API call returned an unauthorized error"
   metric_name         = var.log_metric_unauthorized_api_call
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -36,6 +38,7 @@ resource "aws_cloudwatch_metric_alarm" "unauthorized_api_call" {
 
 resource "aws_cloudwatch_metric_alarm" "login_without_mfa" {
   alarm_name          = "${var.log_metric_login_without_mfa}-alarm"
+  alarm_description   = "A user logged into AWS without MFA"
   metric_name         = var.log_metric_login_without_mfa
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -54,6 +57,7 @@ resource "aws_cloudwatch_metric_alarm" "login_without_mfa" {
 
 resource "aws_cloudwatch_metric_alarm" "iam_policy" {
   alarm_name          = "${var.log_metric_iam_policy}-alarm"
+  alarm_description   = "An IAM policy was modified"
   metric_name         = var.log_metric_iam_policy
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -72,6 +76,7 @@ resource "aws_cloudwatch_metric_alarm" "iam_policy" {
 
 resource "aws_cloudwatch_metric_alarm" "cloudtrail" {
   alarm_name          = "${var.log_metric_cloudtrail}-alarm"
+  alarm_description   = "CloudTrail configurations were modified"
   metric_name         = var.log_metric_cloudtrail
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -90,6 +95,7 @@ resource "aws_cloudwatch_metric_alarm" "cloudtrail" {
 
 resource "aws_cloudwatch_metric_alarm" "login_failure" {
   alarm_name          = "${var.log_metric_login_failure}-alarm"
+  alarm_description   = "A user sign in to AWS failed"
   metric_name         = var.log_metric_login_failure
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -108,6 +114,7 @@ resource "aws_cloudwatch_metric_alarm" "login_failure" {
 
 resource "aws_cloudwatch_metric_alarm" "cmk_delete_disable" {
   alarm_name          = "${var.log_metric_cmk_delete_disable}-alarm"
+  alarm_description   = "A customer-managed key was disabled or scheduled for deletion"
   metric_name         = var.log_metric_cmk_delete_disable
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -126,6 +133,7 @@ resource "aws_cloudwatch_metric_alarm" "cmk_delete_disable" {
 
 resource "aws_cloudwatch_metric_alarm" "s3_bucket_policy" {
   alarm_name          = "${var.log_metric_s3_bucket_policy}-alarm"
+  alarm_description   = "An S3 bucket policy was modified"
   metric_name         = var.log_metric_s3_bucket_policy
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -144,6 +152,7 @@ resource "aws_cloudwatch_metric_alarm" "s3_bucket_policy" {
 
 resource "aws_cloudwatch_metric_alarm" "aws_config" {
   alarm_name          = "${var.log_metric_aws_config}-alarm"
+  alarm_description   = "AWS Config was modified"
   metric_name         = var.log_metric_aws_config
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -162,6 +171,7 @@ resource "aws_cloudwatch_metric_alarm" "aws_config" {
 
 resource "aws_cloudwatch_metric_alarm" "security_group" {
   alarm_name          = "${var.log_metric_security_group}-alarm"
+  alarm_description   = "A security group was modified"
   metric_name         = var.log_metric_security_group
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -180,6 +190,7 @@ resource "aws_cloudwatch_metric_alarm" "security_group" {
 
 resource "aws_cloudwatch_metric_alarm" "nacl" {
   alarm_name          = "${var.log_metric_nacl}-alarm"
+  alarm_description   = "A network ACL was modified"
   metric_name         = var.log_metric_nacl
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -198,6 +209,7 @@ resource "aws_cloudwatch_metric_alarm" "nacl" {
 
 resource "aws_cloudwatch_metric_alarm" "network_gateway" {
   alarm_name          = "${var.log_metric_network_gateway}-alarm"
+  alarm_description   = "A network gateway was modified"
   metric_name         = var.log_metric_network_gateway
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -216,6 +228,7 @@ resource "aws_cloudwatch_metric_alarm" "network_gateway" {
 
 resource "aws_cloudwatch_metric_alarm" "route_table" {
   alarm_name          = "${var.log_metric_route_table}-alarm"
+  alarm_description   = "A route table was modified"
   metric_name         = var.log_metric_route_table
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -252,6 +265,7 @@ resource "aws_cloudwatch_metric_alarm" "vpc" {
 
 resource "aws_cloudwatch_metric_alarm" "ec2_shutdown" {
   alarm_name          = "${var.log_metric_ec2_shutdown}-alarm"
+  alarm_description   = "An EC2 instance was shut down"
   metric_name         = var.log_metric_ec2_shutdown
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -270,6 +284,7 @@ resource "aws_cloudwatch_metric_alarm" "ec2_shutdown" {
 
 resource "aws_cloudwatch_metric_alarm" "db_shutdown" {
   alarm_name          = "${var.log_metric_db_shutdown}-alarm"
+  alarm_description   = "An RDS instance was shut down"
   metric_name         = var.log_metric_db_shutdown
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]
@@ -288,6 +303,7 @@ resource "aws_cloudwatch_metric_alarm" "db_shutdown" {
 
 resource "aws_cloudwatch_metric_alarm" "db_deletion" {
   alarm_name          = "${var.log_metric_db_deletion}-alarm"
+  alarm_description   = "An RDS instance was deleted"
   metric_name         = var.log_metric_db_deletion
   namespace           = var.log_metric_namespace
   alarm_actions       = [aws_sns_topic.alarms.arn]

diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars
@@ -11,6 +11,7 @@ db_port                              = 5432
 db_table_name                        = "cfproddb"
 db_instance_class                    = "db.t3.2xlarge"
 log_metric_namespace                 = "LogMetrics"
+log_metric_api_error_rate            = "crossfeed-prod-APIErrorRate"
 log_metric_root_user                 = "crossfeed-prod-RootUserAccess"
 log_metric_unauthorized_api_call     = "crossfeed-prod-UnauthorizedApiCall"
 log_metric_login_without_mfa         = "crossfeed-prod-ConsoleSignInWithoutMFA"

diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars
@@ -12,6 +12,7 @@ db_table_name                        = "cfstagingdb"
 db_instance_class                    = "db.t3.2xlarge"
 log_metric_namespace                 = "LogMetrics"
 log_metric_root_user                 = "crossfeed-staging-RootUserAccess"
+log_metric_api_error_rate            = "crossfeed-staging-APIErrorRate"
 log_metric_unauthorized_api_call     = "crossfeed-staging-UnauthorizedApiCall"
 log_metric_login_without_mfa         = "crossfeed-staging-ConsoleSignInWithoutMFA"
 log_metric_iam_policy                = "crossfeed-staging-IAMPolicyChange"

diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf
@@ -70,6 +70,12 @@ variable "log_metric_namespace" {
   default     = "LogMetrics"
 }
 
+variable "log_metric_api_error_rate" {
+  description = "log_metric_filter_api_error_rate"
+  type        = string
+  default     = "crossfeed-staging-APIErrorRate"
+}
+
 variable "log_metric_root_user" {
   description = "log_metric_filter_root_user"
   type        = string