From bf9b10164bd72a1b16ebac7c8f09f5ee2ba5b81d Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 09:53:25 -0400 Subject: [PATCH 01/20] Add SQS/Lamda/Fargate Scan execution - Create the SQS queue in terraform - In Serverless Framework, define new Lambda function that will trigger fargate from message command --- backend/env.yml | 2 + backend/serverless.yml | 9 ++++ backend/src/tasks/functions.yml | 10 ++++ backend/src/tasks/scanExecution.ts | 75 ++++++++++++++++++++++++++++++ infrastructure/prod.tfvars | 1 + infrastructure/sqs.tf | 14 ++++++ infrastructure/stage.tfvars | 1 + infrastructure/vars.tf | 7 +++ 8 files changed, 119 insertions(+) create mode 100644 backend/src/tasks/scanExecution.ts create mode 100644 infrastructure/sqs.tf diff --git a/backend/env.yml b/backend/env.yml index 301404984..d0d2df1f5 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -41,6 +41,7 @@ staging: EXPORT_BUCKET_NAME: cisa-crossfeed-staging-exports PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports + SQS_QUEUE_NAME: crossfeed-staging-worker-queue prod: DB_DIALECT: 'postgres' @@ -76,6 +77,7 @@ prod: EXPORT_BUCKET_NAME: cisa-crossfeed-prod-exports PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports + SQS_QUEUE_NAME: crossfeed-prod-worker-queue dev-vpc: securityGroupIds: diff --git a/backend/serverless.yml b/backend/serverless.yml index 825aafd00..09005180e 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -57,6 +57,15 @@ provider: - s3:PutObject - s3:PutObjectAcl Resource: '*' + - Effect: Allow + Action: + - sts:AssumeRole + Resource: '*' + - Effect: Allow + Action: + - sqs:ReceiveMessage + - sqs:SendMessage + Resource: '*' functions: - ${file(./src/tasks/functions.yml)} diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index a7da691b5..20bd0f1c8 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -21,6 +21,16 @@ bastion: makeGlobalAdmin: handler: src/tasks/makeGlobalAdmin.handler +scanExecution: + handler: src/tasks/scanExecution.handler + events: + - sqs: + arn: + Fn::GetAtt: + - ${file(env.yml):${self:provider.stage}.SQS_QUEUE_NAME, ''} + - Arn + batchSize: 5 # Number of messages the lambda can continue to process while a fargate is still running + updateScanTaskStatus: handler: src/tasks/updateScanTaskStatus.handler events: diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts new file mode 100644 index 000000000..ded5194b9 --- /dev/null +++ b/backend/src/tasks/scanExecution.ts @@ -0,0 +1,75 @@ +import { SQSEvent, SQSRecord } from 'aws-lambda'; +import * as AWS from 'aws-sdk'; + +const ecs = new AWS.ECS(); +const sqs = new AWS.SQS(); + +export const invokeFargateTask = async (event: SQSEvent): Promise => { + try { + const sqsRecord: SQSRecord = event.Records[0]; + const commandOptions: string = sqsRecord.body; + + // Get the ARN of the SQS queue from the event + const sqsQueueArn: string | undefined = sqsRecord.eventSourceARN; + + if (!sqsQueueArn) { + throw new Error('SQS Queue ARN not found in event'); + } + + // Describe the SQS queue to get its URL + const sqsQueue = { + QueueUrl: sqsQueueArn // Use the ARN as the QueueUrl + }; + const queueAttributesResponse = await sqs + .getQueueAttributes(sqsQueue) + .promise(); + const sqsQueueUrl = queueAttributesResponse.Attributes?.QueueUrl; + + if (!sqsQueueUrl) { + throw new Error('SQS Queue URL not found'); + } + + const params: AWS.ECS.RunTaskRequest = { + cluster: process.env.FARGATE_CLUSTER_NAME!, + taskDefinition: process.env.FARGATE_TASK_DEFINITION_NAME!, + launchType: 'FARGATE', + networkConfiguration: { + awsvpcConfiguration: { + assignPublicIp: 'ENABLED', + securityGroups: [process.env.FARGATE_SG_ID!], + subnets: [process.env.FARGATE_SUBNET_ID!] + } + }, + platformVersion: '1.4.0', + overrides: { + containerOverrides: [ + { + name: 'main', // from task definition + command: [commandOptions] // Pass the command options as an array + } + ] + } + }; + + const data = await ecs.runTask(params).promise(); + console.log('Fargate task started:', data); + + // Send a message to the SQS queue to trigger processing + const sqsParams: AWS.SQS.SendMessageRequest = { + MessageBody: 'Start processing...', + QueueUrl: sqsQueueUrl + }; + await sqs.sendMessage(sqsParams).promise(); + + return { + statusCode: 200, + body: JSON.stringify('Fargate task started and message sent to SQS queue') + }; + } catch (error) { + console.error('Error starting Fargate task:', error); + return { + statusCode: 500, + body: JSON.stringify('Error starting Fargate task') + }; + } +}; diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars index 84a565b77..dff1c1233 100644 --- a/infrastructure/prod.tfvars +++ b/infrastructure/prod.tfvars @@ -67,3 +67,4 @@ create_db_accessor_instance = true db_accessor_instance_class = "t3.2xlarge" create_elk_instance = false elk_instance_class = "t3.2xlarge" +sqs_queue_name = "crossfeed-prod-worker-queue" diff --git a/infrastructure/sqs.tf b/infrastructure/sqs.tf new file mode 100644 index 000000000..2a62f8449 --- /dev/null +++ b/infrastructure/sqs.tf @@ -0,0 +1,14 @@ + +# SQS Queue +resource "aws_sqs_queue" "terraform_queue" { + name = var.sqs_queue_name + delay_seconds = 90 + max_message_size = 262144 + message_retention_seconds = 345600 # 4 days + receive_wait_time_seconds = 10 + + tags = { + Project = var.project + Stage = var.stage + } +} \ No newline at end of file diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index 9312408f2..f80fe20d5 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -67,3 +67,4 @@ create_db_accessor_instance = true db_accessor_instance_class = "t3.2xlarge" create_elk_instance = true elk_instance_class = "t3.2xlarge" +sqs_queue_name = "crossfeed-staging-worker-queue" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index c1e49cf6e..de9a093c6 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -411,3 +411,10 @@ variable "create_elk_instance" { type = bool default = false } + +variable "sqs_queue_name" { + description = "sqs_queue_name" + type = string + default = "crossfeed-staging-worker-queue" +} + From 36fc6a851b8a877336dfc44ae3d1fb5bfac44dcb Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 10:52:01 -0400 Subject: [PATCH 02/20] Add policy allowing Accessor EC2 to send SQS messages --- infrastructure/database.tf | 74 ++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 4c25fe6f3..cb3ab1be1 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -75,6 +75,33 @@ data "aws_ami" "ubuntu" { owners = ["099720109477"] } +# DB Accessor EC2 +resource "aws_instance" "db_accessor" { + count = var.create_db_accessor_instance ? 1 : 0 + ami = data.aws_ami.ubuntu.id + instance_type = var.db_accessor_instance_class + associate_public_ip_address = false + + tags = { + Project = var.project + Stage = var.stage + } + root_block_device { + volume_size = 1000 + } + + vpc_security_group_ids = [aws_security_group.allow_internal.id] + subnet_id = aws_subnet.backend.id + + iam_instance_profile = aws_iam_instance_profile.db_accessor.id + user_data = file("./ssm-agent-install.sh") + + lifecycle { + # prevent_destroy = true + ignore_changes = [ami] + } +} + resource "aws_iam_role" "db_accessor" { name = "crossfeed-db-accessor-${var.stage}" assume_role_policy = < Date: Wed, 4 Oct 2023 10:54:48 -0400 Subject: [PATCH 03/20] Add additional permissions for EC2 to interact with SQS "sqs:SendMessage", "sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes", "sqs:ListQueues", "sqs:GetQueueUrl" --- infrastructure/database.tf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index cb3ab1be1..bda061d8f 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -172,7 +172,12 @@ resource "aws_iam_policy" "sqs_send_message_policy" { Statement = [ { Action = [ - "sqs:SendMessage" + "sqs:SendMessage", + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ListQueues", + "sqs:GetQueueUrl" ], Effect = "Allow", Resource = aws_sqs_queue.terraform_queue.arn From 51715daed4c921297a5f29ac2a8823c48dadb135 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 11:05:13 -0400 Subject: [PATCH 04/20] Add 10 minute timeout to the lambda function --- backend/src/tasks/functions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index f2093fef0..8d6b54209 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -31,6 +31,7 @@ makeGlobalAdmin: scanExecution: handler: src/tasks/scanExecution.handler + timeout: 600 # 10 minutes events: - sqs: arn: From f0dfdf959b2324967173ded7e4337fda9fdb65cb Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 11:18:15 -0400 Subject: [PATCH 05/20] remove extra lines in database.tf --- infrastructure/database.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index bda061d8f..bbdf78339 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -237,7 +237,6 @@ resource "aws_ssm_parameter" "worker_subnet_id" { } } - resource "aws_ssm_parameter" "crossfeed_send_db_host" { name = var.ssm_db_host type = "SecureString" @@ -296,7 +295,6 @@ resource "aws_s3_bucket_logging" "reports_bucket" { target_prefix = "reports_bucket/" } - # P&E DB Backups S3 bucket resource "aws_s3_bucket" "pe_db_backups_bucket" { bucket = var.pe_db_backups_bucket_name From b9f86c196a80e75ba68dbd00f97b7631eac5a33f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 12:28:06 -0400 Subject: [PATCH 06/20] Edit accessor permission to be inline instead of a new policy --- backend/src/tasks/functions.yml | 4 ++-- infrastructure/database.tf | 12 +++--------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index 8d6b54209..c12b3cfbe 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -31,14 +31,14 @@ makeGlobalAdmin: scanExecution: handler: src/tasks/scanExecution.handler - timeout: 600 # 10 minutes + timeout: 300 # 5 minutes events: - sqs: arn: Fn::GetAtt: - ${file(env.yml):${self:provider.stage}.SQS_QUEUE_NAME, ''} - Arn - batchSize: 5 # Number of messages the lambda can continue to process while a fargate is still running + batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running updateScanTaskStatus: handler: src/tasks/updateScanTaskStatus.handler diff --git a/infrastructure/database.tf b/infrastructure/database.tf index bbdf78339..615f930ba 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -164,9 +164,9 @@ resource "aws_iam_role_policy" "db_accessor_s3_policy" { EOF } -resource "aws_iam_policy" "sqs_send_message_policy" { - name = "ec2-send-sqs-message-${var.stage}" - description = "IAM policy to allow sending messages to SQS queue" +resource "aws_iam_role_policy" "sqs_send_message_policy" { + name_prefix = "ec2-send-sqs-message-${var.stage}" + role = aws_iam_role.db_accessor.id policy = jsonencode({ Version = "2012-10-17", Statement = [ @@ -186,12 +186,6 @@ resource "aws_iam_policy" "sqs_send_message_policy" { }) } -resource "aws_iam_policy_attachment" "db_accessor_3" { - name = "crossfeed-db-accessor-${var.stage}" - roles = [aws_iam_role.db_accessor.name] - policy_arn = aws_iam_policy.sqs_send_message_policy.arn -} - # Lambda and Fargate SSM Parameters resource "aws_ssm_parameter" "lambda_sg_id" { name = var.ssm_lambda_sg From 40a95076272cdd57175b313cf0092510af8d417f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 12:31:28 -0400 Subject: [PATCH 07/20] Format terraform --- infrastructure/database.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index a50c24462..0090c6907 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -165,8 +165,8 @@ EOF } resource "aws_iam_role_policy" "sqs_send_message_policy" { - name_prefix = "ec2-send-sqs-message-${var.stage}" - role = aws_iam_role.db_accessor.id + name_prefix = "ec2-send-sqs-message-${var.stage}" + role = aws_iam_role.db_accessor.id policy = jsonencode({ Version = "2012-10-17", Statement = [ From c1794f4c813a74aa9aa4a76dd3e2d56e05458afe Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 13:45:47 -0400 Subject: [PATCH 08/20] Fix SQS in serverless deploy --- backend/env.yml | 4 ++-- backend/src/tasks/functions.yml | 3 +-- infrastructure/prod.tfvars | 4 +--- infrastructure/sqs.tf | 7 +++++++ infrastructure/stage.tfvars | 4 +--- infrastructure/vars.tf | 24 ++++++------------------ 6 files changed, 18 insertions(+), 28 deletions(-) diff --git a/backend/env.yml b/backend/env.yml index ac16d4ddc..9f140e4e5 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -41,7 +41,7 @@ staging: EXPORT_BUCKET_NAME: cisa-crossfeed-staging-exports PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports - SQS_QUEUE_NAME: crossfeed-staging-worker-queue + SQS_QUEUE_ARN: ${ssm:/crossfeed/staging/SQS_QUEUE_ARN} CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-staging-cloudwatch prod: @@ -78,7 +78,7 @@ prod: EXPORT_BUCKET_NAME: cisa-crossfeed-prod-exports PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports - SQS_QUEUE_NAME: crossfeed-prod-worker-queue + SQS_QUEUE_ARN: ${ssm:/crossfeed/prod/SQS_QUEUE_ARN} CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-prod-cloudwatch dev-vpc: diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index c12b3cfbe..8a219641f 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -36,8 +36,7 @@ scanExecution: - sqs: arn: Fn::GetAtt: - - ${file(env.yml):${self:provider.stage}.SQS_QUEUE_NAME, ''} - - Arn + - ${self:custom.sqsEnvVar}.${self:provider.stage}.SQS_QUEUE_ARN batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running updateScanTaskStatus: diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars index 3db2dd255..ba9997689 100644 --- a/infrastructure/prod.tfvars +++ b/infrastructure/prod.tfvars @@ -42,12 +42,9 @@ worker_ecs_task_definition_family = "crossfeed-prod-worker" worker_ecs_log_group_name = "crossfeed-prod-worker" worker_ecs_role_name = "crossfeed-prod-worker" logging_bucket_name = "cisa-crossfeed-prod-logging" -cloudtrail_name = "crossfeed-prod-all-events" cloudtrail_bucket_name = "cisa-crossfeed-prod-cloudtrail" cloudtrail_role_name = "cisa-crossfeed-prod-cloudtrail" cloudtrail_log_group_name = "cisa-crossfeed-prod-cloudtrail" -cloudwatch_bucket_name = "cisa-crossfeed-prod-cloudwatch" -cloudwatch_log_group_name = "crossfeed-prod-cloudwatch-bucket" export_bucket_name = "cisa-crossfeed-prod-exports" reports_bucket_name = "cisa-crossfeed-prod-reports" pe_db_backups_bucket_name = "cisa-crossfeed-prod-pe-db-backups" @@ -71,3 +68,4 @@ db_accessor_instance_class = "t3.2xlarge" create_elk_instance = false elk_instance_class = "t3.2xlarge" sqs_queue_name = "crossfeed-prod-worker-queue" +ssm_sqs_queue_arn = "/crossfeed/prod/SQS_QUEUE_ARN" diff --git a/infrastructure/sqs.tf b/infrastructure/sqs.tf index 2a62f8449..a7e182501 100644 --- a/infrastructure/sqs.tf +++ b/infrastructure/sqs.tf @@ -11,4 +11,11 @@ resource "aws_sqs_queue" "terraform_queue" { Project = var.project Stage = var.stage } +} + +resource "aws_ssm_parameter" "sqs_queue_arn" { + name = var.ssm_sqs_queue_arn + description = "ARN of the SQS queue" + type = "String" + value = aws_sqs_queue.terraform_queue.arn } \ No newline at end of file diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index 6eac78731..7c6cb4b24 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -42,12 +42,9 @@ worker_ecs_task_definition_family = "crossfeed-staging-worker" worker_ecs_log_group_name = "crossfeed-staging-worker" worker_ecs_role_name = "crossfeed-staging-worker" logging_bucket_name = "cisa-crossfeed-staging-logging" -cloudtrail_name = "crossfeed-staging-all-events" cloudtrail_bucket_name = "cisa-crossfeed-staging-cloudtrail" cloudtrail_role_name = "cisa-crossfeed-staging-cloudtrail" cloudtrail_log_group_name = "cisa-crossfeed-staging-cloudtrail" -cloudwatch_bucket_name = "cisa-crossfeed-staging-cloudwatch" -cloudwatch_log_group_name = "crossfeed-staging-cloudwatch-bucket" export_bucket_name = "cisa-crossfeed-staging-exports" reports_bucket_name = "cisa-crossfeed-staging-reports" pe_db_backups_bucket_name = "cisa-crossfeed-staging-pe-db-backups" @@ -71,3 +68,4 @@ db_accessor_instance_class = "t3.2xlarge" create_elk_instance = true elk_instance_class = "t3.2xlarge" sqs_queue_name = "crossfeed-staging-worker-queue" +ssm_sqs_queue_arn = "/crossfeed/staging/SQS_QUEUE_ARN" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index b80769e84..00b23d750 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -117,6 +117,12 @@ variable "ssm_worker_subnet" { default = "/crossfeed/staging/WORKER_SUBNET_ID" } +variable "ssm_sqs_queue_arn" { + description = "ssm_sqs_queue_arn" + type = string + default = "/crossfeed/staging/SQS_QUEUE_ARN" +} + variable "ssm_worker_arn" { description = "ssm_worker_arn" type = string @@ -262,12 +268,6 @@ variable "logging_bucket_name" { default = "cisa-crossfeed-staging-logging" } -variable "cloudtrail_name" { - description = "cloudtrail_name" - type = string - default = "crossfeed-staging-all-events" -} - variable "cloudtrail_bucket_name" { description = "cloudtrail_bucket_name" type = string @@ -286,18 +286,6 @@ variable "cloudtrail_log_group_name" { default = "crossfeed-staging-cloudtrail-logs" } -variable "cloudwatch_bucket_name" { - description = "cloudwatch_bucket_name" - type = string - default = "cisa-crossfeed-staging-cloudwatch" -} - -variable "cloudwatch_log_group_name" { - description = "cloudwatch_log_group_name" - type = string - default = "crossfeed-staging-cloudwatch-bucket" -} - variable "export_bucket_name" { description = "export_bucket_name" type = string From fcebb6f155fef4a3d2572fc4fb1c7bc219e3465d Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 13:48:08 -0400 Subject: [PATCH 09/20] Run terraform fmt --- infrastructure/sqs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/sqs.tf b/infrastructure/sqs.tf index a7e182501..b023c4e88 100644 --- a/infrastructure/sqs.tf +++ b/infrastructure/sqs.tf @@ -17,5 +17,5 @@ resource "aws_ssm_parameter" "sqs_queue_arn" { name = var.ssm_sqs_queue_arn description = "ARN of the SQS queue" type = "String" - value = aws_sqs_queue.terraform_queue.arn + value = aws_sqs_queue.terraform_queue.arn } \ No newline at end of file From 0735f20369f9c0f8637f3067a28332addb8f196d Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 13:53:01 -0400 Subject: [PATCH 10/20] Fix merge conflict error --- infrastructure/prod.tfvars | 3 +++ infrastructure/stage.tfvars | 3 +++ infrastructure/vars.tf | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars index ba9997689..54de413ad 100644 --- a/infrastructure/prod.tfvars +++ b/infrastructure/prod.tfvars @@ -42,9 +42,12 @@ worker_ecs_task_definition_family = "crossfeed-prod-worker" worker_ecs_log_group_name = "crossfeed-prod-worker" worker_ecs_role_name = "crossfeed-prod-worker" logging_bucket_name = "cisa-crossfeed-prod-logging" +cloudtrail_name = "crossfeed-prod-all-events" cloudtrail_bucket_name = "cisa-crossfeed-prod-cloudtrail" cloudtrail_role_name = "cisa-crossfeed-prod-cloudtrail" cloudtrail_log_group_name = "cisa-crossfeed-prod-cloudtrail" +cloudwatch_bucket_name = "cisa-crossfeed-prod-cloudwatch" +cloudwatch_log_group_name = "crossfeed-prod-cloudwatch-bucket" export_bucket_name = "cisa-crossfeed-prod-exports" reports_bucket_name = "cisa-crossfeed-prod-reports" pe_db_backups_bucket_name = "cisa-crossfeed-prod-pe-db-backups" diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index 7c6cb4b24..52d17d558 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -42,9 +42,12 @@ worker_ecs_task_definition_family = "crossfeed-staging-worker" worker_ecs_log_group_name = "crossfeed-staging-worker" worker_ecs_role_name = "crossfeed-staging-worker" logging_bucket_name = "cisa-crossfeed-staging-logging" +cloudtrail_name = "crossfeed-staging-all-events" cloudtrail_bucket_name = "cisa-crossfeed-staging-cloudtrail" cloudtrail_role_name = "cisa-crossfeed-staging-cloudtrail" cloudtrail_log_group_name = "cisa-crossfeed-staging-cloudtrail" +cloudwatch_bucket_name = "cisa-crossfeed-staging-cloudwatch" +cloudwatch_log_group_name = "crossfeed-staging-cloudwatch-bucket" export_bucket_name = "cisa-crossfeed-staging-exports" reports_bucket_name = "cisa-crossfeed-staging-reports" pe_db_backups_bucket_name = "cisa-crossfeed-staging-pe-db-backups" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index 00b23d750..3e4ea5005 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -268,6 +268,12 @@ variable "logging_bucket_name" { default = "cisa-crossfeed-staging-logging" } +variable "cloudtrail_name" { + description = "cloudtrail_name" + type = string + default = "crossfeed-staging-all-events" +} + variable "cloudtrail_bucket_name" { description = "cloudtrail_bucket_name" type = string @@ -286,6 +292,18 @@ variable "cloudtrail_log_group_name" { default = "crossfeed-staging-cloudtrail-logs" } +variable "cloudwatch_bucket_name" { + description = "cloudwatch_bucket_name" + type = string + default = "cisa-crossfeed-staging-cloudwatch" +} + +variable "cloudwatch_log_group_name" { + description = "cloudwatch_log_group_name" + type = string + default = "crossfeed-staging-cloudwatch-bucket" +} + variable "export_bucket_name" { description = "export_bucket_name" type = string From b4169d6a1cbf81d92ff1e662317f212aace2f682 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 13:59:22 -0400 Subject: [PATCH 11/20] simplify lambda definition --- backend/src/tasks/functions.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index 8a219641f..afde85895 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -34,9 +34,7 @@ scanExecution: timeout: 300 # 5 minutes events: - sqs: - arn: - Fn::GetAtt: - - ${self:custom.sqsEnvVar}.${self:provider.stage}.SQS_QUEUE_ARN + arn: ${self:custom.sqsEnvVar}.${self:provider.stage}.SQS_QUEUE_ARN batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running updateScanTaskStatus: From 380522ff318766077e717735fd13f12a171eab0e Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 14:10:41 -0400 Subject: [PATCH 12/20] fix arn call --- backend/src/tasks/functions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index afde85895..9cd391d7b 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -34,7 +34,7 @@ scanExecution: timeout: 300 # 5 minutes events: - sqs: - arn: ${self:custom.sqsEnvVar}.${self:provider.stage}.SQS_QUEUE_ARN + arn: ${file(env.yml):${self:provider.stage}.SQS_QUEUE_ARN batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running updateScanTaskStatus: From c8e661911a277bb0d1f998370986c1f4c0395b61 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 14:18:27 -0400 Subject: [PATCH 13/20] fix typo --- backend/src/tasks/functions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index 9cd391d7b..456d6872f 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -34,7 +34,7 @@ scanExecution: timeout: 300 # 5 minutes events: - sqs: - arn: ${file(env.yml):${self:provider.stage}.SQS_QUEUE_ARN + arn: ${file(env.yml):${self:provider.stage}.SQS_QUEUE_ARN, ''} batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running updateScanTaskStatus: From f2404bf4ac3640e16e186b5b41ab57edaed06549 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 14:32:18 -0400 Subject: [PATCH 14/20] fix typo --- backend/src/tasks/functions.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index 456d6872f..2208ea57b 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -34,7 +34,8 @@ scanExecution: timeout: 300 # 5 minutes events: - sqs: - arn: ${file(env.yml):${self:provider.stage}.SQS_QUEUE_ARN, ''} + arn: + - ${file(env.yml):${self:provider.stage}.SQS_QUEUE_ARN, ''} batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running updateScanTaskStatus: From 6df8f35bbccb90d2a39187bfc484322af258fb8f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 15:03:39 -0400 Subject: [PATCH 15/20] fix sqs arn definition --- backend/src/tasks/functions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index 2208ea57b..538241ad4 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -35,7 +35,7 @@ scanExecution: events: - sqs: arn: - - ${file(env.yml):${self:provider.stage}.SQS_QUEUE_ARN, ''} + Fn::ImportValue: ${file(env.yml):${self:provider.stage}.SQS_QUEUE_ARN, ''} batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running updateScanTaskStatus: From 3f3316afbe27b0f44dedf588fa8ae4319f4291ea Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 15:18:02 -0400 Subject: [PATCH 16/20] Create the queue in Serverless instead of terraform --- backend/env.yml | 2 -- backend/serverless.yml | 7 +++++++ backend/src/tasks/functions.yml | 4 +++- infrastructure/prod.tfvars | 2 -- infrastructure/sqs.tf | 21 --------------------- infrastructure/stage.tfvars | 2 -- infrastructure/vars.tf | 13 ------------- 7 files changed, 10 insertions(+), 41 deletions(-) delete mode 100644 infrastructure/sqs.tf diff --git a/backend/env.yml b/backend/env.yml index 9f140e4e5..4a922725a 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -41,7 +41,6 @@ staging: EXPORT_BUCKET_NAME: cisa-crossfeed-staging-exports PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports - SQS_QUEUE_ARN: ${ssm:/crossfeed/staging/SQS_QUEUE_ARN} CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-staging-cloudwatch prod: @@ -78,7 +77,6 @@ prod: EXPORT_BUCKET_NAME: cisa-crossfeed-prod-exports PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports - SQS_QUEUE_ARN: ${ssm:/crossfeed/prod/SQS_QUEUE_ARN} CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-prod-cloudwatch dev-vpc: diff --git a/backend/serverless.yml b/backend/serverless.yml index 0524e0af4..c8e4f94d3 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -79,6 +79,13 @@ provider: - logs:StopLiveTail Resource: '*' +resources: + Resources: + MySQSQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: ${self:provider.stage}-worker-queue + functions: - ${file(./src/tasks/functions.yml)} - ${file(./src/api/functions.yml)} diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index 538241ad4..cc373c320 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -35,7 +35,9 @@ scanExecution: events: - sqs: arn: - Fn::ImportValue: ${file(env.yml):${self:provider.stage}.SQS_QUEUE_ARN, ''} + Fn::GetAtt: + - ${self:provider.stage}-worker-queue + - Arn batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running updateScanTaskStatus: diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars index 54de413ad..db987357d 100644 --- a/infrastructure/prod.tfvars +++ b/infrastructure/prod.tfvars @@ -70,5 +70,3 @@ create_db_accessor_instance = true db_accessor_instance_class = "t3.2xlarge" create_elk_instance = false elk_instance_class = "t3.2xlarge" -sqs_queue_name = "crossfeed-prod-worker-queue" -ssm_sqs_queue_arn = "/crossfeed/prod/SQS_QUEUE_ARN" diff --git a/infrastructure/sqs.tf b/infrastructure/sqs.tf deleted file mode 100644 index b023c4e88..000000000 --- a/infrastructure/sqs.tf +++ /dev/null @@ -1,21 +0,0 @@ - -# SQS Queue -resource "aws_sqs_queue" "terraform_queue" { - name = var.sqs_queue_name - delay_seconds = 90 - max_message_size = 262144 - message_retention_seconds = 345600 # 4 days - receive_wait_time_seconds = 10 - - tags = { - Project = var.project - Stage = var.stage - } -} - -resource "aws_ssm_parameter" "sqs_queue_arn" { - name = var.ssm_sqs_queue_arn - description = "ARN of the SQS queue" - type = "String" - value = aws_sqs_queue.terraform_queue.arn -} \ No newline at end of file diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index 52d17d558..a0a05cef0 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -70,5 +70,3 @@ create_db_accessor_instance = true db_accessor_instance_class = "t3.2xlarge" create_elk_instance = true elk_instance_class = "t3.2xlarge" -sqs_queue_name = "crossfeed-staging-worker-queue" -ssm_sqs_queue_arn = "/crossfeed/staging/SQS_QUEUE_ARN" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index 3e4ea5005..c3d05237b 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -117,12 +117,6 @@ variable "ssm_worker_subnet" { default = "/crossfeed/staging/WORKER_SUBNET_ID" } -variable "ssm_sqs_queue_arn" { - description = "ssm_sqs_queue_arn" - type = string - default = "/crossfeed/staging/SQS_QUEUE_ARN" -} - variable "ssm_worker_arn" { description = "ssm_worker_arn" type = string @@ -435,10 +429,3 @@ variable "create_elk_instance" { type = bool default = false } - -variable "sqs_queue_name" { - description = "sqs_queue_name" - type = string - default = "crossfeed-staging-worker-queue" -} - From a06a90623b6a1f4342f8f1f81800937c474e6a01 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 15:20:12 -0400 Subject: [PATCH 17/20] fix sqs policy for db accessor ec2 --- infrastructure/database.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 125fc33be..fd516fb92 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -180,7 +180,7 @@ resource "aws_iam_role_policy" "sqs_send_message_policy" { "sqs:GetQueueUrl" ], Effect = "Allow", - Resource = aws_sqs_queue.terraform_queue.arn + Resource = "*" } ] }) From 0c6913bf05df02ce49df606b75d911e9bfa114ef Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 15:57:14 -0400 Subject: [PATCH 18/20] Fix reference to SQS queue --- backend/serverless.yml | 2 +- backend/src/tasks/functions.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/serverless.yml b/backend/serverless.yml index c8e4f94d3..dcd57fd65 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -81,7 +81,7 @@ provider: resources: Resources: - MySQSQueue: + WorkerQueue: Type: AWS::SQS::Queue Properties: QueueName: ${self:provider.stage}-worker-queue diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index cc373c320..504dd17c1 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -32,11 +32,13 @@ makeGlobalAdmin: scanExecution: handler: src/tasks/scanExecution.handler timeout: 300 # 5 minutes + environment: + SQS_QUEUE_NAME: ${self:provider.stage}-worker-queue events: - sqs: arn: Fn::GetAtt: - - ${self:provider.stage}-worker-queue + - WorkerQueue - Arn batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running From 651d46e830ddef957923c044edf88d857b16c28f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 16:33:28 -0400 Subject: [PATCH 19/20] Add visibility timeout to SQS --- backend/src/tasks/functions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index 504dd17c1..be638ff86 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -41,6 +41,7 @@ scanExecution: - WorkerQueue - Arn batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running + visibilityTimeout: 300 # Should match or exceed function timeout updateScanTaskStatus: handler: src/tasks/updateScanTaskStatus.handler From ede6b97272175089a4ad4fffe976f85820f97773 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 4 Oct 2023 16:49:11 -0400 Subject: [PATCH 20/20] Add other properties --- backend/serverless.yml | 3 +++ backend/src/tasks/functions.yml | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/serverless.yml b/backend/serverless.yml index dcd57fd65..defa5aa9f 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -85,6 +85,9 @@ resources: Type: AWS::SQS::Queue Properties: QueueName: ${self:provider.stage}-worker-queue + VisibilityTimeout: 300 # Should match or exceed function timeout + MaximumMessageSize: 262144 # 256 KB + MessageRetentionPeriod: 604800 # 7 days functions: - ${file(./src/tasks/functions.yml)} diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index be638ff86..504dd17c1 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -41,7 +41,6 @@ scanExecution: - WorkerQueue - Arn batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running - visibilityTimeout: 300 # Should match or exceed function timeout updateScanTaskStatus: handler: src/tasks/updateScanTaskStatus.handler