diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index c3956d29c..b668501d7 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -9,6 +9,8 @@ COPY src ./src RUN apt update && apt install git zlib1g-dev +RUN apt-get update && apt-get install -y jq + RUN wget -c https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tar.xz && tar -Jxvf Python-3.10.11.tar.xz RUN cd Python-3.10.11 && ./configure && make -j4 && make altinstall RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.10 1 @@ -21,4 +23,6 @@ RUN apt remove dav1d && apt autoclean && apt autoremove # Sync the latest from cf-staging branch RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && pip install . +COPY worker worker + CMD ["./worker/pe-worker-entry.sh"] diff --git a/backend/serverless.yml b/backend/serverless.yml index c8aad3d8d..2923c88b1 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -43,6 +43,9 @@ provider: Action: - ecs:RunTask - ecs:ListTasks + - ecs:DescribeTasks + - ecs:DescribeServices + - ecs:UpdateService - iam:PassRole Resource: '*' - Effect: Allow @@ -65,7 +68,9 @@ provider: - Effect: Allow Action: - sqs:ReceiveMessage + - sqs:DeleteMessage - sqs:SendMessage + - sqs:GetQueueAttributes Resource: '*' - Effect: Allow Action: @@ -108,7 +113,6 @@ resources: MaximumMessageSize: 262144 # 256 KB MessageRetentionPeriod: 604800 # 7 days - functions: - ${file(./src/tasks/functions.yml)} - ${file(./src/api/functions.yml)} diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts index 5bff875c8..cb4fad528 100644 --- a/backend/src/tasks/scanExecution.ts +++ b/backend/src/tasks/scanExecution.ts @@ -29,6 +29,24 @@ export const handler: Handler = async (event) => { } else { console.log('Shodan is the only script type available right now.'); } + + // After processing each message, check if the SQS queue is empty + const sqsAttributes = await sqs + .getQueueAttributes({ + QueueUrl: process.env.SHODAN_QUEUE_URL!, + AttributeNames: ['ApproximateNumberOfMessages'] + }) + .promise(); + + const approximateNumberOfMessages = parseInt( + sqsAttributes.Attributes?.ApproximateNumberOfMessages || '0', + 10 + ); + + // If the queue is empty, scale down to zero tasks + if (approximateNumberOfMessages === 0) { + await startFargateTask(clusterName, process.env.SHODAN_SERVICE_NAME!, 0); + } } catch (error) { console.error(error); return { @@ -59,7 +77,7 @@ export async function startFargateTask( const service = serviceDescription.services[0]; // Check if the desired task count is less than # provided - if (service.desiredCount! < desiredCountNum) { + if (service.desiredCount! !== desiredCountNum) { const updateServiceParams = { cluster: clusterName, service: serviceName, diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh old mode 100644 new mode 100755 index a50779725..c5392092d --- a/backend/worker/pe-worker-entry.sh +++ b/backend/worker/pe-worker-entry.sh @@ -1,7 +1,9 @@ -#!/bin/sh +#!/bin/bash set -e +echo "Starting pe-worker-entry.sh script" + # Check if the SHODAN_QUEUE_URL environment variable is set if [ -z "$SHODAN_QUEUE_URL" ]; then echo "SHODAN_QUEUE_URL environment variable is not set. Exiting." diff --git a/infrastructure/log_alarms.tf b/infrastructure/log_alarms.tf index fd0456545..e75aeb2cc 100644 --- a/infrastructure/log_alarms.tf +++ b/infrastructure/log_alarms.tf @@ -7,7 +7,7 @@ resource "aws_cloudwatch_metric_alarm" "root_user" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -25,7 +25,7 @@ resource "aws_cloudwatch_metric_alarm" "unauthorized_api_call" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -43,7 +43,7 @@ resource "aws_cloudwatch_metric_alarm" "login_without_mfa" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -61,7 +61,7 @@ resource "aws_cloudwatch_metric_alarm" "iam_policy" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -79,7 +79,7 @@ resource "aws_cloudwatch_metric_alarm" "cloudtrail" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -97,7 +97,7 @@ resource "aws_cloudwatch_metric_alarm" "login_failure" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -115,7 +115,7 @@ resource "aws_cloudwatch_metric_alarm" "cmk_delete_disable" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -133,7 +133,7 @@ resource "aws_cloudwatch_metric_alarm" "s3_bucket_policy" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -151,7 +151,7 @@ resource "aws_cloudwatch_metric_alarm" "aws_config" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -169,7 +169,7 @@ resource "aws_cloudwatch_metric_alarm" "security_group" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -187,7 +187,7 @@ resource "aws_cloudwatch_metric_alarm" "nacl" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -205,7 +205,7 @@ resource "aws_cloudwatch_metric_alarm" "network_gateway" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -223,7 +223,7 @@ resource "aws_cloudwatch_metric_alarm" "route_table" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -241,7 +241,7 @@ resource "aws_cloudwatch_metric_alarm" "vpc" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -259,7 +259,7 @@ resource "aws_cloudwatch_metric_alarm" "ec2_shutdown" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -277,7 +277,7 @@ resource "aws_cloudwatch_metric_alarm" "db_shutdown" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project @@ -295,7 +295,7 @@ resource "aws_cloudwatch_metric_alarm" "db_deletion" { evaluation_periods = 1 period = 60 threshold = 1 - statistic = "SampleCount" + statistic = "Sum" tags = { Project = var.project diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf index cf039c9fd..8c0c1d6d7 100644 --- a/infrastructure/pe_worker.tf +++ b/infrastructure/pe_worker.tf @@ -177,7 +177,20 @@ resource "aws_ecs_service" "shodan_service" { launch_type = "FARGATE" desired_count = 0 # Initially set to 0, plan to start it dynamically network_configuration { - subnets = aws_subnet.worker.*.id - security_groups = [aws_security_group.worker.id] + subnets = [aws_subnet.worker.id] + security_groups = [aws_security_group.worker.id] + assign_public_ip = true + } +} + +# Create the log group +resource "aws_cloudwatch_log_group" "pe_worker" { + name = var.pe_worker_ecs_log_group_name + retention_in_days = 3653 + kms_key_id = aws_kms_key.key.arn + tags = { + Project = var.project + Stage = var.stage + Owner = "Crossfeed managed resource" } } \ No newline at end of file diff --git a/infrastructure/sns.tf b/infrastructure/sns.tf index 9b1820c80..58fbe284f 100644 --- a/infrastructure/sns.tf +++ b/infrastructure/sns.tf @@ -1,5 +1,6 @@ resource "aws_sns_topic" "alarms" { - name = var.sns_topic_alarms + name = var.sns_topic_alarms + kms_master_key_id = "alias/aws/sns" tags = { Project = var.project