Merge branch 'master' into 2323-update-notification-with-new-messaging

cisagov · Nov 27, 2023 · 0351c02 · 0351c02
2 parents 476d650 + 84188ba
commit 0351c02
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 23 deletions.
diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe
@@ -9,6 +9,8 @@ COPY src ./src
 
 RUN apt update && apt install git zlib1g-dev
 
+RUN apt-get update && apt-get install -y jq
+
 RUN wget -c https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tar.xz && tar -Jxvf Python-3.10.11.tar.xz
 RUN cd Python-3.10.11 && ./configure && make -j4 && make altinstall
 RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.10 1
@@ -21,4 +23,6 @@ RUN apt remove dav1d && apt autoclean && apt autoremove
 # Sync the latest from cf-staging branch
 RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && pip install .
 
+COPY worker worker
+
 CMD ["./worker/pe-worker-entry.sh"]
diff --git a/backend/serverless.yml b/backend/serverless.yml
@@ -43,6 +43,9 @@ provider:
           Action:
             - ecs:RunTask
             - ecs:ListTasks
+            - ecs:DescribeTasks
+            - ecs:DescribeServices
+            - ecs:UpdateService
             - iam:PassRole
           Resource: '*'
         - Effect: Allow
@@ -65,7 +68,9 @@ provider:
         - Effect: Allow
           Action:
             - sqs:ReceiveMessage
+            - sqs:DeleteMessage
             - sqs:SendMessage
+            - sqs:GetQueueAttributes
           Resource: '*'
         - Effect: Allow
           Action:
@@ -108,7 +113,6 @@ resources:
         MaximumMessageSize: 262144 # 256 KB
         MessageRetentionPeriod: 604800 # 7 days
 
-
 functions:
   - ${file(./src/tasks/functions.yml)}
   - ${file(./src/api/functions.yml)}

diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts
@@ -29,6 +29,24 @@ export const handler: Handler = async (event) => {
     } else {
       console.log('Shodan is the only script type available right now.');
     }
+
+    // After processing each message, check if the SQS queue is empty
+    const sqsAttributes = await sqs
+      .getQueueAttributes({
+        QueueUrl: process.env.SHODAN_QUEUE_URL!,
+        AttributeNames: ['ApproximateNumberOfMessages']
+      })
+      .promise();
+
+    const approximateNumberOfMessages = parseInt(
+      sqsAttributes.Attributes?.ApproximateNumberOfMessages || '0',
+      10
+    );
+
+    // If the queue is empty, scale down to zero tasks
+    if (approximateNumberOfMessages === 0) {
+      await startFargateTask(clusterName, process.env.SHODAN_SERVICE_NAME!, 0);
+    }
   } catch (error) {
     console.error(error);
     return {
@@ -59,7 +77,7 @@ export async function startFargateTask(
       const service = serviceDescription.services[0];
 
       // Check if the desired task count is less than # provided
-      if (service.desiredCount! < desiredCountNum) {
+      if (service.desiredCount! !== desiredCountNum) {
         const updateServiceParams = {
           cluster: clusterName,
           service: serviceName,

diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh
@@ -1,7 +1,9 @@
-#!/bin/sh
+#!/bin/bash
 
 set -e
 
+echo "Starting pe-worker-entry.sh script"
+
 # Check if the SHODAN_QUEUE_URL environment variable is set
 if [ -z "$SHODAN_QUEUE_URL" ]; then
   echo "SHODAN_QUEUE_URL environment variable is not set. Exiting."

diff --git a/infrastructure/log_alarms.tf b/infrastructure/log_alarms.tf
@@ -7,7 +7,7 @@ resource "aws_cloudwatch_metric_alarm" "root_user" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -25,7 +25,7 @@ resource "aws_cloudwatch_metric_alarm" "unauthorized_api_call" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -43,7 +43,7 @@ resource "aws_cloudwatch_metric_alarm" "login_without_mfa" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -61,7 +61,7 @@ resource "aws_cloudwatch_metric_alarm" "iam_policy" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -79,7 +79,7 @@ resource "aws_cloudwatch_metric_alarm" "cloudtrail" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -97,7 +97,7 @@ resource "aws_cloudwatch_metric_alarm" "login_failure" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -115,7 +115,7 @@ resource "aws_cloudwatch_metric_alarm" "cmk_delete_disable" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -133,7 +133,7 @@ resource "aws_cloudwatch_metric_alarm" "s3_bucket_policy" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -151,7 +151,7 @@ resource "aws_cloudwatch_metric_alarm" "aws_config" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -169,7 +169,7 @@ resource "aws_cloudwatch_metric_alarm" "security_group" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -187,7 +187,7 @@ resource "aws_cloudwatch_metric_alarm" "nacl" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -205,7 +205,7 @@ resource "aws_cloudwatch_metric_alarm" "network_gateway" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -223,7 +223,7 @@ resource "aws_cloudwatch_metric_alarm" "route_table" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -241,7 +241,7 @@ resource "aws_cloudwatch_metric_alarm" "vpc" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -259,7 +259,7 @@ resource "aws_cloudwatch_metric_alarm" "ec2_shutdown" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -277,7 +277,7 @@ resource "aws_cloudwatch_metric_alarm" "db_shutdown" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project
@@ -295,7 +295,7 @@ resource "aws_cloudwatch_metric_alarm" "db_deletion" {
   evaluation_periods  = 1
   period              = 60
   threshold           = 1
-  statistic           = "SampleCount"
+  statistic           = "Sum"
 
   tags = {
     Project  = var.project

diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf
@@ -177,7 +177,20 @@ resource "aws_ecs_service" "shodan_service" {
   launch_type     = "FARGATE"
   desired_count   = 0 # Initially set to 0, plan to start it dynamically
   network_configuration {
-    subnets         = aws_subnet.worker.*.id
-    security_groups = [aws_security_group.worker.id]
+    subnets          = [aws_subnet.worker.id]
+    security_groups  = [aws_security_group.worker.id]
+    assign_public_ip = true
+  }
+}
+
+# Create the  log group
+resource "aws_cloudwatch_log_group" "pe_worker" {
+  name              = var.pe_worker_ecs_log_group_name
+  retention_in_days = 3653
+  kms_key_id        = aws_kms_key.key.arn
+  tags = {
+    Project = var.project
+    Stage   = var.stage
+    Owner   = "Crossfeed managed resource"
   }
 }
diff --git a/infrastructure/sns.tf b/infrastructure/sns.tf
@@ -1,5 +1,6 @@
 resource "aws_sns_topic" "alarms" {
-  name = var.sns_topic_alarms
+  name              = var.sns_topic_alarms
+  kms_master_key_id = "alias/aws/sns"
 
   tags = {
     Project = var.project