Merge branch 'master' into 2246-splunk

backend/env.yml

@@ -42,6 +42,7 @@ staging:
   PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL}
   REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports
   CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-staging-cloudwatch
+  SQS_QUEUE_URL: { Ref: WorkerQueue }
 
 prod:
   DB_DIALECT: 'postgres'
@@ -78,6 +79,7 @@ prod:
   PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL}
   REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports
   CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-prod-cloudwatch
+  SQS_QUEUE_URL: { Ref: WorkerQueue }
 
 dev-vpc:
   securityGroupIds:

backend/serverless.yml

@@ -56,6 +56,17 @@ provider:
             - s3:GetObjectAcl
             - s3:PutObject
             - s3:PutObjectAcl
+            - s3:PutBucketAcl
+            - s3:GetBucketAcl
+          Resource: '*'
+        - Effect: Allow
+          Action:
+            - sts:AssumeRole
+          Resource: '*'
+        - Effect: Allow
+          Action:
+            - sqs:ReceiveMessage
+            - sqs:SendMessage
           Resource: '*'
         - Effect: Allow
           Action:
@@ -69,6 +80,24 @@ provider:
             - logs:StartLiveTail
             - logs:StopLiveTail
           Resource: '*'
+        - Effect: Allow
+          Action:
+            - ssm:DescribeParameters
+            - ssm:GetParameter
+            - ssm:GetParameters
+            - ssm:GetParametersByPath
+            - ssm:PutParameter
+          Resource: '*'
+
+resources:
+  Resources:
+    WorkerQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: ${self:provider.stage}-worker-queue
+        VisibilityTimeout: 300 # Should match or exceed function timeout
+        MaximumMessageSize: 262144 # 256 KB
+        MessageRetentionPeriod: 604800 # 7 days
 
 functions:
   - ${file(./src/tasks/functions.yml)}

backend/src/tasks/cloudwatchToS3.ts

@@ -34,8 +34,9 @@ export const handler = async () => {
 
   while (true) {
     const response = await logs.send(new DescribeLogGroupsCommand(extra_args));
+    console.log(`response: ${JSON.stringify(response)}`);
     log_groups = log_groups.concat(response.logGroups!);
-
+    console.log(`log_groups: ${JSON.stringify(log_groups)}`);
     if (!response.nextToken) {
       break;
     }
@@ -46,16 +47,23 @@ export const handler = async () => {
     const command = new ListTagsForResourceCommand({
       resourceArn: `arn:aws:logs:${region}:${accountId}:log-group:${log_group.logGroupName}`
     });
+    console.log(`Processing log group: ${log_group.logGroupName}`);
+    console.log(`command: ${JSON.stringify(command)}`);
     const response = await logs.send(command);
+    console.log(`log group response: ${JSON.stringify(response)}`);
     const log_group_tags = response.tags || {};
 
     if (log_group_tags.ExportToS3 === 'true') {
       log_groups_to_export.push(log_group.logGroupName!);
     }
+    console.log(
+      `log_groups_to_export: ${JSON.stringify(log_groups_to_export)}`
+    );
     await delay(10 * 1000); // prevents LimitExceededException (AWS allows only one export task at a time)
   }
 
   for (const log_group_name of log_groups_to_export) {
+    console.log('Processing log group: ' + log_group_name);
     const ssm_parameter_name = (
       '/log-exporter-last-export/' + log_group_name
     ).replace('//', '/');
@@ -70,6 +78,7 @@ export const handler = async () => {
       if (error.name !== 'ParameterNotFound') {
         console.error('Error fetching SSM parameter: ' + error.message);
       }
+      console.error(`error: ${error.message}`);
     }
 
     const export_to_time = Math.round(Date.now());

backend/src/tasks/functions.yml

@@ -29,6 +29,19 @@ bastion:
 makeGlobalAdmin:
   handler: src/tasks/makeGlobalAdmin.handler
 
+scanExecution:
+  handler: src/tasks/scanExecution.handler
+  timeout: 300 # 5 minutes
+  environment:
+    SQS_QUEUE_NAME: ${self:provider.stage}-worker-queue
+  events:
+    - sqs:
+        arn:
+          Fn::GetAtt:
+            - WorkerQueue
+            - Arn
+        batchSize: 5 # Number of messages the lambda can continue to process while a Fargate is still running
+
 updateScanTaskStatus:
   handler: src/tasks/updateScanTaskStatus.handler
   events:

backend/src/tasks/scanExecution.ts

@@ -0,0 +1,57 @@
+import { Handler, SQSRecord } from 'aws-lambda';
+import * as AWS from 'aws-sdk';
+
+const ecs = new AWS.ECS();
+const sqs = new AWS.SQS();
+
+export const handler: Handler = async (event) => {
+  try {
+    // Get the SQS record and message body
+    const sqsRecord: SQSRecord = event.Records[0];
+    const body: string = sqsRecord.body;
+
+    console.log(body);
+
+    let commandOptions;
+    if (body === 'SHODAN') {
+      commandOptions = './worker/shodan.sh';
+    } else {
+      commandOptions = body;
+    }
+    // Run command in queue message in Fargate
+    const params: AWS.ECS.RunTaskRequest = {
+      cluster: process.env.FARGATE_CLUSTER_NAME!,
+      taskDefinition: process.env.FARGATE_TASK_DEFINITION_NAME!,
+      launchType: 'FARGATE',
+      networkConfiguration: {
+        awsvpcConfiguration: {
+          assignPublicIp: 'ENABLED',
+          securityGroups: [process.env.FARGATE_SG_ID!],
+          subnets: [process.env.FARGATE_SUBNET_ID!]
+        }
+      },
+      platformVersion: '1.4.0',
+      overrides: {
+        containerOverrides: [
+          {
+            name: 'main', // from task definition
+            command: [commandOptions] // Pass the command options as an array
+          }
+        ]
+      }
+    };
+    const data = await ecs.runTask(params).promise();
+    console.log('Fargate task started:', data);
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify('Fargate task started and message sent to SQS queue')
+    };
+  } catch (error) {
+    console.error('Error starting Fargate task:', error);
+    return {
+      statusCode: 500,
+      body: JSON.stringify('Error starting Fargate task')
+    };
+  }
+};

backend/src/tasks/test/scanExecution.test.ts

@@ -0,0 +1,38 @@
+import { handler } from '../scanExecution';
+import { SQSRecord } from 'aws-lambda';
+
+// Mock the AWS SDK methods using aws-sdk-mock
+jest.mock('aws-sdk', () => {
+  return {
+    ECS: jest.fn(() => ({
+      runTask: jest.fn().mockReturnThis(),
+      promise: jest.fn()
+    })),
+    SQS: jest.fn(() => ({
+      sendMessage: jest.fn().mockReturnThis(),
+      promise: jest.fn()
+    }))
+  };
+});
+
+describe('Scan Execution', () => {
+  process.env.SQS_QUEUE_URL = 'YOUR_SQS_QUEUE_URL';
+  it('should handle the event', async () => {
+    const event = {
+      Records: [
+        {
+          body: 'test command',
+          eventSourceARN: 'SQSQueueARN'
+        } as SQSRecord
+      ]
+    };
+
+    const result = await handler(event, {} as any, () => void 0);
+
+    // Add your assertions here
+    expect(result.statusCode).toEqual(200);
+    expect(result.body).toContain(
+      'Fargate task started and message sent to SQS queue'
+    );
+  });
+});

backend/worker/shodan.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+cd /app/pe-reports
+
+echo "Starting Shodan"
+
+pe-source shodan --orgs=DHS --soc_med_included
+
+echo "Done"

infrastructure/cloudtrail_bucket_policy.tpl

@@ -2,7 +2,7 @@
   "Version": "2012-10-17",
   "Statement": [
     {
-      "Sid": "AWSCloudTrailAclCheck20150319",
+      "Sid": "AWSCloudTrailAclCheck20121017",
       "Effect": "Allow",
       "Principal": {
         "Service": "cloudtrail.amazonaws.com"
@@ -11,18 +11,36 @@
       "Resource": ["arn:aws:s3:::${bucketName}"]
     },
     {
-      "Sid": "AWSCloudTrailWrite20150319",
+      "Sid": "AWSCloudTrailWrite20121017",
       "Effect": "Allow",
       "Principal": {
         "Service": "cloudtrail.amazonaws.com"
       },
       "Action": ["s3:PutObject"],
-      "Resource": ["arn:aws:s3:::${bucketName}/AWSLogs/${accountId}/*"],
+      "Resource": [
+        "arn:aws:s3:::${bucketName}/AWSLogs/${accountId}",
+        "arn:aws:s3:::${bucketName}/AWSLogs/${accountId}/*"
+      ],
       "Condition": {
         "StringEquals": {
           "s3:x-amz-acl": "bucket-owner-full-control"
         }
       }
+    },
+    {
+      "Sid": "RequireSSLRequests",
+      "Action": "s3:*",
+      "Effect": "Deny",
+      "Principal": "*",
+      "Resource": [
+        "arn:aws:s3:::${bucketName}",
+        "arn:aws:s3:::${bucketName}/*"
+      ],
+      "Condition": {
+        "Bool": {
+          "aws:SecureTransport": "false"
+        }
+      }
     }
   ]
 }

infrastructure/cloudwatch.tf

@@ -23,27 +23,45 @@ resource "aws_s3_bucket_policy" "cloudwatch_bucket" {
     "Version" : "2012-10-17",
     "Statement" : [
       {
-        "Sid" : "Allow Cloudwatch to check bucket permissions",
+        "Sid" : "AWSLogDeliveryGetBucketACL",
         "Effect" : "Allow",
         "Principal" : {
           "Service" : "logs.amazonaws.com"
         },
         "Action" : "s3:GetBucketAcl",
-        "Resource" : "arn:aws:s3:::${var.cloudwatch_bucket_name}"
+        "Resource" : aws_s3_bucket.cloudwatch_bucket.arn
       },
       {
-        "Sid" : "Allow Cloudwatch to write to bucket",
+        "Sid" : "AWSLogDeliveryWrite",
         "Effect" : "Allow",
         "Principal" : {
           "Service" : "logs.amazonaws.com"
         },
         "Action" : "s3:PutObject",
-        "Resource" : "arn:aws:s3:::${var.cloudwatch_bucket_name}/*",
+        "Resource" : [
+          aws_s3_bucket.cloudwatch_bucket.arn,
+          "${aws_s3_bucket.cloudwatch_bucket.arn}/*"
+        ],
         "Condition" : {
           "StringEquals" : {
             "s3:x-amz-acl" : "bucket-owner-full-control"
           }
         }
+      },
+      {
+        "Sid" : "RequireSSLRequests",
+        "Action" : "s3:*",
+        "Effect" : "Deny",
+        "Principal" : "*",
+        "Resource" : [
+          aws_s3_bucket.cloudwatch_bucket.arn,
+          "${aws_s3_bucket.cloudwatch_bucket.arn}/*"
+        ],
+        "Condition" : {
+          "Bool" : {
+            "aws:SecureTransport" : "false"
+          }
+        }
       }
     ]
   })