From e6455b7fa9a12303e8622d5f4784415183a8f0e4 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 7 Nov 2023 11:03:16 -0500 Subject: [PATCH 1/4] Add network_configuration to aws_ecs_service --- backend/src/tasks/functions.yml | 2 +- infrastructure/pe_worker.tf | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index 8bc1a46f0..ab000f094 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -33,7 +33,7 @@ scanExecution: handler: src/tasks/scanExecution.handler timeout: 300 # 5 minutes environment: - SQS_QUEUE_NAME: ${self:provider.stage}-worker-queue + SQS_QUEUE_NAME: ${self:provider.stage}-worker-control-queue events: - sqs: arn: diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf index 743ded08d..dec81237c 100644 --- a/infrastructure/pe_worker.tf +++ b/infrastructure/pe_worker.tf @@ -176,4 +176,8 @@ resource "aws_ecs_service" "shodan_service" { task_definition = aws_ecs_task_definition.pe_worker.arn launch_type = "FARGATE" desired_count = 0 # Initially set to 0, plan to start it dynamically + network_configuration { + subnets = aws_subnet.worker.id + security_groups = [aws_security_group.worker.id] + } } \ No newline at end of file From 7f8704bbc2a9f86d3e9f6ba0c70e32cb35e93d1a Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 7 Nov 2023 11:14:44 -0500 Subject: [PATCH 2/4] Fix subnets assignment --- infrastructure/pe_worker.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf index dec81237c..cf039c9fd 100644 --- a/infrastructure/pe_worker.tf +++ b/infrastructure/pe_worker.tf @@ -177,7 +177,7 @@ resource "aws_ecs_service" "shodan_service" { launch_type = "FARGATE" desired_count = 0 # Initially set to 0, plan to start it dynamically network_configuration { - subnets = aws_subnet.worker.id + subnets = aws_subnet.worker.*.id security_groups = [aws_security_group.worker.id] } } \ No newline at end of file From 95bfb3303ccc79fc687aabd9643a0f012fe18708 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 7 Nov 2023 12:50:49 -0500 Subject: [PATCH 3/4] Remove reference to old WorkerQueue in env.yml --- backend/env.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/env.yml b/backend/env.yml index 115246cef..92a7f6013 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -42,7 +42,6 @@ staging: PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-staging-cloudwatch - SQS_QUEUE_URL: { Ref: WorkerQueue } STAGE: staging PE_CLUSTER_NAME: pe-staging-worker SHODAN_QUEUE_URL: ${ssm:/crossfeed/staging/SHODAN_QUEUE_URL} @@ -83,7 +82,6 @@ prod: PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-prod-cloudwatch - SQS_QUEUE_URL: { Ref: WorkerQueue } STAGE: prod PE_CLUSTER_NAME: pe-prod-worker SHODAN_QUEUE_URL: ${ssm:/crossfeed/prod/SHODAN_QUEUE_URL} From 6a6edfd870ea4efd18804c75bd8364b658ff430b Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Nov 2023 11:29:34 -0500 Subject: [PATCH 4/4] Add generate_config and update Dockerfile --- backend/Dockerfile.pe | 21 +++++++++++++-- backend/worker/generate_config.sh | 44 +++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) create mode 100755 backend/worker/generate_config.sh diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index c3956d29c..63384b852 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -17,8 +17,25 @@ RUN pip3.10 install --upgrade pip RUN apt remove dav1d && apt autoclean && apt autoremove +# Install AWS CLI +RUN curl --insecure "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +RUN unzip awscliv2.zip +RUN ./aws/install + # Install pe-source module # Sync the latest from cf-staging branch -RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && pip install . +RUN git clone -b AL-staging-SQS https://github.com/cisagov/pe-reports.git && \ + cd pe-reports && \ + git checkout 6dcd017551ba14022e110c073e1bdbc804c795f8 && \ + pip install . + +RUN python -m spacy download en_core_web_lg + +# Create database.ini +RUN echo "[database]" > database.ini \ + && echo "user=$(cat db_user.txt)" >> database.ini \ + && echo "password=$(cat db_password.txt)" >> database.ini + +COPY worker worker -CMD ["./worker/pe-worker-entry.sh"] +CMD ["./worker/generate_config.sh", "./worker/pe-worker-entry.sh"] diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh new file mode 100755 index 000000000..3791d5f24 --- /dev/null +++ b/backend/worker/generate_config.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Generate database.ini +cat < pe-reports/src/pe_reports/data/database.ini +[postgres] +host=$(DB_HOST) +database=$(PE_DB_NAME) +user=$(PE_DB_USERNAME) +password=$(PE_DB_PASSWORD) +port=5432 + +[shodan] +key1=$(PE_SHODAN_API_KEYS) + +[staging] +[cyhy_mongo] +[sixgill] +[whoisxml] +key= +[intelx] +[dnsmonitor] +[pe_db_password_key] +[blocklist] +[dehashed] +[dnstwist] +[hibp] +[API_Client_ID] +[API_Client_secret] +[API_WHOIS] +[pe_api] + +EOF + +# Find the path to the pe_reports package in site-packages +pe_reports_path=$(pip show pe-reports | grep -E '^Location:' | awk '{print $2}') + +# Ensure pe_reports_path ends with /pe_reports +pe_reports_path="${pe_reports_path%/pe-reports}/pe_reports" + + +# Copy database.ini to the module's installation directory +cp /app/pe-reports/src/pe_reports/data/database.ini "${pe_reports_path}/data/" + +exec "$@" \ No newline at end of file