Merge branch 'master' into 2367-move-documentation-site-to-aws

cisagov · Dec 7, 2023 · d083943 · d083943
2 parents 9feb4df + 531f2f9
commit d083943
Show file tree

Hide file tree

Showing 61 changed files with 6,072 additions and 453 deletions.
diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe
@@ -9,6 +9,8 @@ COPY src ./src
 
 RUN apt update && apt install git zlib1g-dev
 
+RUN apt-get update && apt-get install -y jq
+
 RUN wget -c https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tar.xz && tar -Jxvf Python-3.10.11.tar.xz
 RUN cd Python-3.10.11 && ./configure && make -j4 && make altinstall
 RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.10 1
@@ -17,8 +19,25 @@ RUN pip3.10 install --upgrade pip
 
 RUN apt remove dav1d && apt autoclean && apt autoremove
 
+# Install AWS CLI
+RUN curl --insecure "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip
+RUN ./aws/install
+
 # Install pe-source module
 # Sync the latest from cf-staging branch
-RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && pip install .
+RUN git clone -b AL-staging-SQS https://github.com/cisagov/pe-reports.git && \
+    cd pe-reports && \
+    git checkout 6dcd017551ba14022e110c073e1bdbc804c795f8 && \
+    pip install .
+
+RUN python -m spacy download en_core_web_lg
+
+# Create database.ini 
+RUN echo "[database]" > database.ini \ 
+    && echo "user=$(cat db_user.txt)" >> database.ini \ 
+    && echo "password=$(cat db_password.txt)" >> database.ini
+
+COPY worker worker
 
-CMD ["./worker/pe-worker-entry.sh"]
+CMD ["./worker/generate_config.sh", "./worker/pe-worker-entry.sh"]
diff --git a/backend/env.yml b/backend/env.yml
@@ -42,7 +42,6 @@ staging:
   PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL}
   REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports
   CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-staging-cloudwatch
-  SQS_QUEUE_URL: { Ref: WorkerQueue }
   STAGE: staging
   PE_CLUSTER_NAME: pe-staging-worker
   SHODAN_QUEUE_URL: ${ssm:/crossfeed/staging/SHODAN_QUEUE_URL}
@@ -83,7 +82,6 @@ prod:
   PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL}
   REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports
   CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-prod-cloudwatch
-  SQS_QUEUE_URL: { Ref: WorkerQueue }
   STAGE: prod
   PE_CLUSTER_NAME: pe-prod-worker
   SHODAN_QUEUE_URL: ${ssm:/crossfeed/prod/SHODAN_QUEUE_URL}