Merge branch 'develop' into 2842-cat-4-related-records

.circleci/build-and-test/jobs.yml

@@ -1,10 +1,9 @@
-# jobs:
   test-backend:
     executor: machine-executor
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
           command: |
@@ -47,7 +46,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-backend
       - docker-compose-up-frontend
       - install-nodejs-machine
       - disable-npm-audit

.circleci/build-and-test/workflows.yml

@@ -5,7 +5,7 @@
       - secrets-check
       - test-backend:
           requires:
-            - secrets-check      
+            - secrets-check
       - test-frontend:
           requires:
             - secrets-check
@@ -30,7 +30,7 @@
                 - master
                 - /^release.*/
           requires:
-            - secrets-check      
+            - secrets-check
       - test-frontend:
           filters:
             branches:

.circleci/deployment/jobs.yml

@@ -1,4 +1,3 @@
-# jobs:
   deploy-dev:
     parameters:
       target_env:

.circleci/owasp/jobs.yml

@@ -1,4 +1,3 @@
-# jobs:
   backend-owasp-scan:
     executor: large-machine-executor
     working_directory: ~/tdp-apps
@@ -85,23 +84,23 @@
           command: |
             # Construct the project slug from the current branch name and user
             PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-
             # These environment variables are exported to Circle CI's BASH_ENV
             # by the zap-scanner.sh script for each respective app target.
             CMD_ARGS=(
               "$CIRCLE_BUILD_NUM"
-              --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0}
-              --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0}
-              --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0}
-              --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0}
-              --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0}
-              --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0}
-              --project-slug $PROJECT_SLUG
+              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
+              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
+              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
+              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
+              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
+              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+              --project-slug "$PROJECT_SLUG"
             )
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan ${CMD_ARGS[@]}"
-            # Submit a CF Task for execution that will run the necessary command
+            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+            # Submit a CF Task for execution after a 4 minute sleep to ensure all of the scan's previous state has been closed.
+            sleep 240
             cf run-task tdp-backend-<< parameters.target_env >> \
               --command "$CMD" \
               --name nightly-owasp-scan

.circleci/util/jobs.yml

@@ -1,4 +1,3 @@
-# jobs:
   make_erd:
     executor: machine-executor
     working_directory: ~/tdp_apps
@@ -9,7 +8,7 @@
           name: Run graph_models
           command: |
             cd tdrs-backend
-            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then
             docker network create external-net
             fi
             docker-compose run --rm web bash -c \

scripts/zap-scanner.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # pipefail is needed to correctly carry over the exit code from zap-full-scan.py
-set -o pipefail
+set -uxo pipefail
 
 TARGET=$1
 ENVIRONMENT=$2
@@ -39,7 +39,7 @@ fi
 cd "$TARGET_DIR" || exit 2
 
 
-if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then 
+if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then
   docker network create external-net
 fi
 

tdrs-backend/clamav-router/nginx.conf

@@ -2,23 +2,43 @@ events {
     worker_connections 1024;
 }
 # This opens a route to clamav prod
-http{ 
-    resolver {{nameservers}} valid=10s;
+http{
+    charset utf-8;
+    log_format cloudfoundry 'NginxLog "$request" $status $body_bytes_sent';
+    access_log /dev/stdout cloudfoundry;
+
+    resolver {{nameservers}} valid=5s;
+
+    log_format compression '$remote_addr - $remote_user [$time_local] '
+                            '"proxy_host and upstream_addr": $proxy_host $upstream_addr, '
+                           ' "request": $request, '
+                           '"body_bytes_sent" : $body_bytes_sent, '
+                           '"request_body": $request_body, '
+                           '"http_x_forwarded_for": $http_x_forwarded_for, '
+                            '"host": $host, '
+                            ' "status": $status, '
+                            '"proxy_add_x_forwarded_for": $proxy_add_x_forwarded_for, '
+                           '"http_referer": $http_referer, '
+                           '"http_user_agent": $http_user_agent, '
+                           '"cookies=$http_cookie;"  "server=$server_name" "http_host=$http_host"'
+                           ' Proxy: "$proxy_host" "$upstream_addr"';
+
     server {
-        client_max_body_size 100m;
         listen {{port}};
         client_max_body_size 100m;
-        location /scan {
-            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+        location ~* ^/scan(.*)$ {
+            set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+            proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
         }
     }
+
     server {
-        client_max_body_size 100m;
         listen 9000;
         client_max_body_size 100m;
-        location /scan {
-            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+        location ~* ^/scan(.*)$ {
+            set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+            proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
         }
     }

tdrs-backend/docker-compose.yml

@@ -46,7 +46,7 @@ services:
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:7.4.2
+    image: docker.elastic.co/kibana/kibana-oss:7.10.2
     ports:
       - 5601:5601
     environment:
@@ -59,11 +59,10 @@ services:
       - elastic
 
   elastic:
-    image: elasticsearch:7.17.6
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
     environment:
       - discovery.type=single-node
       - logger.discovery.level=debug
-      - xpack.security.enabled=false
     ports:
       - 9200:9200
       - 9300:9300