Merge branch '2842-cat-4-related-records' into 2842-cat-4-remaining-s…

…2-validators
raft-tech · May 1, 2024 · 1269d2c · 1269d2c
2 parents c3b9c33 + b9314ec
commit 1269d2c
Show file tree

Hide file tree

Showing 42 changed files with 698 additions and 575 deletions.
diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml
@@ -1,10 +1,9 @@
-# jobs:
   test-backend:
     executor: machine-executor
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
           command: |
@@ -47,7 +46,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-backend
       - docker-compose-up-frontend
       - install-nodejs-machine
       - disable-npm-audit

diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml
@@ -5,7 +5,7 @@
       - secrets-check
       - test-backend:
           requires:
-            - secrets-check      
+            - secrets-check
       - test-frontend:
           requires:
             - secrets-check
@@ -30,7 +30,7 @@
                 - master
                 - /^release.*/
           requires:
-            - secrets-check      
+            - secrets-check
       - test-frontend:
           filters:
             branches:

diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   deploy-dev:
     parameters:
       target_env:

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   backend-owasp-scan:
     executor: large-machine-executor
     working_directory: ~/tdp-apps
@@ -85,23 +84,23 @@
           command: |
             # Construct the project slug from the current branch name and user
             PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-
             # These environment variables are exported to Circle CI's BASH_ENV
             # by the zap-scanner.sh script for each respective app target.
             CMD_ARGS=(
               "$CIRCLE_BUILD_NUM"
-              --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0}
-              --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0}
-              --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0}
-              --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0}
-              --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0}
-              --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0}
-              --project-slug $PROJECT_SLUG
+              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
+              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
+              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
+              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
+              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
+              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+              --project-slug "$PROJECT_SLUG"
             )
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan ${CMD_ARGS[@]}"
-            # Submit a CF Task for execution that will run the necessary command
+            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+            # Submit a CF Task for execution after a 4 minute sleep to ensure all of the scan's previous state has been closed.
+            sleep 240
             cf run-task tdp-backend-<< parameters.target_env >> \
               --command "$CMD" \
               --name nightly-owasp-scan
diff --git a/.circleci/util/jobs.yml b/.circleci/util/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   make_erd:
     executor: machine-executor
     working_directory: ~/tdp_apps
@@ -9,7 +8,7 @@
           name: Run graph_models
           command: |
             cd tdrs-backend
-            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then
             docker network create external-net
             fi
             docker-compose run --rm web bash -c \

diff --git a/docs/Sprint-Review/sprint-96-summary.md b/docs/Sprint-Review/sprint-96-summary.md
@@ -0,0 +1,85 @@
+# Sprint 96 Summary
+
+03-27-2024 - 04-09-2024
+
+
+## Sprint Goal
+* Dev:
+*Support closing out (14) tickets in review & continue progress on Cat4, and walk on more parsing enhancement tickets*
+    * Category 4 Validators #2842
+    * Further validation enhancements #2757, #2807, #2818, ... 
+
+* DevOps:
+*Successful deployments across environments and pipeline stability investments*
+    * ES re-indexing automation #2870
+    * OWASP scan fix #2768 
+
+* Design: 
+*QASP review Tribal friendly names and continue on to TANF & SSP, Continue KC work*
+    * Error Categories GitHub Documentation
+    * Friendly name fixes (#2801) 
+    * Knowledge Center Content (#2847, #2846)
+        * Submission History
+        * Error reports / data file structure
+
+
+
+## Tickets
+### Completed/Merged
+* [#1441 As OFA tech lead, I need a new permissions group for OFA data analysts](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1441)
+* [#2845 [Design Deliverable] GitHub Error Categories Guide](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2845)
+* [#2871 As tech lead I need file transfer bug resolved](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2871)
+* [#2877 Move ES AWS routing docker image to RAFT hub](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2877)
+* [#2886 [bug] SSP feedback reports and files not downloadable from submission history tab](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2886)
+* [#2887 as tech lead, I need SSP Section 2 item # 18A (REC_OASDI_INSURANCE) schema def updated as not a required field](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2887) 
+* [#2673 Cat 1 Errors Audit Fixes](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2673)
+
+
+### Ready to Merge
+
+
+
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2908 As a `DIGIT Team` user, I need the ability to export parsed data from DAC to csv#2908](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2908)
+* [#2846 [Design Deliverable] Submission History Knowledge Center Explainer#2846](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2846)
+* [#2801 Friendly name cleanup](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2801)
+* [#2757 Generate preparser errors when multi-record rows are the wrong length or are missing space-filled second records](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/27570)
+
+
+
+
+
+### Closed (not merged)
+* [#2840 DIGIT Kibana Access](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2840)
+
+
+---
+
+## Moved to Next Sprint (In Progress, Blocked, Raft Review)
+### In Progress
+*	[#2509 As a data analyst I need to know when my data has been processed with or w/o errors](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2509)
+*	[#2847 Report Knowledge Center Explainer](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2847)
+* [2870 Spike: As tech lead, I need elastic re-indexing to be automated](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2870)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+*	[#2796 Add `STT_CODE` (or location information) to tanf/tribal t1-t7 and m1-m7 records in the django admin](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2796)
+*	[#2797 [Knowledge Center a11y] The purpose of a link must be descriptive](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2798)
+*	[#2814 Aggregate Cloud.gov ES instances](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2814)
+*	[#2818 [As tech lead, I need TDP to reject files that do not have an update indicator == DD]](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2818)
+*	[#2688 TANF Section 2 validation clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2688)
+* [#2749 As tech lead, I need validation checks to be consistent with FTANF validation checks](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2749)
+* [#2884 Generate preparser errors when multi-record rows are the wrong length or are missing space-filled second records - M3 and Tribal T3](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2884)
+
+### Blocked
+* [#2883 Pre-Made Reporting Dashboards on Kibana](https://github.com/raft-tech/TANF-app/issues/2883)
+* [#2870 Spike: As tech lead, I need elastic re-indexing to be automated](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2870)
+
+### Raft Review
+* [#1349 Terraform: Automate Deployment of Elasticsearch#1349](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1349)
+* [#2839 Spike - Cloud.gov Kibana Reschedule Route](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2839)
+* [#2826 As tech lead, I need some record types that currently require trailing spaces to be parsed](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2826)
+* [#2846 [Design Deliverable] Submission History Knowledge Center Explainer](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2846)
diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # pipefail is needed to correctly carry over the exit code from zap-full-scan.py
-set -o pipefail
+set -uxo pipefail
 
 TARGET=$1
 ENVIRONMENT=$2
@@ -39,7 +39,7 @@ fi
 cd "$TARGET_DIR" || exit 2
 
 
-if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then 
+if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then
   docker network create external-net
 fi
 

diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf
@@ -2,23 +2,43 @@ events {
     worker_connections 1024;
 }
 # This opens a route to clamav prod
-http{ 
-    resolver {{nameservers}} valid=10s;
+http{
+    charset utf-8;
+    log_format cloudfoundry 'NginxLog "$request" $status $body_bytes_sent';
+    access_log /dev/stdout cloudfoundry;
+
+    resolver {{nameservers}} valid=5s;
+
+    log_format compression '$remote_addr - $remote_user [$time_local] '
+                            '"proxy_host and upstream_addr": $proxy_host $upstream_addr, '
+                           ' "request": $request, '
+                           '"body_bytes_sent" : $body_bytes_sent, '
+                           '"request_body": $request_body, '
+                           '"http_x_forwarded_for": $http_x_forwarded_for, '
+                            '"host": $host, '
+                            ' "status": $status, '
+                            '"proxy_add_x_forwarded_for": $proxy_add_x_forwarded_for, '
+                           '"http_referer": $http_referer, '
+                           '"http_user_agent": $http_user_agent, '
+                           '"cookies=$http_cookie;"  "server=$server_name" "http_host=$http_host"'
+                           ' Proxy: "$proxy_host" "$upstream_addr"';
+
     server {
-        client_max_body_size 100m;
         listen {{port}};
         client_max_body_size 100m;
-        location /scan {
-            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+        location ~* ^/scan(.*)$ {
+            set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+            proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
         }
     }
+
     server {
-        client_max_body_size 100m;
         listen 9000;
         client_max_body_size 100m;
-        location /scan {
-            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+        location ~* ^/scan(.*)$ {
+            set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+            proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
         }
     }

diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
@@ -46,7 +46,7 @@ services:
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:7.4.2
+    image: docker.elastic.co/kibana/kibana-oss:7.10.2
     ports:
       - 5601:5601
     environment:
@@ -59,11 +59,10 @@ services:
       - elastic
 
   elastic:
-    image: elasticsearch:7.17.6
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
     environment:
       - discovery.type=single-node
       - logger.discovery.level=debug
-      - xpack.security.enabled=false
     ports:
       - 9200:9200
       - 9300:9300

diff --git a/tdrs-backend/elastic_setup/Dockerfile b/tdrs-backend/elastic_setup/Dockerfile
diff --git a/tdrs-backend/elastic_setup/entrypoint.sh b/tdrs-backend/elastic_setup/entrypoint.sh