Merged from 1858-kibana-ofa-access, resolved conflicts

docs/Security-Compliance/boundary-diagram.md

@@ -6,7 +6,11 @@
 
 ### Data flow
 
-Users with `OFA Admin` and (STT) `Data Analyst` roles can upload data on upload data files locally into the web application which will store the files in cloud.gov AWS S3 buckets only after the files are successfully scanned for viruses via [ClamAV](../Technical-Documentation/Architecture-Decision-Record/012-antivirus-strategy.md). Developers will deploy new code through GitHub, initiating the continuous integration process through Circle CI.
+Users with `OFA Admin` and (STT) `Data Analyst` roles can upload data on upload data files locally into the web application which will store the files in cloud.gov AWS S3 buckets only after the files are successfully scanned for viruses via [ClamAV](../Technical-Documentation/Architecture-Decision-Record/012-antivirus-strategy.md). For lower environments, we use an NGINX server to function as a proxy, routing to the ClamAV-rest server in the production space. The NGINX server also functions as a gatekeeper, allowing documents for scanning to only come from backend servers, and only able to route them directly to the ClamAV-rest server.
+
+### Code Repository and CI Pipeline
+
+Developers will deploy new code through GitHub, initiating the continuous integration process through Circle CI.
 
 ### Environments/Spaces
 

docs/Sprint-Review/sprint-86-summary.md

@@ -0,0 +1,69 @@
+# Sprint 86 Summary
+11/08/23 - 11/21/23
+
+Velocity (Dev): 6
+
+## Sprint Goal
+* Dev:
+    * Continue parsing engine development
+        * Review all SSP Sec (01-04)
+    * #2730 Resolve deployment blocker
+    * #2683 - ZAP CORS Misconfiguration
+    * Coordinate w/ OFA and draft dev contingency plan for future gov shutdown
+        - Document any further planning (if any beyond Andrew's use of ACF laptop)
+* DevOps:
+    * 2429 - Singular Clam AV
+    * 2722 - Singular deployment workflow
+        * Scoped out of 2419
+    * #2729 - Migrations via CircleCI 
+
+---
+
+## Tickets
+### Completed/Merged
+* [#2116 Container registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116)
+* [#2709 SSP Active Data (01) Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709)
+
+
+### Ready to Merge
+* [#1119 SSP Aggregate (03) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1119)
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2683 ZAP result - CORS config issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2683)
+* [#1118 SSP Closed Data (02) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1118)
+* [#1120 SSP Stratum (04) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1120)
+* [#2748 Fix parser/preparser validation of empty strings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2748)
+
+### Closed (not merged)
+* [Debug migration deployments (resolved by other deployment fix work)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2740)
+
+---
+
+## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog)
+### In Progress
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+* [#2729 - Migrations via CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2729)
+
+
+
+### Blocked
+* N/A
+
+### Raft Review
+
+
+* [#2599 Readability enhancements for error reports](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2599)
+* [#2722 simplify workflows and de-bloat pipeline code](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2722)
+* [Spike - Investigate OWASP nightly scan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663)
+* [Tribal TANF Active Data (01) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1114)
+
+### Current Sprint Backlog
+
+
+### Demo
+* N/A
+

scripts/deploy-backend.sh

@@ -91,7 +91,7 @@ update_backend()
     cd tdrs-backend || exit
     cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL"
 
-    if ["$CF_SPACE" = "tanf-prod" ]; then
+    if [ "$CF_SPACE" = "tanf-prod" ]; then
       cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan"
     else
       # Add environment varilables for clamav
@@ -100,7 +100,6 @@ update_backend()
 
     if [ "$1" = "rolling" ] ; then
         set_cf_envs
-
         # Do a zero downtime deploy.  This requires enough memory for
         # two apps to exist in the org/space at one time.
         cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1
@@ -121,7 +120,7 @@ update_backend()
     # Add network policy to allow frontend to access backend
     cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080
 
-    if ["$CF_SPACE" = "tanf-prod" ]; then
+    if [ "$CF_SPACE" = "tanf-prod" ]; then
       # Add network policy to allow backend to access tanf-prod services
       cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000
     else

scripts/zap-scanner.sh

@@ -40,7 +40,7 @@ cd "$TARGET_DIR" || exit 2
 
 
 if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then 
-    docker network create external-net
+  docker network create external-net
 fi
 
 # Ensure the APP_URL is reachable from the zaproxy container
@@ -112,10 +112,6 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(14\).description='Site - FontAwesome.com' \
   -config globalexcludeurl.url_list.url\(14\).enabled=true \
 
-  -config globalexcludeurl.url_list.url\(15\).regex='^https:\/\/.*\.cloud.gov\/.*$' \
-  -config globalexcludeurl.url_list.url\(15\).description='Site - Cloud.gov' \
-  -config globalexcludeurl.url_list.url\(15\).enabled=true \
-
   -config globalexcludeurl.url_list.url\(16\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \
   -config globalexcludeurl.url_list.url\(16\).description='Site - googletagmanager.com' \
   -config globalexcludeurl.url_list.url\(16\).enabled=true \
@@ -140,7 +136,6 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(21\).description='Site - IdentitySandbox.gov' \
   -config globalexcludeurl.url_list.url\(21\).enabled=true \
   -config spider.postform=true"
-
 # How long ZAP will crawl the app with the spider process
 ZAP_SPIDER_MINS=10
 

tdrs-backend/Pipfile

@@ -58,7 +58,7 @@ django-elasticsearch-dsl = "==7.3"
 django-elasticsearch-dsl-drf = "==0.22.5"
 requests-aws4auth = "==1.1.2"
 cerberus = "==1.3.4"
-xlsxwriter = "==3.0.1"
+xlsxwriter = "==3.1.9"
 sendgrid = "==6.10.0"
 
 [requires]