Merge branch 'develop' into 2683-zap-result-cors-misconfiguration

commands.sh

@@ -227,7 +227,7 @@ tdrs-run-deploy() {
     --url https://circleci.com/api/v2/project/github/raft-tech/TANF-app/pipeline \
     --header 'Circle-Token: '$CIRCLE_CI_TOKEN \
     --header 'content-type: application/json' \
-    --data '{"parameters":{"run_dev_deployment": true, "target_env":"'$TARGET_ENV'"}, "branch":"'$BRANCH'"}'
+    --data '{"parameters":{"triggered": true, "run_dev_deployment": true, "target_env":"'$TARGET_ENV'"}, "branch":"'$BRANCH'"}'
 }
 
 # List all aliases and functions associated with tdrs

docs/Sprint-Review/sprint-83-summary.md

@@ -0,0 +1,49 @@
+
+# Sprint 83 Summary
+
+09/30/23 - 10/11/23
+
+Velocity: Dev (18)
+
+## Sprint Goal
+* Complete parsing engine development for TANF Section (04) and begin SSP (01), close out subsmission history and metadata workflows (1613/12/10).
+* UX to continue regional staff and in-app messaging research, errors audit approach, and bridge onboarding to >95% of total users
+* DevOps to investigate singluar ClamAV (2429), resolve utlity images for CircleCI and evaluate CI/CD pipeline.
+
+
+## Tickets
+### Completed/Merged
+* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612)
+* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610)
+* [#1111 TANF (04) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1111)
+
+### Ready to Merge
+* N/A
+
+### Submitted (QASP Review, OCIO Review)
+* N/A
+
+### Closed (not merged)
+* N/A
+
+## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog)
+### In Progress
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709)
+* [#2663 Investigate OWASP NightlyScan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663)
+
+### Blocked
+* N/A
+
+### Raft Review
+* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429)
+* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664)
+* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695)
+* [#2411 As system admin, I need to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411)
+
+### Demo
+* Internal:
+    * 1111, 1610, 1612
+* External:
+    * 1111, 1610, 1612
+

docs/Sprint-Review/sprint-84-summary.md

@@ -0,0 +1,61 @@
+# Sprint 84 Summary
+10/10/23 - 10/24/23
+
+Velocity: Dev (10)
+
+### Sprint Goal
+* Dev:
+    * Continue parsing engine development
+        * Complete SSP Sec (01) and SSP Sec (02)
+    * Resolve deployment blocker
+    * Coordinate w/ OFA and draft dev contingency plan for future gov shutdown
+* DevOps:
+    * 2429 - Singular Clam AV
+    * 2722 - Singular deployment workflow
+* UX: Resume regional staff research, synthesize in-app messaging research, continue supporting onboarding/utilization 
+* Prod: Find path forward on Sendgrid 
+
+## Tickets
+### Completed/Merged
+* [#2411 As system admin, I want to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411)
+* [#2429 Singular ClamAV Scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429)
+* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664)
+
+
+
+### Ready to Merge
+* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695)
+* [#2725 file input render issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2725)
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2701 FETCH_STTS Infinite Request](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2701)
+* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709)
+
+### Closed (not merged)
+* N/A
+
+## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog)
+### In Progress
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#1119 SSP Aggregate (03) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1119)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+* [#2599 Readability enhancements for error reports](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2599)
+* [#2683 ZAP result - CORS config issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2683)
+* [#2722 simplify workflows and de-bloat pipeline code](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2722)
+
+
+### Blocked
+* N/A
+
+### Raft Review
+* [#1118 SSP Closed Data (02) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1118)
+* [#1120 SSP Stratum (04) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1120)
+* [#2116 Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116)
+* [Spike - Investigate OWASP nightly scan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663)
+
+### Demo
+* N/A
+
+
+

docs/Sprint-Review/sprint-85-summary.md

@@ -0,0 +1,68 @@
+# Sprint 85 Summary
+10/24/23 - 11/07/23
+Velocity (Dev): 4
+
+## Sprint Goal
+
+* Dev:
+    * Continue parsing engine development
+        * Review all SSP Sec (01-04)
+    * #2701 - Infinite FETCH_STT bugfix
+    * #2730 Resolve deployment blocker
+    * #2683 - ZAP CORS Misconfiguration
+    * Coordinate w/ OFA and draft dev contingency plan for future gov shutdown
+        - Document any further planning (if any beyond Andrew's use of ACF laptop)
+* DevOps:
+    * 2429 - Singular Clam AV
+    * 2722 - Singular deployment workflow
+        * Scoped out of 2419
+    * #2729 - Migrations via CircleCI 
+
+---
+
+## Tickets
+### Completed/Merged
+* [#2695 As tech lead, I need some space-filled values to be allowed in TANF Section 1 data files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695)
+* [#2725 file input render issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2725)
+* [#2701 - Infinite FETCH_STT bugfix](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2701)
+
+
+
+### Ready to Merge
+* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709)
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* N/A
+
+### Closed (not merged)
+* N/A
+
+---
+
+## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog)
+### In Progress
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2599 Readability enhancements for error reports](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2599)
+* [#2722 simplify workflows and de-bloat pipeline code](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2722)
+
+
+### Blocked
+* N/A
+
+### Raft Review
+
+* [#2683 ZAP result - CORS config issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2683)
+* [#1118 SSP Closed Data (02) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1118)
+* [#1119 SSP Aggregate (03) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1119)
+* [#1120 SSP Stratum (04) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1120)
+* [#2116 Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116)
+* [Spike - Investigate OWASP nightly scan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663)
+* [Tribal TANF Active Data (01) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1114)
+
+### Current Sprint Backlog
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+
+### Demo
+* N/A

docs/Sprint-Review/sprint-86-summary.md

@@ -0,0 +1,69 @@
+# Sprint 86 Summary
+11/08/23 - 11/21/23
+
+Velocity (Dev): 6
+
+## Sprint Goal
+* Dev:
+    * Continue parsing engine development
+        * Review all SSP Sec (01-04)
+    * #2730 Resolve deployment blocker
+    * #2683 - ZAP CORS Misconfiguration
+    * Coordinate w/ OFA and draft dev contingency plan for future gov shutdown
+        - Document any further planning (if any beyond Andrew's use of ACF laptop)
+* DevOps:
+    * 2429 - Singular Clam AV
+    * 2722 - Singular deployment workflow
+        * Scoped out of 2419
+    * #2729 - Migrations via CircleCI 
+
+---
+
+## Tickets
+### Completed/Merged
+* [#2116 Container registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116)
+* [#2709 SSP Active Data (01) Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709)
+
+
+### Ready to Merge
+* [#1119 SSP Aggregate (03) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1119)
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2683 ZAP result - CORS config issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2683)
+* [#1118 SSP Closed Data (02) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1118)
+* [#1120 SSP Stratum (04) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1120)
+* [#2748 Fix parser/preparser validation of empty strings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2748)
+
+### Closed (not merged)
+* [Debug migration deployments (resolved by other deployment fix work)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2740)
+
+---
+
+## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog)
+### In Progress
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+* [#2729 - Migrations via CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2729)
+
+
+
+### Blocked
+* N/A
+
+### Raft Review
+
+
+* [#2599 Readability enhancements for error reports](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2599)
+* [#2722 simplify workflows and de-bloat pipeline code](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2722)
+* [Spike - Investigate OWASP nightly scan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663)
+* [Tribal TANF Active Data (01) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1114)
+
+### Current Sprint Backlog
+
+
+### Demo
+* N/A
+

docs/Technical-Documentation/README.md

@@ -17,6 +17,7 @@ This directory contains system and architecture documentation including diagrams
 - [data-file-downloads.md](./data-file-downloads.md) : Provides an architecture-level view of data file storage and downloading.
 - [django-admin-logging.md](./django-admin-logging.md) : Outlines sections of the Django Administrator Console and details what should be logged.
 - [jwt-key-rotation.md](./jwt-key-rotation.md) : Describes the process for rotating JWT keys in Login.gov.
+- [nexus-repo.md](./nexus-repo.md) : Setup, connection information, and how to use our Nexus Artifact Repository
 - [openid-connect.md](./openid-connect.md) : Provides an architecture-level view of the OpenID Connect prototocol.
 - [rafts-accessibility-dos-and-donts.md](./rafts-accessibility-dos-and-donts.md) : A succint list of UX guidelines for frontend accessibility.
 - [remote-development.md](./remote-development.md) : A guide on doing live remote development in Cloud.gov.

docs/Technical-Documentation/nexus-repo.md

@@ -0,0 +1,68 @@
+# Nexus Artifact Repository
+
+We are using Nexus as an artifact store in order to retain docker images and other artifacts needed for our apps and pipelines.
+
+Nexus UI can be accessed at [https://tdp-nexus.dev.raftlabs.tech/](https://tdp-nexus.dev.raftlabs.tech/)
+
+## Nexus Image Management
+
+### Host Information
+
+The VM that runs the [Sonatype Nexus Image](https://help.sonatype.com/repomanager3/product-information/download) currently resides at 172.10.4.102 on Raft's internal network. You must first be connected to the labs.goraft.tech Raft Labs VPN before SSHing to the container. Current points of contact for getting setup with the VPN are Connor Meehan and Barak Stout.
+
+### Nexus Container Setup
+
+From our virtual machine, here is how to get Nexus up and running.
+Pull and run nexus image:
+```
+docker pull sonatype/nexus3
+docker volume create --name nexus-data
+docker run -d -p 8081:8081 -p 8082:8082 -p 8083:8083 --name nexus -v nexus-data:/nexus-data sonatype/nexus3
+```
+
+wait for nexus to be running
+```
+docker logs -f nexus
+```
+
+The first time you need to log in as root, you will need the auto-generated admin password that is created upon initialization of the container.
+exec into the container and get docker admin.password:
+```
+docker exec -it nexus /bin/bash
+cat /nexus-data/admin.password
+```
+
+After logging in as root for the first time, you will be taken to a page to set a new password.
+
+## Hosted Docker Repository
+
+### Setup
+
+In order to use Nexus as a Docker repository, the DNS for the repo needs to be able to terminate https. We are currently using cloudflare to do this.
+
+When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port. 
+
+In order to allow [Docker client login and connections](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats/docker-registry/docker-authentication) you must set up the Docker Bearer Token Realm in Settings -> Security -> Realms -> and move the Docker Bearer Token Realm over to Active.
+Also, any users will need nx-repository-view-docker-#{RepoName}-(browse && read) at a minimum and (add and edit) in order to push images.
+
+We have a separate endpoint to connect specifically to the docker repository.
+[https://tdp-docker.dev.raftlabs.tech](tdp-docker.dev.raftlabs.tech)
+
+e.g. `docker login https://tdp-docker.dev.raftlabs.tech`
+
+### Pushing Images
+
+Before an image can be pushed to the nexus repository, it must be tagged for that repo:
+
+`docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}`
+
+then you can push:
+
+`docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}`
+
+### Pulling Images
+
+We have set up a proxy mirror to dockerhub that can pull and cache DockerHub images.
+Then we have created a group docker repository that can be pulled from. If the container is in our hosted repo, the group will return that container. If not, it will see if we have a cached version of that container in our proxy repo and, if not, pull that from dockerhub, cache it and allow the docker pull to happen.
+
+`docker pull https://tdp-docker-store.dev.raftlabs.tech/${ImageName}:${Version}`

scripts/deploy-backend.sh

@@ -51,6 +51,8 @@ set_cf_envs()
   "FRONTEND_BASE_URL"
   "LOGGING_LEVEL"
   "REDIS_URI"
+  "JWT_KEY"
+  "STAGING_JWT_KEY"
   )
 
   echo "Setting environment variables for $CGAPPNAME_BACKEND"
@@ -62,9 +64,13 @@ set_cf_envs()
         cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}"
         $cf_cmd
         continue
+    elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then
+        sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g')
+        cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}"
+    else
+      cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}"
     fi
-
-    cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}"
+
     echo "Setting var : $var_name"
     $cf_cmd
   done
@@ -127,7 +133,7 @@ update_backend()
 bind_backend_to_services() {
     echo "Binding services to app: $CGAPPNAME_BACKEND"
 
-    if [ "$CFAPPNAME_BACKEND" = "tdp-backend-develop" ]; then
+    if [ "$CGAPPNAME_BACKEND" = "tdp-backend-develop" ]; then
       # TODO: this is technical debt, we should either make staging mimic tanf-dev 
       #       or make unique services for all apps but we have a services limit
       #       Introducing technical debt for release 3.0.0 specifically.

tdrs-backend/apt.yml

@@ -2,7 +2,7 @@ cleancache: true
 keys:
   - https://www.postgresql.org/media/keys/ACCC4CF8.asc
 repos:
-  - deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main
+  - deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main
 packages:
   - postgresql-client-12
   - libjemalloc-dev

tdrs-backend/clamav-router/nginx.conf

@@ -6,6 +6,7 @@ http{
     server {
         client_max_body_size 100m;
         listen {{port}};
+        client_max_body_size 100m;
         location /scan {
             proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass_request_headers      on;
@@ -14,6 +15,7 @@ http{
     server {
         client_max_body_size 100m;
         listen 9000;
+        client_max_body_size 100m;
         location /scan {
             proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass_request_headers      on;