Merge pull request #509 from HHS/main

v3.3.0.93 to prod
HHS · Mar 15, 2024 · c2a940f · c2a940f
2 parents cbe86f4 + caa7519
commit c2a940f
Show file tree

Hide file tree

Showing 98 changed files with 2,484 additions and 1,128 deletions.
diff --git a/.circleci/README.md b/.circleci/README.md
@@ -79,7 +79,7 @@ updated along with the secondary apps running in CF.
 
 ### Frontend/Backend
  - Before updating, make sure the current buildpacks that these apps use are supported by the latest OS. If they aren't you can update the manifest to point them to the correct buildpacks.
- - To update the apps you can either deploy each of the environments (sandbox, raft, qasp, etc) from CircleCi or you can use the `tdrs-deploy <ENVIRONMENT>` command from `commands.sh`. Assuming the buildpacks are up to date, that is all you need to do.
+ - To update the apps you can either deploy each of the environments (raft, qasp, etc) from CircleCi or you can use the `tdrs-deploy <ENVIRONMENT>` command from `commands.sh`. Assuming the buildpacks are up to date, that is all you need to do.
 
 ### Secondary apps
  - Before you can make the update, you need to ensure you have the CF plugin that allows you to do so. Download the binary for your respective OS [HERE](https://github.com/cloudfoundry/stack-auditor/releases) and follow the installation instructions [HERE](https://docs.cloudfoundry.org/adminguide/stack-auditor.html#install).

diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml
@@ -4,7 +4,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-backend
+      - docker-compose-up-with-elastic-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
           command: |
@@ -47,7 +47,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-backend
+      - docker-compose-up-with-elastic-backend
       - docker-compose-up-frontend
       - install-nodejs-machine
       - disable-npm-audit
@@ -61,7 +61,7 @@
                wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
       - run:
           name: apply the migrations
-          command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate" 
+          command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate"
       - run:
           name: Remove existing cypress test users
           command: cd tdrs-backend; docker-compose exec web python manage.py delete_cypress_users -usernames [email protected] [email protected]

diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml
@@ -79,19 +79,33 @@
       frontend-appname:
         default: tdp-frontend
         type: string
+      kibana-appname:
+        default: tdp-kibana
+        type: string
+      proxy-appname:
+        default: tdp-elastic-proxy
+        type: string
       cf-space:
         default: tanf-dev
         type: string
     steps:
       - get-app-deploy-strategy:
           appname: <<parameters.backend-appname>>
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt update
+            sudo add-apt-repository ppa:rmescandon/yq
+            sudo apt-get install yq
       - run:
           name: Deploy backend application
           command: |
             bash ./scripts/deploy-backend.sh \
               $DEPLOY_STRATEGY \
               <<parameters.frontend-appname>> \
               <<parameters.backend-appname>> \
+              <<parameters.kibana-appname>> \
+              <<parameters.proxy-appname>> \
               <<parameters.cf-space>>
 
   deploy-clamav:
@@ -115,6 +129,9 @@
       frontend-appname:
         default: tdp-frontend
         type: string
+      kibana-appname:
+        default: tdp-kibana
+        type: string
 # So the frontend knows what space its in for the banner.
 # I am unclear if the domain is a reliable metric to make this function
 # It seems like it might not be working
@@ -136,6 +153,7 @@
               $DEPLOY_STRATEGY \
               <<parameters.frontend-appname>> \
               <<parameters.backend-appname>> \
+              <<parameters.kibana-appname>> \
               <<parameters.cf-space>> \
               <<parameters.environment>>
 

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
@@ -56,6 +56,9 @@
       cf_space:
         type: string
         default: tanf-staging
+      cf_org:
+        type: string
+        default: "CF_ORG"
       target_env:
         type: enum
         enum: [ "staging", "develop", "prod" ]
@@ -64,6 +67,11 @@
       - sudo-check
       - cf-check
       - docker-compose-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf_password>>
+          cf-space: <<parameters.cf_space>>
+          cf-org: <<parameters.cf_org>>
+          cf-username: <<parameters.cf_username>>
       - run-owasp-scan:
           environment: nightly
           target: backend
@@ -72,10 +80,6 @@
           environment: nightly
           target: frontend
           target_env: <<parameters.target_env>>
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf_password>>
-          cf-space: <<parameters.cf_space>>
-          cf-username: <<parameters.cf_username>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |

diff --git a/.circleci/util/commands.yml b/.circleci/util/commands.yml
@@ -11,6 +11,12 @@
           name: Build and spin-up Django API service
           command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
 
+  docker-compose-up-with-elastic-backend:
+    steps:
+      - run:
+          name: Build and spin-up Django API service
+          command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build
+
   cf-check:
     steps:
       - run:

diff --git a/.gitconfig b/.gitconfig
@@ -7,6 +7,7 @@
 	allowed = [A-Z]+_KEY=..echo \".{S3_CREDENTIALS}\" [|] jq -r .+
 	allowed = ./tdrs-backend/.env.example:.*
 	allowed = ./tdrs-backend/docker-compose.yml:57:.*
+	allowed = ./tdrs-backend/manifest.proxy.yml:*
 	allowed = regexes.json:.*
 	allowed = ./scripts/copy-login-gov-keypair.sh:14:JWT_KEY=.*
 	allowed = scripts/deploy-backend.sh:.+:DJANGO_SECRET_KEY=..python -c .from secrets import token_urlsafe. print.token_urlsafe..*

diff --git a/docs/Sprint-Review/sprint-91-summary.md b/docs/Sprint-Review/sprint-91-summary.md
@@ -0,0 +1,62 @@
+# Sprint 91 Summary
+
+01/17/2024 - 01/30/2024
+
+Velocity (Dev): 24
+
+## Sprint Goal
+* Dev:
+    * Continue parsing engine development and begin work on enhancement tickets
+        * #2536 Cat 4 validation 
+    * #1858 Secure OFA staff access to Kibana 
+        * Unblocks #1350 when complete 
+* DevOps:
+    * #2790 - Update deployment code to support Kibana and integrate with Standing Elastic instance
+* Design: 
+    * Tie up current documentation work
+    * Continue refinement of research roadmap
+
+
+## Tickets
+### Completed/Merged
+* [#2751 Resource Card updated with latest coding instructions](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2751)
+
+### Ready to Merge
+* [#2772 Elastic bulk document creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2772)
+* [#1350 Kibana access from TDP](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1350)
+* [#1858 Spike: Secure Kibana access](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1858)
+* [#2711 Catch report month / year mismatches](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2711)
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2790 Kibana Deployment](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2790)
+* [#2681 Section 1 Validation clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2681)
+
+
+
+### Closed (not merged)
+* N/A
+
+
+---
+
+## Moved to Next Sprint (In Progress, Blocked, Raft Review)
+### In Progress
+* [#2646 - Populate data file summary case aggregates differently per section](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2646)
+* [#2820 [bug] Uncaught exception re: parsing error preventing feedback report generation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2820)
+* [#2768 Fix production OWASP scan reporting](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2768)
+* [#2799 Generate error mismatching field rpt_month_year w/ header](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2799)
+* [#2781 As a developer,  I want to have documentation on django migration best practices](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2781)
+
+
+### Blocked
+* N/A
+
+### Raft Review
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+* [#2746 As an STT, I need to know if there are issues with the DOBs reported in my data files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2746)
+* [#2813 Reduce dev environment count](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2813)
+* [#2729 As a developer, I want to move migration commands in the pipeline to CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2729)
diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/008-deployment-flow.md b/docs/Technical-Documentation/Architecture-Decision-Record/008-deployment-flow.md
@@ -33,7 +33,6 @@ Within the dev space, there is no correlation for branch to environment as these
 
 | Dev Site | Frontend URL | Backend URL | Purpose                                          |
 | -------- | -------- | -------- |--------------------------------------------------|
-| Sandbox     | https://tdp-frontend-sandbox.app.cloud.gov | https://tdp-backend-sandbox.app.cloud.gov/admin/     | Space for development in a deployed environment |
 | A11y | https://tdp-frontend-a11y.app.cloud.gov | https://tdp-backend-a11y.app.cloud.gov/admin/ | Space for accessibility testing                  |
 | QASP | https://tdp-frontend-qasp.app.cloud.gov | https://tdp-backend-qasp.app.cloud.gov/admin/ | Space for QASP review                            |
 | raft | https://tdp-frontend-raft.app.cloud.gov | https://tdp-backend-raft.app.cloud.gov/admin/ | Space for Raft review                          |
@@ -53,4 +52,4 @@ Within the dev space, there is no correlation for branch to environment as these
 
 ## Notes
 
-- As of June 2022, CircleCI supplies environment variable key-value pairs to multiple environments (e.g. vendor's CircleCI deploys applications to dev and staging environments). The values from CircleCI are expected to be unique per environment, so until [#1826](https://github.com/raft-tech/TANF-app/issues/1826) is researched and addressed, these values will need to be manually corrected in cloud.gov immediately following the execution of the execution of the [`<env>-deployment` CircleCI workflow](../../.circleci/config.yml) CircleCI workflow. This workaround applies to backend applications in the TDP staging environment.
+- As of June 2022, CircleCI supplies environment variable key-value pairs to multiple environments (e.g. vendor's CircleCI deploys applications to dev and staging environments). The values from CircleCI are expected to be unique per environment, so until [#1826](https://github.com/raft-tech/TANF-app/issues/1826) is researched and addressed, these values will need to be manually corrected in cloud.gov immediately following the execution of the execution of the [`<env>-deployment` CircleCI workflow](../../.circleci/config.yml) CircleCI workflow. This workaround applies to backend applications in the TDP staging environment.
diff --git a/docs/Technical-Documentation/Failed-Deployment-Troubleshooting.md b/docs/Technical-Documentation/Failed-Deployment-Troubleshooting.md
@@ -13,6 +13,7 @@ In preparation for production-ready infrastructure, we wanted to create a living
 + [CircleCI failures](./Failed-Deployment-Troubleshooting.md#circleci-failures)
 + [Runtime failures](./Failed-Deployment-Troubleshooting.md#compilationruntime-failure)
 + [App Connectivity issues](./Failed-Deployment-Troubleshooting.md#app-connectivity-issues)
++ [App roll-back](./Failed-Deployment-Troubleshooting.md#revision-rollback)
 
 ## CircleCI failures
 **Symptom:** I deployed new code (via merging) but the app in Cloud.gov didn't update and is still running old code.
@@ -77,4 +78,26 @@ export [email protected]
 export LOGGING_LEVEL=DEBUG
 [...]
 bash scripts/deploy-backend.sh rebuild tdp-backend-raft tanf-dev
-```
+```
+
+## Revision Rollback
+
+First we need to get list of revisions and select a stable revision id.
+```cf revisions {app-name}```
+
+Then use the last successful guid, we can populate this reversion command:
+```
+cf curl v3/deployments \        
+-X POST \
+-d '{
+  "revision": {
+    "guid": "{last stable guid from list above}"
+  },
+  "relationships": {
+    "app": {
+      "data": {
+        "guid": "{current app guid}"
+      }
+    }
+  }
+}'```
diff --git a/docs/Technical-Documentation/README.md b/docs/Technical-Documentation/README.md
@@ -19,7 +19,7 @@ This directory contains system and architecture documentation including diagrams
 - [jwt-key-rotation.md](./jwt-key-rotation.md) : Describes the process for rotating JWT keys in Login.gov.
 - [nexus-repo.md](./nexus-repo.md) : Setup, connection information, and how to use our Nexus Artifact Repository
 - [openid-connect.md](./openid-connect.md) : Provides an architecture-level view of the OpenID Connect prototocol.
-- [rafts-accessibility-dos-and-donts.md](./rafts-accessibility-dos-and-donts.md) : A succint list of UX guidelines for frontend accessibility.
+- [accessibility-guide.md](./accessibility-guide.md) : A guide on getting started with accessibility testing tools and TDP-relevant resources.
 - [remote-development.md](./remote-development.md) : A guide on doing live remote development in Cloud.gov.
 - [unit-tests.md](./unit-tests.md) : Outlines our unit testing frameworks and how to run these manually.
 - [user_role_management.md](./user_role_management.md) : Provides an overview of our user management in Django Administrator Console.
diff --git a/docs/Technical-Documentation/TDP-environments-README.md b/docs/Technical-Documentation/TDP-environments-README.md
@@ -4,7 +4,6 @@
 
 | Dev Site | Frontend URL | Backend URL | Purpose |
 | -------- | -------- | -------- | -------- |
-| Sandbox     | https://tdp-frontend-sandbox.app.cloud.gov | https://tdp-frontend-sandbox.app.cloud.gov/admin/     | Space for devs to test in a deployed environment |
 | A11y | https://tdp-frontend-a11y.app.cloud.gov | https://tdp-frontend-a11y.app.cloud.gov/admin/ | Space for accessibility testing |
 | QASP | https://tdp-frontend-qasp.app.cloud.gov | https://tdp-frontend-qasp.app.cloud.gov/admin/ | Space for QASP review |
 | raft | https://tdp-frontend-raft.app.cloud.gov | https://tdp-frontend-raft.app.cloud.gov/admin/ | Space for raft review |