Merge branch 'develop' into feat/2768-owasp-scan-new

.circleci/README.md

@@ -79,7 +79,7 @@ updated along with the secondary apps running in CF.
 
 ### Frontend/Backend
  - Before updating, make sure the current buildpacks that these apps use are supported by the latest OS. If they aren't you can update the manifest to point them to the correct buildpacks.
- - To update the apps you can either deploy each of the environments (sandbox, raft, qasp, etc) from CircleCi or you can use the `tdrs-deploy <ENVIRONMENT>` command from `commands.sh`. Assuming the buildpacks are up to date, that is all you need to do.
+ - To update the apps you can either deploy each of the environments (raft, qasp, etc) from CircleCi or you can use the `tdrs-deploy <ENVIRONMENT>` command from `commands.sh`. Assuming the buildpacks are up to date, that is all you need to do.
 
 ### Secondary apps
  - Before you can make the update, you need to ensure you have the CF plugin that allows you to do so. Download the binary for your respective OS [HERE](https://github.com/cloudfoundry/stack-auditor/releases) and follow the installation instructions [HERE](https://docs.cloudfoundry.org/adminguide/stack-auditor.html#install).

.circleci/base_config.yml

@@ -13,11 +13,11 @@ executors:
   machine-executor:
     machine:
       docker_layer_caching: false
-      image: ubuntu-2204:2022.10.1
+      image: ubuntu-2204:2024.01.1
   large-machine-executor:
     machine:
       docker_layer_caching: false
-      image: ubuntu-2204:2022.10.1
+      image: ubuntu-2204:2024.01.1
     resource_class: large
 
 parameters:

.circleci/build-and-test/jobs.yml

@@ -4,7 +4,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-backend
+      - docker-compose-up-with-elastic-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
           command: |
@@ -31,7 +31,7 @@
           command: cd tdrs-frontend; npm run lint
       - run:
           name: Run Pa11y Accessibility Tests
-          command: cd tdrs-frontend; mkdir pa11y-screenshots/; npm run test:accessibility
+          command: cd tdrs-frontend; mkdir pa11y-screenshots/; export NODE_OPTIONS=--openssl-legacy-provider; npm run test:accessibility
       - run:
           name: Run Jest Unit Tests
           command: cd tdrs-frontend; npm run test:ci
@@ -47,7 +47,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-backend
+      - docker-compose-up-with-elastic-backend
       - docker-compose-up-frontend
       - install-nodejs-machine
       - disable-npm-audit
@@ -61,7 +61,7 @@
                wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
       - run:
           name: apply the migrations
-          command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate" 
+          command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate"
       - run:
           name: Remove existing cypress test users
           command: cd tdrs-backend; docker-compose exec web python manage.py delete_cypress_users -usernames [email protected] [email protected]

.circleci/deployment/commands.yml

@@ -79,19 +79,37 @@
       frontend-appname:
         default: tdp-frontend
         type: string
+      kibana-appname:
+        default: tdp-kibana
+        type: string
+      proxy-appname:
+        default: tdp-elastic-proxy
+        type: string
       cf-space:
         default: tanf-dev
         type: string
     steps:
       - get-app-deploy-strategy:
           appname: <<parameters.backend-appname>>
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt update
+            sudo apt-get install -y wget
+            sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_amd64 && chmod +x /usr/local/bin/yq
+      - run:
+          name: Apply database migrations
+          command: |
+            bash ./scripts/apply-remote-migrations.sh <<parameters.backend-appname>>
       - run:
           name: Deploy backend application
           command: |
             bash ./scripts/deploy-backend.sh \
               $DEPLOY_STRATEGY \
               <<parameters.frontend-appname>> \
               <<parameters.backend-appname>> \
+              <<parameters.kibana-appname>> \
+              <<parameters.proxy-appname>> \
               <<parameters.cf-space>>
 
   deploy-clamav:
@@ -115,6 +133,9 @@
       frontend-appname:
         default: tdp-frontend
         type: string
+      kibana-appname:
+        default: tdp-kibana
+        type: string
 # So the frontend knows what space its in for the banner.
 # I am unclear if the domain is a reliable metric to make this function
 # It seems like it might not be working
@@ -136,6 +157,7 @@
               $DEPLOY_STRATEGY \
               <<parameters.frontend-appname>> \
               <<parameters.backend-appname>> \
+              <<parameters.kibana-appname>> \
               <<parameters.cf-space>> \
               <<parameters.environment>>
 
@@ -253,7 +275,7 @@
           cf-space: <<parameters.cf-space>>
       - run:
           name: Ensure s3 versioning key exists
-          # use cf to check if a service key exists for tdp-datafiles-dev(or prod) 
+          # use cf to check if a service key exists for tdp-datafiles-dev(or prod)
           # exit 0 if it does, create it if it doesn't
           command: |
             chmod +x ./scripts/create_s3_versioning_key.sh

.circleci/owasp/jobs.yml

@@ -56,6 +56,9 @@
       cf_space:
         type: string
         default: tanf-staging
+      cf_org:
+        type: string
+        default: "CF_ORG"
       target_env:
         type: enum
         enum: [ "staging", "develop", "prod" ]
@@ -64,6 +67,11 @@
       - sudo-check
       - cf-check
       - docker-compose-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf_password>>
+          cf-space: <<parameters.cf_space>>
+          cf-org: <<parameters.cf_org>>
+          cf-username: <<parameters.cf_username>>
       - run-owasp-scan:
           environment: nightly
           target: backend
@@ -72,10 +80,6 @@
           environment: nightly
           target: frontend
           target_env: <<parameters.target_env>>
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf_password>>
-          cf-space: <<parameters.cf_space>>
-          cf-username: <<parameters.cf_username>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |

.circleci/util/commands.yml

@@ -11,6 +11,12 @@
           name: Build and spin-up Django API service
           command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
 
+  docker-compose-up-with-elastic-backend:
+    steps:
+      - run:
+          name: Build and spin-up Django API service
+          command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build
+
   cf-check:
     steps:
       - run:

.gitattributes

@@ -44,4 +44,4 @@
 # Annoying product-updates zip files
 product-updates/knowledge-center/FTANF_2009.zip binary eol=lf
 product-updates/knowledge-center/SSPMOE_2009.zip binary eol=lf
-product-updates/knowledge-center/ftanf.zip binary eol=lf
+product-updates/knowledge-center/ftanf.zip binary eol=lf

.gitconfig

@@ -7,6 +7,7 @@
 	allowed = [A-Z]+_KEY=..echo \".{S3_CREDENTIALS}\" [|] jq -r .+
 	allowed = ./tdrs-backend/.env.example:.*
 	allowed = ./tdrs-backend/docker-compose.yml:57:.*
+	allowed = ./tdrs-backend/manifest.proxy.yml:*
 	allowed = regexes.json:.*
 	allowed = ./scripts/copy-login-gov-keypair.sh:14:JWT_KEY=.*
 	allowed = scripts/deploy-backend.sh:.+:DJANGO_SECRET_KEY=..python -c .from secrets import token_urlsafe. print.token_urlsafe..*

.gitignore

@@ -21,6 +21,7 @@ tdrs-backend/coverage.xml
 tdrs-backend/htmlcov/*
 tdrs-backend/.env
 tdrs-backend/.env.production
+tdrs-backend/.env.ci
 tdrs-backend/ADS*
 tdrs-backend/temp_key_file
 tdrs-backend/test

docs/Sprint-Review/sprint-91-summary.md

@@ -0,0 +1,62 @@
+# Sprint 91 Summary
+
+01/17/2024 - 01/30/2024
+
+Velocity (Dev): 24
+
+## Sprint Goal
+* Dev:
+    * Continue parsing engine development and begin work on enhancement tickets
+        * #2536 Cat 4 validation 
+    * #1858 Secure OFA staff access to Kibana 
+        * Unblocks #1350 when complete 
+* DevOps:
+    * #2790 - Update deployment code to support Kibana and integrate with Standing Elastic instance
+* Design: 
+    * Tie up current documentation work
+    * Continue refinement of research roadmap
+
+
+## Tickets
+### Completed/Merged
+* [#2751 Resource Card updated with latest coding instructions](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2751)
+
+### Ready to Merge
+* [#2772 Elastic bulk document creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2772)
+* [#1350 Kibana access from TDP](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1350)
+* [#1858 Spike: Secure Kibana access](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1858)
+* [#2711 Catch report month / year mismatches](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2711)
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2790 Kibana Deployment](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2790)
+* [#2681 Section 1 Validation clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2681)
+
+
+
+### Closed (not merged)
+* N/A
+
+
+---
+
+## Moved to Next Sprint (In Progress, Blocked, Raft Review)
+### In Progress
+* [#2646 - Populate data file summary case aggregates differently per section](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2646)
+* [#2820 [bug] Uncaught exception re: parsing error preventing feedback report generation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2820)
+* [#2768 Fix production OWASP scan reporting](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2768)
+* [#2799 Generate error mismatching field rpt_month_year w/ header](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2799)
+* [#2781 As a developer,  I want to have documentation on django migration best practices](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2781)
+
+
+### Blocked
+* N/A
+
+### Raft Review
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+* [#2746 As an STT, I need to know if there are issues with the DOBs reported in my data files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2746)
+* [#2813 Reduce dev environment count](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2813)
+* [#2729 As a developer, I want to move migration commands in the pipeline to CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2729)

docs/Sprint-Review/sprint-92-summary.md

@@ -0,0 +1,64 @@
+# Sprint 92 Summary
+
+01/31/2024 - 02/13/2024
+
+Velocity (Dev): 6
+
+## Sprint Goal
+* Dev:
+    * Continue parsing engine development and begin work on enhancement tickets
+        * #2536 Cat 4 validation 
+    * #1858 Secure OFA staff access to Kibana 
+        * Unblocks #1350 when complete 
+* DevOps:
+    * #2790 - Update deployment code to support Kibana and integrate with Standing Elastic instance
+* Design: 
+    * Tie up current documentation work
+    * Continue refinement of research roadmap
+
+
+## Tickets
+### Completed/Merged
+* [#2751 Resource Card updated with latest coding instructions](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2751)
+* [#1858 Spike: Secure Kibana access](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1858)
+* [#2781 As a developer,  I want to have documentation on django migration best practices](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2781)
+* [#2820 [bug] Uncaught exception re: parsing error preventing feedback report generation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2820)
+
+### Ready to Merge
+
+
+
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2790 Kibana Deployment](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2790)
+* [#2681 Section 1 Validation clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2681)
+
+
+
+### Closed (not merged)
+* N/A
+
+
+---
+
+## Moved to Next Sprint (In Progress, Blocked, Raft Review)
+### In Progress
+* [#1350 Kibana access from TDP](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1350)
+* [#2646 - Populate data file summary case aggregates differently per section](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2646)
+* [#2768 Fix production OWASP scan reporting](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2768)
+* [#2799 Generate error mismatching field rpt_month_year w/ header](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2799)
+
+
+
+### Blocked
+* N/A
+
+### Raft Review
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+* [#2746 As an STT, I need to know if there are issues with the DOBs reported in my data files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2746)
+* [#2813 Reduce dev environment count](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2813)
+* [#2729 As a developer, I want to move migration commands in the pipeline to CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2729)