Merge branch 'develop' into bug/e2e-minimal

.circleci/base_config.yml

@@ -13,11 +13,11 @@ executors:
   machine-executor:
     machine:
       docker_layer_caching: false
-      image: ubuntu-2204:2024.01.1
+      image: ubuntu-2204:2024.05.1
   large-machine-executor:
     machine:
       docker_layer_caching: false
-      image: ubuntu-2204:2024.01.1
+      image: ubuntu-2204:2024.05.1
     resource_class: large
 
 parameters:

.circleci/build-and-test/commands.yml

@@ -25,7 +25,7 @@
             fi
             echo "export CURRENT_FLAG=$CURRENT_FLAG" >> $BASH_ENV
       - run:
-          name: Upload code coverage report if target branch
+          name: Upload code coverage report of target branch
           command: codecov -t "$CODECOV_TOKEN" -f <<parameters.coverage-report>> -F "$CURRENT_FLAG"
 
   install-nodejs-machine:
@@ -49,6 +49,13 @@
       - run:
           name: Disable npm audit warnings in CI
           command: npm set audit false
-          
+
   # This allows us to use the node orb to install packages within other commands
   install-nodejs-packages: node/install-packages
+
+  docker-login:
+    steps:
+      - run:
+          name: Docker login
+          command: |
+            echo "$CIRCI_DOCKER_LOGIN" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-circi --password-stdin

.circleci/build-and-test/jobs.yml

@@ -3,6 +3,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - docker-compose-up-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
@@ -46,6 +47,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - docker-compose-up-backend
       - docker-compose-up-frontend
       - install-nodejs-machine

.circleci/build-and-test/workflows.yml

@@ -49,6 +49,12 @@
                 - /^release.*/
           requires:
             - secrets-check
+      - make_erd: # from ../util folder
+          filters:
+            branches:
+              only:
+                - develop
+                - master
 
   build-and-test-backend:
     when: << pipeline.parameters.build_and_test_backend >>

.circleci/deployment/commands.yml

@@ -1,4 +1,33 @@
 # commands:
+  init-deploy:
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+
+  build-and-tag-images:
+    parameters:
+      backend-appname:
+        default: tdp-backend
+        type: string
+      frontend-appname:
+        default: tdp-frontend
+        type: string
+    steps:
+      - run:
+          name: Update Docker daemon
+          command: |
+            sudo echo '{"max-concurrent-uploads": 1}' | sudo tee /etc/docker/daemon.json
+            sudo service docker restart
+      - run:
+          name: Create builder
+          command: |
+            docker buildx create --name container-builder --driver docker-container --use --bootstrap
+      - run:
+          name: Build and tag images
+          command: |
+            ./scripts/build-and-tag-images.sh <<parameters.backend-appname>> <<parameters.frontend-appname>> ./tdrs-backend ./tdrs-frontend $CIRCLE_BUILD_NUM $CIRCLE_SHA1 "$CIRCI_DOCKER_LOGIN" tdp-circi
+
   deploy-cloud-dot-gov:
     parameters:
       environment:
@@ -25,9 +54,6 @@
         default: tdp-frontend
         type: string
     steps:
-      - checkout
-      - sudo-check
-      - cf-check
       - login-cloud-dot-gov:
           cf-password: <<parameters.cf-password>>
           cf-org: <<parameters.cf-org>>

.circleci/deployment/jobs.yml

@@ -1,10 +1,41 @@
+  build-and-tag-develop:
+    executor: large-machine-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - checkout
+      - sudo-check
+      - build-and-tag-images:
+          backend-appname: tdp-backend-develop
+          frontend-appname: tdp-frontend-develop
+
+  build-and-tag-staging:
+    executor: large-machine-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - checkout
+      - sudo-check
+      - build-and-tag-images:
+          backend-appname: tdp-backend-staging
+          frontend-appname: tdp-frontend-staging
+
+  build-and-tag-production:
+    executor: large-machine-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - checkout
+      - sudo-check
+      - build-and-tag-images:
+          backend-appname: tdp-backend-production
+          frontend-appname: tdp-frontend-production
+
   deploy-dev:
     parameters:
       target_env:
         type: string
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
+      - init-deploy
       - deploy-cloud-dot-gov:
           backend-appname: tdp-backend-<< parameters.target_env >>
           frontend-appname: tdp-frontend-<< parameters.target_env >>
@@ -13,6 +44,7 @@
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
+      - init-deploy
       - deploy-cloud-dot-gov:
           backend-appname: tdp-backend-staging
           frontend-appname: tdp-frontend-staging
@@ -24,6 +56,7 @@
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
+      - init-deploy
       - deploy-cloud-dot-gov:
           backend-appname: tdp-backend-develop
           frontend-appname: tdp-frontend-develop
@@ -133,6 +166,7 @@
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
+      - init-deploy
       - deploy-cloud-dot-gov:
           environment: production
           backend-appname: tdp-backend-prod

.circleci/deployment/workflows.yml

@@ -93,27 +93,48 @@
                 - develop
                 - main
                 - master
-      - deploy-develop:
+      - build-and-tag-develop:
           requires:
             - deploy-infrastructure-staging
           filters:
             branches:
               only:
                 - develop
-      - deploy-staging:
+      - deploy-develop:
+          requires:
+            - build-and-tag-develop
+          filters:
+            branches:
+              only:
+                - develop
+      - build-and-tag-staging:
           requires:
             - deploy-infrastructure-staging
           filters:
             branches:
               only:
                 - main
-      - deploy-production:
+      - deploy-staging:
+          requires:
+            - build-and-tag-staging
+          filters:
+            branches:
+              only:
+                - main
+      - build-and-tag-production:
           requires:
             - deploy-infrastructure-production
           filters:
             branches:
               only:
                 - master
+      - deploy-production:
+          requires:
+            - build-and-tag-production
+          filters:
+            branches:
+              only:
+                - master
       - test-deployment-e2e:
           requires:
             - deploy-develop

.circleci/owasp/jobs.yml

@@ -4,6 +4,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - docker-compose-up-backend
       - docker-compose-up-frontend
       - run:
@@ -26,6 +27,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - docker-compose-up-backend
       - docker-compose-up-frontend
       - run:
@@ -66,6 +68,7 @@
       - sudo-check
       - cf-check
       - docker-compose-check
+      - docker-login
       - login-cloud-dot-gov:
           cf-password: <<parameters.cf_password>>
           cf-space: <<parameters.cf_space>>

.circleci/util/jobs.yml

@@ -4,6 +4,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - run:
           name: Run graph_models
           command: |
@@ -13,5 +14,9 @@
             fi
             docker-compose run --rm web bash -c \
             "./manage.py graph_models -a -g -o tdp_erd.png"
+            if [[ ! -f tdp_erd.png ]]; then
+              echo "Entity Relationship Diagram not found."
+              exit 1
+            fi
       - store_artifacts:
           path: tdrs-backend/tdp_erd.png

.gitconfig

@@ -14,3 +14,4 @@
 	allowed = .git/config:.*
 	allowed = .gitconfig:.*
 	allowed = .*DJANGO_SECRET_KEY=.*
+	allowed = ./tdrs-backend/plg/loki/manifest.yml:*

.github/ISSUE_TEMPLATE/dev-issue-template.md

@@ -9,6 +9,7 @@ assignees: ''
 
 **Description:**
 _Provide a brief background and justification for this issue_
+_OFA & UX team: Please include bullets capturing the value to STT end users, related guidance, and/or talking points that should be communicated to STTs in release notes_
 
 
 **Acceptance Criteria:** 

.gitignore

@@ -108,4 +108,10 @@ tfapply
 cypress.env.json
 
 # Patches
-*.patch
+*.patch
+
+# Logs
+*.log
+
+# DB seeds
+tdrs-backend/*.pg

Taskfile.yml

@@ -29,7 +29,7 @@ tasks:
     desc: Create Sentry service
     dir: sentry
     cmds:
-      # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask :  --cpuset-cpus 0 
+      # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask :  --cpuset-cpus 0
       - (docker run --privileged -p 9001:9000 -d --memory="8g" --memory-swap="8g" --name sentry docker:dind) || true
       - docker exec sentry sh -c "git clone https://github.com/getsentry/self-hosted.git || true"
 
@@ -109,6 +109,23 @@ tasks:
     cmds:
       - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py shell"
 
+  backend-exec:
+    desc: Execute a command in the backend container
+    dir: tdrs-backend
+    vars:
+      CMD: '{{.CMD}}'
+    cmds:
+      - docker-compose -f docker-compose.yml exec web sh -c "python manage.py {{.CMD}}"
+
+  backend-exec-seed-db:
+    desc: Execute seed_db command in the backend container
+    dir: tdrs-backend
+    vars:
+      CMD: '{{.CMD}}'
+    cmds:
+      - docker-compose -f docker-compose.yml up -d
+      - docker-compose -f docker-compose.yml exec web sh -c "python manage.py populate_stts; python ./manage.py seed_db"
+
   backend-pytest:
     desc: 'Run pytest in the backend container E.g: task backend-pytest PYTEST_ARGS="tdpservice/test/ -s -vv"'
     dir: tdrs-backend
@@ -155,7 +172,7 @@ tasks:
       - docker rm $(docker ps -aq) || true
       - docker rmi $(docker images -q) || true
       - docker volume rm $(docker volume ls -q) || true
-  
+
   clamav-up:
     desc: Start clamav service
     dir: tdrs-backend
@@ -187,7 +204,7 @@ tasks:
       - task: frontend-up
       - task: clamav-up
 
-  
+
   # need more work
   frontend-init:
     desc: Initialize the frontend project
@@ -237,7 +254,7 @@ tasks:
     desc: Open a shell in the frontend container
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml exec tdp-frontend sh
+      - docker-compose -f docker-compose.yml exec tdp-frontend bash
 
   up:
     desc: Start both frontend and backend web servers

codecov.yml

@@ -42,4 +42,7 @@ flags:
     carryforward: true
 
 ignore:
-  - "tdrs-backend/tdpservice/scheduling/db_backup.py"
+  - "tdrs-backend/tdpservice/scheduling/db_backup.py"
+  - "tdrs-backend/tdpservice/search_indexes/admin/mulitselect_filter.py"
+  - "tdrs-backend/tdpservice/email/helpers/account_access_requests.py"
+  - "tdrs-backend/tdpservice/search_indexes/admin/filters.py"

docs/How-We-Work/team-meetings.md

@@ -68,12 +68,12 @@ A typical sprint schedule is described in the table below.
       - External factors outside of the author spending time building the ticket (ie need external team's input, see how a feature develops, etc.)
       - Ex. Waiting on X ticket to finish the scope of said unrefined ticket, problem found / unsure how big it is and knows other work will unearth it
       - If we know the ACs but not the tasks, then its unrefined
+      - Release Notes summary is empty or incomplete as applicable, to be provided by UX/Product by default.
    - Refined: Ticket is complete and is ready to be executed.
    - Refined & Ready to Go (Next Sprint)
       - "Earmarked" work for the upcoming sprint.
 - **Labelling:**
    - WIP
-      - Author knows the 5 W's or darn near (90%) 
       - Drafted ticket – either still on the author the finish their part or a short team conversation is needed. 
       - Administrative in nature
       - Ex. Stub, ticket that doesn't feel there's enough to warrant an introduction