Merge branch 'develop' into 3137-queryset-iterator

raft-tech · Sep 5, 2024 · 50a0276 · 50a0276
2 parents 7989afd + db43855
commit 50a0276
Show file tree

Hide file tree

Showing 19 changed files with 1,601 additions and 245 deletions.
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -5,16 +5,66 @@ tasks:
   create-network:
     desc: Create the external network
     cmds:
-      - docker network create external-net
+      - (docker network create external-net) || true
 
   init-backend:
     desc: Initialize the backend project
     dir: tdrs-backend
     cmds:
+      - task: create-network
       - docker-compose -f docker-compose.yml up -d --build
       - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py makemigrations"
       - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py migrate"
       - docker-compose -f docker-compose.yml down
+      - task: sentry-down
+
+  clone-sentry-repo:
+    desc: Clone the sentry repo
+    dir: sentry
+    cmds:
+      - git clone https://github.com/getsentry/self-hosted.git || true
+
+
+  create-sentry:
+    desc: Create Sentry service
+    dir: sentry
+    cmds:
+      # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask :  --cpuset-cpus 0 
+      - (docker run --privileged -p 9001:9000 -d --memory="8g" --memory-swap="8g" --name sentry docker:dind) || true
+      - docker exec sentry sh -c "git clone https://github.com/getsentry/self-hosted.git || true"
+
+      # need sleep 10 for docker to start
+      # there is a bug with other version of self-hosted. looks like they are trying to upgrade to Django 5.0 (July 2024)
+      - docker exec sentry sh -c "cd self-hosted && sleep 10 && git checkout tags/23.10.1"
+
+      # add bash
+      - docker exec sentry sh -c "apk add bash"
+      - docker cp docker-compose.yml sentry:/self-hosted/docker-compose.yml
+      - docker cp .env sentry:/self-hosted/.env
+      - docker exec sentry bash -c "cd self-hosted && ./install.sh --skip-user-creation --no-report-self-hosted-issues"
+      # create a new user
+      - docker exec sentry bash -c "cd self-hosted && docker-compose run --rm web createuser --email [email protected] --password admin --superuser"
+      # copy backup.json file to sentry
+      - docker cp backup.json sentry:/self-hosted/sentry/backup.json
+      # restore backup
+      - docker exec sentry bash -c "cd self-hosted && docker compose up -d"
+      - docker exec sentry bash -c "docker cp /self-hosted/sentry/backup.json sentry-self-hosted-web-1:/home/sentry/backup.json"
+      - docker exec sentry bash -c "docker exec sentry-self-hosted-web-1 bash -c 'sentry import /home/sentry/backup.json'"
+      - docker exec sentry bash -c "cd self-hosted && docker compose down"
+      - docker exec sentry bash -c "cd self-hosted && docker compose up -d"
+
+
+  sentry-up:
+    desc: Start sentry service
+    dir: sentry
+    cmds:
+      - docker exec sentry bash -c "cd self-hosted && docker-compose up -d"
+
+  sentry-down:
+    desc: Stop sentry service
+    dir: sentry
+    cmds:
+      - docker exec sentry bash -c "cd self-hosted && docker-compose down"
 
   drop-db:
     desc: Drop the backend database
@@ -45,7 +95,7 @@ tasks:
     desc: Restart backend web server
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml restart -d
+      - docker-compose -f docker-compose.yml restart
 
   backend-bash:
     desc: Open a shell in the backend container
@@ -78,13 +128,15 @@ tasks:
     desc: Run flake8 in the backend container
     dir: tdrs-backend
     cmds:
+      - task backend-up
       - docker-compose -f docker-compose.yml exec web sh -c "flake8 . && if [ $? -eq 0 ]; then echo 'Flake8 linter found no issues'; fi"
 
   backend-pip-lock:
     #TODO: Add a task to lock the pip dependencies
     desc: Lock the pip dependencies
     dir: tdrs-backend
     cmds:
+      - task: backend-up
       - docker-compose -f docker-compose.yml exec web sh -c "pipenv lock"
 
   psql:
@@ -99,9 +151,10 @@ tasks:
   clean:
     desc: Remove all containers, networks, and volumes
     cmds:
-      - docker-compose -f tdrs-backend/docker-compose.yml down -v
-      - docker-compose -f tdrs-frontend/docker-compose.yml down -v
-      - docker system prune -f -a
+      - docker stop $(docker ps -aq) || true
+      - docker rm $(docker ps -aq) || true
+      - docker rmi $(docker images -q) || true
+      - docker volume rm $(docker volume ls -q) || true
 
   clamav-up:
     desc: Start clamav service
@@ -125,7 +178,7 @@ tasks:
     desc: Restart frontend web server
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml restart -d
+      - docker-compose -f docker-compose.yml restart
 
   frontend-av:
     desc: Start frontend with optional clamav service

diff --git a/docs/Sprint-Review/sprint-105-summary.md b/docs/Sprint-Review/sprint-105-summary.md
@@ -0,0 +1,92 @@
+# sprint-105-summary
+
+7/31/2024 - 8/14/2024
+
+### Priority Setting
+
+* Reparsing&#x20;
+  * Tickets:&#x20;
+    * \#3064 — Re-parse Meta Model
+    * \#3113 —  As tech lead, I need the validation on the header update indicator revised to unblock parsing
+    * \#3073 — \[bug] TDP is raising cat 4 error on TANF/SSP closed case files that is not present
+* System Monitoring &#x20;
+* DIGIT Work  &#x20;
+
+### Sprint Goal
+
+**Dev:**
+
+_**Plain Language Error Messaging and Application Health Monitoring work, improved dev tooling, and fixing bugs**_
+
+* \#2792  — \[Error Audit] Category 3 error messages clean-up&#x20;
+* \#2965 — As tech lead, I want a database seed implemented for testing
+* \#3064 — Re-parse Meta Model
+* \#3113 —  As tech lead, I need the validation on the header update indicator revised to unblock parsing
+* \#3073 — \[bug] TDP is raising cat 4 error on TANF/SSP closed case files that is not present
+* \#3062  — bug: ES docker image for non-dev spaces stored in personal dockerhub
+* \#1646 — \[A11y Fix] Correct TDP home : aria label mismatch
+
+**DevOps:**
+
+_**Successful deployments across environments and pipeline stability investments**_
+
+* \#2458  — Integrate Nexus into CircleCI
+
+**Design:**
+
+_**Support reviews, Complete Research Synthesis, Continue Error Audit (Cat 4)**_
+
+* \#3078 — DIGIT Admin Experience Synthesis
+* \#3114  — \[Design Spike] In-app banner for submission history pages
+* \#2968  — \[Design Deliverable] Update Error Audit for Cat 4 / QA
+
+## Tickets
+
+### Completed/Merged
+
+* [#1621 As a TDP user, I'd like to see a descriptive error message page if authentication source is unavailable.](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1621)
+* [#1646 \[A11y Fix\] Correct TDP home : aria label mismatch](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1646)
+* [#3033 As tech lead, I need the sections 3 and 4 calendar quarter logic updated](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3033)
+* [#3055 Service timeout blocks parsing completion](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3055)
+* [#3057 \[Design Deliverable\] Spec for light-lift fiscal quarter / calendar quarter explainer in TDP](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3057)
+* [#3113 As tech lead, I need the validation on the header update indicator revised to unblock parsing ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3113)
+
+### Submitted (QASP Review, OCIO Review)
+
+* [#2954 Extend SESSION\_COOKIE\_AGE](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2954)
+* [#3061 \[a11y fix\] Django multi-select filter ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3061)
+* [#3079 DB Backup Script Fix](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3079)
+* [#2883 Pre-Made Reporting Dashboards on Kibana](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2883)
+* [#2985 \[Design Deliverable\] Email spec for Admin Notification for stuck files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2985)
+* [#2996 Add dynamic field name to cat4 error messages](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2996)
+* [#2993 Kibana Dashboard MVP](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2993)
+
+### Ready to Merge
+
+* [#3058 \[Design Deliverable\] Release notes email template](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3058)
+* [#3062 bug: ES docker image for non-dev spaces stored in personal dockerhub](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3062)
+* [#3073 \[bug\] TDP is raising cat 4 error on TANF/SSP closed case files that is not present](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3073)
+* [#3107 \[Re-parse command\] Retain original submission date when command runs](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3107)
+
+### Closed (Not Merged)
+
+* [#1355 Research questions around DIGIT teams query usage for parsed data](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1355)
+
+### Moved to Next Sprint&#x20;
+
+**In Progress**&#x20;
+
+* [#2965 As tech lead, I want a database seed implemented for testing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2965)
+
+#### Blocked
+
+* [#2458 Integrate Nexus into CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2458)
+
+**Raft Review**
+
+* [#3043 Sentry: Local environment for Debugging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3043)
+* [#3064 Re-parse Meta Model](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3064)
+* [#3065 Spike - Guarantee Sequential Execution of Re-parse Command](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3065)
+* [#3078 \[Research Synthesis\] DIGIT Admin Experience Improvements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3078)
+* [#3087 Admin By Newest Filter Enhancements for Data Files Page](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3087)
+* [#2792 \[Error Audit\] Category 3 error messages clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2792)
diff --git a/docs/Sprint-Review/sprint-106-summary.md b/docs/Sprint-Review/sprint-106-summary.md
@@ -0,0 +1,97 @@
+# sprint-106-summary
+
+8/14/202 - 8/27/2024
+
+### <mark style="color:orange;">Priority Setting</mark>
+
+* Reparsing &#x20;
+  * Reasoning: OFA has ability to reparse and the data process with latest system logic which will enable STTs to get error reports that are meaningful and relevant and OFA gets the needed data. System, as a whole, will be more flexible with data flow (able to hot fix and hit reparse).  Additionally, we want to avoid missing data and be able to repopulate the database after cleaning the data. &#x20;
+* Data Access Strategy&#x20;
+  * Reasoning: Creating daily blockers &#x20;
+* Admin Console Improvements&#x20;
+  * Reasoning: Trigger has been met to refine tickets (Research Synthesis)
+* Improved Dev Tooling&#x20;
+  * improve test\_parse.py 2641&#x20;
+  * separate celery 2592&#x20;
+  * Reasoning: Developing / refining tickets to support the above priorities – These tickets will enhance capabilities while the above are being flushed out &#x20;
+
+### <mark style="color:orange;">Sprint Goal</mark>
+
+**Dev:**
+
+_**Reparsing, Admin Console Improvements, Application Health Monitoring work, and Improved Dev Tooling**_
+
+* \#2965 — As tech lead, I want a database seed implemented for testing
+* \#3102  — Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export&#x20;
+* \#2561 — As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days
+* \#3110 — Spike - Investigate Custom Filter Integration
+* \#3137 — \[bug] OFA unable to export data to csv by record type and fiscal period
+* \#3074 — TDP Data Files page permissions for DIGIT & Sys Admin user groups
+* \#3076  — Admin Filter Enhancements for Data Files Page&#x20;
+
+**DevOps:**
+
+_**Successful deployments across environments and pipeline stability investments**_
+
+*
+
+**Design:**
+
+_**Support reviews, In-app banner to support parsed data, Continue Error Audit (Cat 4)**_
+
+* \#2968  — \[Design Deliverable] Update Error Audit for Cat 4 / QA
+* \#3114 — \[Design Spike] In-app banner for submission history pages w/ data parsed before May 2024
+* \#3143 — August release notes — Knowledge Center & Email Template
+
+
+
+## Tickets
+
+### Completed/Merged
+
+* [#2985 \[Design Deliverable\] Email spec for Admin Notification for stuck files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2985)
+* [#2996 Add dynamic field name to cat4 error messages](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2996)
+* [#3143 August release notes — Knowledge Center & Email Template](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3143)
+* [#3061 \[a11y fix\] Django multi-select filter ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3061)
+* [#2954 Extend SESSION\_COOKIE\_AGE](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2954)
+* [#3079 DB Backup Script Fix](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3079)
+
+### Submitted (QASP Review, OCIO Review)
+
+* [#3064 Re-parse Meta Model](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3064)
+* [#3065 Spike - Guarantee Sequential Execution of Re-parse Command](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3065)
+* [#2792 \[Error Audit\] Category 3 error messages clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2792)
+* [#2883 Pre-Made Reporting Dashboards on Kibana](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2883)
+* [#2561 As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2561)
+* [#3078 \[Research Synthesis\] DIGIT Admin Experience Improvements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3078)
+
+### Ready to Merge
+
+### Closed (Not Merged)
+
+* [#3147 S3 buckets contain fewer datafiles than DAC](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3147)
+
+### Moved to Next Sprint&#x20;
+
+**In Progress**&#x20;
+
+* [#2965 As tech lead, I want a database seed implemented for testing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2965)
+* [#2458 Integrate Nexus into CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2458)
+* [#3137 \[bug\] OFA unable to export data to csv by record type and fiscal period](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3137)
+* [#2968 \[Design Deliverable\] Update Error Audit for Cat 4 / QA](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2968)
+* [#3060 As a TDP user, I need to stay logged in when I'm actively using the system](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3060)
+* [#3074 TDP Data Files page permissions for DIGIT & Sys Admin user groups](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3074)
+* [#3114 \[Design Spike\] In-app banner for submission history pages w/ data parsed before May 2024](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3114)
+* [#3142 \[Research Spike\] Get more detail about Yun & DIGIT's data workflow and use cases](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3142)
+
+#### Blocked
+
+*
+
+**Raft Review**
+
+* [#3043 Sentry: Local environment for Debugging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3043)
+* [#3110 Spike - Investigate Custom Filter Integration](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3110)
+* [#3102 Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3102)
+* [#3087 Admin By Newest Filter Enhancements for Data Files Page](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3087)
+* [#3076 Admin Filter Enhancements for Data Files Page ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3076)
diff --git a/sentry/.env b/sentry/.env
@@ -0,0 +1,35 @@
+COMPOSE_PROJECT_NAME=sentry-self-hosted
+COMPOSE_PROFILES=feature-complete
+SENTRY_EVENT_RETENTION_DAYS=90
+# You can either use a port number or an IP:PORT combo for SENTRY_BIND
+# See https://docs.docker.com/compose/compose-file/#ports for more
+SENTRY_BIND=9000
+# Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails!
+# SENTRY_MAIL_HOST=example.com
+
+
+# https://hub.docker.com/r/getsentry/sentry/tags?page=1205&page_size=&ordering=&name=
+SENTRY_IMAGE=getsentry/sentry:23.10.1
+
+
+# https://hub.docker.com/r/getsentry/snuba/tags?page=105&page_size=&name=&ordering=
+SNUBA_IMAGE=getsentry/snuba:23.10.1
+
+# https://hub.docker.com/r/getsentry/relay/tags?page=100&page_size=&name=&ordering=
+RELAY_IMAGE=getsentry/relay:23.10.1
+
+
+#https://hub.docker.com/r/getsentry/symbolicator/tags?page=15&page_size=&name=&ordering=
+SYMBOLICATOR_IMAGE=getsentry/symbolicator:23.10.1
+
+# https://hub.docker.com/r/getsentry/vroom/tags?page=15&page_size=&name=&ordering=
+VROOM_IMAGE=getsentry/vroom:23.10.1
+
+
+WAL2JSON_VERSION=latest
+HEALTHCHECK_INTERVAL=30s
+HEALTHCHECK_TIMEOUT=1m30s
+HEALTHCHECK_RETRIES=10
+# Caution: Raising max connections of postgres increases CPU and RAM usage
+# see https://github.com/getsentry/self-hosted/pull/2740 for more information
+POSTGRES_MAX_CONNECTIONS=100