Merge pull request #543 from raft-tech/release/v3.6.1-Sprint-107

Release/v3.6.1 sprint 107

Taskfile.yml

@@ -5,16 +5,66 @@ tasks:
   create-network:
     desc: Create the external network
     cmds:
-      - docker network create external-net
+      - (docker network create external-net) || true
 
   init-backend:
     desc: Initialize the backend project
     dir: tdrs-backend
     cmds:
+      - task: create-network
       - docker-compose -f docker-compose.yml up -d --build
       - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py makemigrations"
       - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py migrate"
       - docker-compose -f docker-compose.yml down
+      - task: sentry-down
+
+  clone-sentry-repo:
+    desc: Clone the sentry repo
+    dir: sentry
+    cmds:
+      - git clone https://github.com/getsentry/self-hosted.git || true
+
+
+  create-sentry:
+    desc: Create Sentry service
+    dir: sentry
+    cmds:
+      # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask :  --cpuset-cpus 0 
+      - (docker run --privileged -p 9001:9000 -d --memory="8g" --memory-swap="8g" --name sentry docker:dind) || true
+      - docker exec sentry sh -c "git clone https://github.com/getsentry/self-hosted.git || true"
+
+      # need sleep 10 for docker to start
+      # there is a bug with other version of self-hosted. looks like they are trying to upgrade to Django 5.0 (July 2024)
+      - docker exec sentry sh -c "cd self-hosted && sleep 10 && git checkout tags/23.10.1"
+
+      # add bash
+      - docker exec sentry sh -c "apk add bash"
+      - docker cp docker-compose.yml sentry:/self-hosted/docker-compose.yml
+      - docker cp .env sentry:/self-hosted/.env
+      - docker exec sentry bash -c "cd self-hosted && ./install.sh --skip-user-creation --no-report-self-hosted-issues"
+      # create a new user
+      - docker exec sentry bash -c "cd self-hosted && docker-compose run --rm web createuser --email [email protected] --password admin --superuser"
+      # copy backup.json file to sentry
+      - docker cp backup.json sentry:/self-hosted/sentry/backup.json
+      # restore backup
+      - docker exec sentry bash -c "cd self-hosted && docker compose up -d"
+      - docker exec sentry bash -c "docker cp /self-hosted/sentry/backup.json sentry-self-hosted-web-1:/home/sentry/backup.json"
+      - docker exec sentry bash -c "docker exec sentry-self-hosted-web-1 bash -c 'sentry import /home/sentry/backup.json'"
+      - docker exec sentry bash -c "cd self-hosted && docker compose down"
+      - docker exec sentry bash -c "cd self-hosted && docker compose up -d"
+
+
+  sentry-up:
+    desc: Start sentry service
+    dir: sentry
+    cmds:
+      - docker exec sentry bash -c "cd self-hosted && docker-compose up -d"
+
+  sentry-down:
+    desc: Stop sentry service
+    dir: sentry
+    cmds:
+      - docker exec sentry bash -c "cd self-hosted && docker-compose down"
 
   drop-db:
     desc: Drop the backend database
@@ -45,7 +95,7 @@ tasks:
     desc: Restart backend web server
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml restart -d
+      - docker-compose -f docker-compose.yml restart
 
   backend-bash:
     desc: Open a shell in the backend container
@@ -78,13 +128,15 @@ tasks:
     desc: Run flake8 in the backend container
     dir: tdrs-backend
     cmds:
+      - task backend-up
       - docker-compose -f docker-compose.yml exec web sh -c "flake8 . && if [ $? -eq 0 ]; then echo 'Flake8 linter found no issues'; fi"
 
   backend-pip-lock:
     #TODO: Add a task to lock the pip dependencies
     desc: Lock the pip dependencies
     dir: tdrs-backend
     cmds:
+      - task: backend-up
       - docker-compose -f docker-compose.yml exec web sh -c "pipenv lock"
 
   psql:
@@ -99,9 +151,10 @@ tasks:
   clean:
     desc: Remove all containers, networks, and volumes
     cmds:
-      - docker-compose -f tdrs-backend/docker-compose.yml down -v
-      - docker-compose -f tdrs-frontend/docker-compose.yml down -v
-      - docker system prune -f -a
+      - docker stop $(docker ps -aq) || true
+      - docker rm $(docker ps -aq) || true
+      - docker rmi $(docker images -q) || true
+      - docker volume rm $(docker volume ls -q) || true
 
   clamav-up:
     desc: Start clamav service
@@ -125,7 +178,7 @@ tasks:
     desc: Restart frontend web server
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml restart -d
+      - docker-compose -f docker-compose.yml restart
 
   frontend-av:
     desc: Start frontend with optional clamav service

docs/Sprint-Review/sprint-106-summary.md

@@ -0,0 +1,97 @@
+# sprint-106-summary
+
+8/14/202 - 8/27/2024
+
+### <mark style="color:orange;">Priority Setting</mark>
+
+* Reparsing &#x20;
+  * Reasoning: OFA has ability to reparse and the data process with latest system logic which will enable STTs to get error reports that are meaningful and relevant and OFA gets the needed data. System, as a whole, will be more flexible with data flow (able to hot fix and hit reparse).  Additionally, we want to avoid missing data and be able to repopulate the database after cleaning the data. &#x20;
+* Data Access Strategy&#x20;
+  * Reasoning: Creating daily blockers &#x20;
+* Admin Console Improvements&#x20;
+  * Reasoning: Trigger has been met to refine tickets (Research Synthesis)
+* Improved Dev Tooling&#x20;
+  * improve test\_parse.py 2641&#x20;
+  * separate celery 2592&#x20;
+  * Reasoning: Developing / refining tickets to support the above priorities – These tickets will enhance capabilities while the above are being flushed out &#x20;
+
+### <mark style="color:orange;">Sprint Goal</mark>
+
+**Dev:**
+
+_**Reparsing, Admin Console Improvements, Application Health Monitoring work, and Improved Dev Tooling**_
+
+* \#2965 — As tech lead, I want a database seed implemented for testing
+* \#3102  — Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export&#x20;
+* \#2561 — As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days
+* \#3110 — Spike - Investigate Custom Filter Integration
+* \#3137 — \[bug] OFA unable to export data to csv by record type and fiscal period
+* \#3074 — TDP Data Files page permissions for DIGIT & Sys Admin user groups
+* \#3076  — Admin Filter Enhancements for Data Files Page&#x20;
+
+**DevOps:**
+
+_**Successful deployments across environments and pipeline stability investments**_
+
+*
+
+**Design:**
+
+_**Support reviews, In-app banner to support parsed data, Continue Error Audit (Cat 4)**_
+
+* \#2968  — \[Design Deliverable] Update Error Audit for Cat 4 / QA
+* \#3114 — \[Design Spike] In-app banner for submission history pages w/ data parsed before May 2024
+* \#3143 — August release notes — Knowledge Center & Email Template
+
+
+
+## Tickets
+
+### Completed/Merged
+
+* [#2985 \[Design Deliverable\] Email spec for Admin Notification for stuck files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2985)
+* [#2996 Add dynamic field name to cat4 error messages](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2996)
+* [#3143 August release notes — Knowledge Center & Email Template](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3143)
+* [#3061 \[a11y fix\] Django multi-select filter ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3061)
+* [#2954 Extend SESSION\_COOKIE\_AGE](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2954)
+* [#3079 DB Backup Script Fix](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3079)
+
+### Submitted (QASP Review, OCIO Review)
+
+* [#3064 Re-parse Meta Model](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3064)
+* [#3065 Spike - Guarantee Sequential Execution of Re-parse Command](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3065)
+* [#2792 \[Error Audit\] Category 3 error messages clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2792)
+* [#2883 Pre-Made Reporting Dashboards on Kibana](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2883)
+* [#2561 As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2561)
+* [#3078 \[Research Synthesis\] DIGIT Admin Experience Improvements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3078)
+
+### Ready to Merge
+
+### Closed (Not Merged)
+
+* [#3147 S3 buckets contain fewer datafiles than DAC](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3147)
+
+### Moved to Next Sprint&#x20;
+
+**In Progress**&#x20;
+
+* [#2965 As tech lead, I want a database seed implemented for testing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2965)
+* [#2458 Integrate Nexus into CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2458)
+* [#3137 \[bug\] OFA unable to export data to csv by record type and fiscal period](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3137)
+* [#2968 \[Design Deliverable\] Update Error Audit for Cat 4 / QA](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2968)
+* [#3060 As a TDP user, I need to stay logged in when I'm actively using the system](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3060)
+* [#3074 TDP Data Files page permissions for DIGIT & Sys Admin user groups](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3074)
+* [#3114 \[Design Spike\] In-app banner for submission history pages w/ data parsed before May 2024](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3114)
+* [#3142 \[Research Spike\] Get more detail about Yun & DIGIT's data workflow and use cases](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3142)
+
+#### Blocked
+
+*
+
+**Raft Review**
+
+* [#3043 Sentry: Local environment for Debugging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3043)
+* [#3110 Spike - Investigate Custom Filter Integration](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3110)
+* [#3102 Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3102)
+* [#3087 Admin By Newest Filter Enhancements for Data Files Page](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3087)
+* [#3076 Admin Filter Enhancements for Data Files Page ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3076)

sentry/.env

@@ -0,0 +1,35 @@
+COMPOSE_PROJECT_NAME=sentry-self-hosted
+COMPOSE_PROFILES=feature-complete
+SENTRY_EVENT_RETENTION_DAYS=90
+# You can either use a port number or an IP:PORT combo for SENTRY_BIND
+# See https://docs.docker.com/compose/compose-file/#ports for more
+SENTRY_BIND=9000
+# Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails!
+# SENTRY_MAIL_HOST=example.com
+
+
+# https://hub.docker.com/r/getsentry/sentry/tags?page=1205&page_size=&ordering=&name=
+SENTRY_IMAGE=getsentry/sentry:23.10.1
+
+
+# https://hub.docker.com/r/getsentry/snuba/tags?page=105&page_size=&name=&ordering=
+SNUBA_IMAGE=getsentry/snuba:23.10.1
+
+# https://hub.docker.com/r/getsentry/relay/tags?page=100&page_size=&name=&ordering=
+RELAY_IMAGE=getsentry/relay:23.10.1
+
+
+#https://hub.docker.com/r/getsentry/symbolicator/tags?page=15&page_size=&name=&ordering=
+SYMBOLICATOR_IMAGE=getsentry/symbolicator:23.10.1
+
+# https://hub.docker.com/r/getsentry/vroom/tags?page=15&page_size=&name=&ordering=
+VROOM_IMAGE=getsentry/vroom:23.10.1
+
+
+WAL2JSON_VERSION=latest
+HEALTHCHECK_INTERVAL=30s
+HEALTHCHECK_TIMEOUT=1m30s
+HEALTHCHECK_RETRIES=10
+# Caution: Raising max connections of postgres increases CPU and RAM usage
+# see https://github.com/getsentry/self-hosted/pull/2740 for more information
+POSTGRES_MAX_CONNECTIONS=100