From 81895abbd3a9e93e44df2e3e4dcc47e76f5785e3 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 15 Nov 2024 10:37:36 -0500 Subject: [PATCH 01/25] - Update configs to use deployed AM endpoint - Added nginx route for AM in cloud.gov --- tdrs-backend/plg/prometheus/alerts.local.yml | 1 + .../plg/prometheus/prometheus.local.yml | 2 +- tdrs-backend/plg/prometheus/prometheus.yml | 7 +++--- tdrs-frontend/nginx/cloud.gov/locations.conf | 24 ++++++++++++++++--- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/plg/prometheus/alerts.local.yml b/tdrs-backend/plg/prometheus/alerts.local.yml index 99183c544..a13cc7543 100644 --- a/tdrs-backend/plg/prometheus/alerts.local.yml +++ b/tdrs-backend/plg/prometheus/alerts.local.yml @@ -32,6 +32,7 @@ groups: rules: - alert: UpTime expr: avg_over_time(up[1m]) < 0.95 + for: 30m labels: severity: WARNING annotations: diff --git a/tdrs-backend/plg/prometheus/prometheus.local.yml b/tdrs-backend/plg/prometheus/prometheus.local.yml index 8b0a4517d..66576951b 100644 --- a/tdrs-backend/plg/prometheus/prometheus.local.yml +++ b/tdrs-backend/plg/prometheus/prometheus.local.yml @@ -15,7 +15,7 @@ alerting: # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. rule_files: - "django-rules.yml" - - "alerts.yml" + - "alerts.local.yml" # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. diff --git a/tdrs-backend/plg/prometheus/prometheus.yml b/tdrs-backend/plg/prometheus/prometheus.yml index 66e35c519..97ab67252 100644 --- a/tdrs-backend/plg/prometheus/prometheus.yml +++ b/tdrs-backend/plg/prometheus/prometheus.yml @@ -9,11 +9,12 @@ alerting: - path_prefix: /alerts static_configs: - targets: - # - alertmanager.apps.internal:8080 + - alertmanager.apps.internal:8080 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. rule_files: - "django-rules.yml" + - "alert.yml" scrape_configs: # The job name is added as a label `job=` to any timeseries scraped from this config. @@ -84,7 +85,7 @@ scrape_configs: - job_name: loki static_configs: - - targets: ["loki.apps.internal:3100"] + - targets: ["loki.apps.internal:8080"] labels: service: "loki" env: "production" @@ -92,7 +93,7 @@ scrape_configs: - job_name: grafana metrics_path: /grafana/metrics static_configs: - - targets: ["grafana.app.cloud.gov:9400"] + - targets: ["grafana.app.cloud.gov:8080"] labels: service: "grafana" env: "production" diff --git a/tdrs-frontend/nginx/cloud.gov/locations.conf b/tdrs-frontend/nginx/cloud.gov/locations.conf index 37bd413e2..85f681543 100644 --- a/tdrs-frontend/nginx/cloud.gov/locations.conf +++ b/tdrs-frontend/nginx/cloud.gov/locations.conf @@ -61,7 +61,7 @@ location = /kibana_auth_check { } location /grafana/ { - auth_request /grafana_auth_check; + auth_request /plg_auth_check; auth_request_set $auth_status $upstream_status; set $grafana http://grafana.apps.internal:8080$request_uri; @@ -78,9 +78,27 @@ location /grafana/ { proxy_buffer_size 4k; } -location = /grafana_auth_check { +location /alerts/ { + auth_request /plg_auth_check; + auth_request_set $auth_status $upstream_status; + + set $alerts http://alertmanager.apps.internal:8080$request_uri; + proxy_pass $alerts; + proxy_set_header Host $host:3000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + + proxy_connect_timeout 300; + proxy_read_timeout 300; + proxy_send_timeout 300; + send_timeout 900; + proxy_buffer_size 4k; +} + +location = /plg_auth_check { internal; - set $endpoint http://{{env "BACKEND_HOST"}}.apps.internal:8080/grafana_auth_check/; + set $endpoint http://{{env "BACKEND_HOST"}}.apps.internal:8080/plg_auth_check/; proxy_pass $endpoint$1$is_args$args; proxy_set_header Host $host:3000; proxy_set_header X-Real-IP $remote_addr; From 327018f3a84ec214244a1737638fd929da76c357 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 15 Nov 2024 13:10:26 -0500 Subject: [PATCH 02/25] - update the deploy script to deploy alertmanager --- tdrs-backend/plg/deploy.sh | 48 ++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index c411f5457..988f85b26 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -46,7 +46,6 @@ deploy_pg_exporter() { deploy_grafana() { pushd grafana - APP_NAME="grafana" DATASOURCES="datasources.yml" cp datasources.template.yml $DATASOURCES MANIFEST=manifest.tmp.yml @@ -57,20 +56,21 @@ deploy_grafana() { yq eval -i ".applications[0].services[0] = \"$1\"" $MANIFEST cf push --no-route -f $MANIFEST -t 180 --strategy rolling - cf map-route $APP_NAME apps.internal --hostname $APP_NAME + cf map-route grafana apps.internal --hostname grafana # Add policy to allow grafana to talk to prometheus and loki - cf add-network-policy $APP_NAME prometheus --protocol tcp --port 8080 - cf add-network-policy $APP_NAME loki --protocol tcp --port 8080 + cf add-network-policy grafana prometheus --protocol tcp --port 8080 + cf add-network-policy grafana loki --protocol tcp --port 8080 # Add network policies to allow grafana to talk to all frontend apps in all environments for app in ${DEV_FRONTEND_APPS[@]}; do - cf add-network-policy "grafana" $app -s "tanf-dev" --protocol tcp --port 80 + cf add-network-policy grafana $app -s tanf-dev --protocol tcp --port 80 done for app in ${STAGING_FRONTEND_APPS[@]}; do - cf add-network-policy "grafana" $app -s "tanf-staging" --protocol tcp --port 80 + cf add-network-policy grafana $app -s tanf-staging --protocol tcp --port 80 done - cf add-network-policy "grafana" $PROD_FRONTEND --protocol tcp --port 80 + cf add-network-policy grafana $PROD_FRONTEND --protocol tcp --port 80 + cf add-network-policy $PROD_FRONTEND grafana -s tanf-prod --protocol tcp --port 8080 rm $DATASOURCES rm $MANIFEST @@ -84,13 +84,12 @@ deploy_prometheus() { # Add network policies to allow prometheus to talk to all backend apps in all environments for app in ${DEV_BACKEND_APPS[@]}; do - cf add-network-policy prometheus $app -s "tanf-dev" --protocol tcp --port 8080 + cf add-network-policy prometheus $app -s tanf-dev --protocol tcp --port 8080 done for app in ${STAGING_BACKEND_APPS[@]}; do - cf add-network-policy prometheus $app -s "tanf-staging" --protocol tcp --port 8080 + cf add-network-policy prometheus $app -s tanf-staging --protocol tcp --port 8080 done cf add-network-policy prometheus $PROD_BACKEND --protocol tcp --port 8080 - popd } @@ -98,26 +97,45 @@ deploy_loki() { pushd loki cf push --no-route -f manifest.yml -t 180 --strategy rolling cf map-route loki apps.internal --hostname loki + cf add-network-policy $PROD_BACKEND loki -s tanf-prod --protocol tcp --port 8080 + popd +} + +deploy_alertmanager() { + pushd alertmanager + cf push --no-route -f manifest.yml -t 180 --strategy rolling + cf map-route alertmanager apps.internal --hostname alertmanager + + # Allow prometheus to talk to alertmanager + cf add-network-policy prometheus alertmanager --protocol tcp --port 8080 + + # Add network policies to allow alertmanager to talk to all frontend apps in all environments + for app in ${DEV_FRONTEND_APPS[@]}; do + cf add-network-policy alertmanager $app -s "tanf-dev" --protocol tcp --port 80 + done + for app in ${STAGING_FRONTEND_APPS[@]}; do + cf add-network-policy alertmanager $app -s "tanf-staging" --protocol tcp --port 80 + done + cf add-network-policy alertmanager $PROD_FRONTEND --protocol tcp --port 80 + cf add-network-policy $PROD_FRONTEND alertmanager -s tanf-prod --protocol tcp --port 8080 popd } setup_extra_net_pols() { - # Add network policies to allow frontend/backend to talk to grafana/loki + # Add network policies to handle routing traffic from lower envs to the prod env cf target -o hhs-acf-ofa -s tanf-dev for i in ${!DEV_BACKEND_APPS[@]}; do cf add-network-policy ${DEV_FRONTEND_APPS[$i]} grafana -s tanf-prod --protocol tcp --port 8080 cf add-network-policy ${DEV_BACKEND_APPS[$i]} loki -s tanf-prod --protocol tcp --port 8080 + cf add-network-policy ${DEV_FRONTEND_APPS[$i]} alertmanager -s tanf-prod --protocol tcp --port 8080 done cf target -o hhs-acf-ofa -s tanf-staging for i in ${!STAGING_BACKEND_APPS[@]}; do cf add-network-policy ${STAGING_FRONTEND_APPS[$i]} grafana -s tanf-prod --protocol tcp --port 8080 cf add-network-policy ${STAGING_BACKEND_APPS[$i]} loki -s tanf-prod --protocol tcp --port 8080 + cf add-network-policy ${STAGING_FRONTEND_APPS[$i]} alertmanager -s tanf-prod --protocol tcp --port 8080 done - - cf target -o hhs-acf-ofa -s tanf-prod - cf add-network-policy $PROD_FRONTEND grafana -s tanf-prod --protocol tcp --port 8080 - cf add-network-policy $PROD_BACKEND loki -s tanf-prod --protocol tcp --port 8080 } err_help_exit() { From 1a99710c1679efac206d12c5b5e33dad106c9291 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 15 Nov 2024 13:36:49 -0500 Subject: [PATCH 03/25] - update deploy script to get sendgrid api key from backend to template into alertmanager config - Update manifest to pull alertmanager binary and start it --- tdrs-backend/plg/alertmanager/manifest.yml | 11 ++++++++--- tdrs-backend/plg/deploy.sh | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/plg/alertmanager/manifest.yml b/tdrs-backend/plg/alertmanager/manifest.yml index 80067f717..fee3420e7 100644 --- a/tdrs-backend/plg/alertmanager/manifest.yml +++ b/tdrs-backend/plg/alertmanager/manifest.yml @@ -1,10 +1,15 @@ version: 1 applications: - name: alertmanager - memory: 512M - disk_quota: 1G + memory: 128M + disk_quota: 5G instances: 1 command: | - mkdir /tmp + mkdir data + wget https://github.com/prometheus/alertmanager/releases/download/v0.27.0/alertmanager-0.27.0.linux-amd64.tar.gz + tar -zxvf alertmanager-0.27.0.linux-amd64.tar.gz + mv alertmanager-0.27.0.linux-amd64 alertmanager + rm -rf alertmanager-0.27.0.linux-amd64.tar.gz + ./alertmanager --config.file=/home/vcap/app/alertmanager.prod.yml --storage.path=/home/vcap/app/data --log.level=debug --web.external-url=http://alertmanager.apps.internal:8080/alerts --web.route-prefix=/alerts --cluster.listen-address="" buildpacks: - https://github.com/cloudfoundry/binary-buildpack diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 988f85b26..62a0420be 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -103,6 +103,10 @@ deploy_loki() { deploy_alertmanager() { pushd alertmanager + CONFIG=alertmanager.prod.yml + cp alertmanager.yml $CONFIG + SENDGRID_API_KEY=$(cf env tdp-backend-prod | grep SENDGRID | cut -d " " -f2-) + yq eval -i ".global.smtp_auth_password = \"$SENDGRID_API_KEY\"" $CONFIG cf push --no-route -f manifest.yml -t 180 --strategy rolling cf map-route alertmanager apps.internal --hostname alertmanager @@ -118,6 +122,7 @@ deploy_alertmanager() { done cf add-network-policy alertmanager $PROD_FRONTEND --protocol tcp --port 80 cf add-network-policy $PROD_FRONTEND alertmanager -s tanf-prod --protocol tcp --port 8080 + rm $CONFIG popd } From 630ff0521359bbf76649970c4212840be94b7ca1 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 15 Nov 2024 14:35:22 -0500 Subject: [PATCH 04/25] - Update manifest to bind AM to 8080 --- tdrs-backend/plg/alertmanager/manifest.yml | 3 +-- tdrs-backend/plg/deploy.sh | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/plg/alertmanager/manifest.yml b/tdrs-backend/plg/alertmanager/manifest.yml index fee3420e7..b66a4758d 100644 --- a/tdrs-backend/plg/alertmanager/manifest.yml +++ b/tdrs-backend/plg/alertmanager/manifest.yml @@ -8,8 +8,7 @@ applications: mkdir data wget https://github.com/prometheus/alertmanager/releases/download/v0.27.0/alertmanager-0.27.0.linux-amd64.tar.gz tar -zxvf alertmanager-0.27.0.linux-amd64.tar.gz - mv alertmanager-0.27.0.linux-amd64 alertmanager rm -rf alertmanager-0.27.0.linux-amd64.tar.gz - ./alertmanager --config.file=/home/vcap/app/alertmanager.prod.yml --storage.path=/home/vcap/app/data --log.level=debug --web.external-url=http://alertmanager.apps.internal:8080/alerts --web.route-prefix=/alerts --cluster.listen-address="" + ./alertmanager-0.27.0.linux-amd64/alertmanager --config.file=/home/vcap/app/alertmanager.prod.yml --web.listen-address=:8080 --storage.path=/home/vcap/app/data --log.level=debug --web.external-url=http://alertmanager.apps.internal:8080/alerts --web.route-prefix=/alerts --cluster.listen-address="" buildpacks: - https://github.com/cloudfoundry/binary-buildpack diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 62a0420be..e2345a6fc 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -185,6 +185,7 @@ if [ "$DEPLOY" == "plg" ]; then deploy_prometheus deploy_loki deploy_grafana $DB_SERVICE_NAME + deploy_alertmanager setup_extra_net_pols fi if [ "$DEPLOY" == "pg-exporter" ]; then From 358dd011b5be011b4c5c12c04b03918a51c3137a Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 15 Nov 2024 14:53:07 -0500 Subject: [PATCH 05/25] - allow rule --- .gitconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitconfig b/.gitconfig index f70bcd581..2569bbbc9 100644 --- a/.gitconfig +++ b/.gitconfig @@ -15,3 +15,4 @@ allowed = .gitconfig:.* allowed = .*DJANGO_SECRET_KEY=.* allowed = ./tdrs-backend/plg/loki/manifest.yml:* + allowed = ./tdrs-backend/plg/deploy.sh:108 From 8e3da814cf8441b417732b1eb669a6b445ff01f7 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 08:08:31 -0500 Subject: [PATCH 06/25] - disable tests - small updates to deploy script --- .circleci/build-and-test/workflows.yml | 112 ++++++++++++------------- tdrs-backend/plg/deploy.sh | 7 +- 2 files changed, 59 insertions(+), 60 deletions(-) diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml index 99d7c4fff..84ea7776d 100644 --- a/.circleci/build-and-test/workflows.yml +++ b/.circleci/build-and-test/workflows.yml @@ -3,15 +3,15 @@ when: << pipeline.parameters.build_and_test_all >> jobs: - secrets-check - - test-backend: - requires: - - secrets-check - - test-frontend: - requires: - - secrets-check - - test-e2e: - requires: - - secrets-check + # - test-backend: + # requires: + # - secrets-check + # - test-frontend: + # requires: + # - secrets-check + # - test-e2e: + # requires: + # - secrets-check ci-build-and-test-all: jobs: @@ -22,52 +22,52 @@ - main - master - /^release.*/ - - test-backend: - filters: - branches: - only: - - main - - master - - /^release.*/ - requires: - - secrets-check - - test-frontend: - filters: - branches: - only: - - main - - master - - /^release.*/ - requires: - - secrets-check - - test-e2e: - filters: - branches: - only: - - main - - master - - /^release.*/ - requires: - - secrets-check - - make_erd: # from ../util folder - filters: - branches: - only: - - develop - - master + # - test-backend: + # filters: + # branches: + # only: + # - main + # - master + # - /^release.*/ + # requires: + # - secrets-check + # - test-frontend: + # filters: + # branches: + # only: + # - main + # - master + # - /^release.*/ + # requires: + # - secrets-check + # - test-e2e: + # filters: + # branches: + # only: + # - main + # - master + # - /^release.*/ + # requires: + # - secrets-check + # - make_erd: # from ../util folder + # filters: + # branches: + # only: + # - develop + # - master - build-and-test-backend: - when: << pipeline.parameters.build_and_test_backend >> - jobs: - - secrets-check - - test-backend: - requires: - - secrets-check + # build-and-test-backend: + # when: << pipeline.parameters.build_and_test_backend >> + # jobs: + # - secrets-check + # - test-backend: + # requires: + # - secrets-check - build-and-test-frontend: - when: << pipeline.parameters.build_and_test_frontend >> - jobs: - - secrets-check - - test-frontend: - requires: - - secrets-check + # build-and-test-frontend: + # when: << pipeline.parameters.build_and_test_frontend >> + # jobs: + # - secrets-check + # - test-frontend: + # requires: + # - secrets-check diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index e2345a6fc..17fa80b65 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -151,6 +151,8 @@ err_help_exit() { exit } +pushd "$(dirname "$0")" + while getopts ":hap:u:d:" option; do case $option in h) # display Help @@ -166,9 +168,7 @@ while getopts ":hap:u:d:" option; do d) # Bind a Postgres exporter or Grafana to $DB_SERVICE_NAME DB_SERVICE_NAME=$OPTARG;; \?) # Invalid option - echo "Error: Invalid option" - help - exit;; + err_help_exit "Error: Invalid option";; esac done @@ -177,7 +177,6 @@ if [ "$#" -eq 0 ]; then exit fi -pushd "$(dirname "$0")" if [ "$DB_SERVICE_NAME" == "" ]; then err_help_exit "Error: you must include a database service name." fi From 9703ebee3b4c77c8065746ef7dbd82b59f4cfcd5 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 09:44:08 -0500 Subject: [PATCH 07/25] - fixed error in alerts config - fixed error in prometheus config --- tdrs-backend/plg/prometheus/alerts.yml | 2 +- tdrs-backend/plg/prometheus/prometheus.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/plg/prometheus/alerts.yml b/tdrs-backend/plg/prometheus/alerts.yml index affe54498..da22ba57f 100644 --- a/tdrs-backend/plg/prometheus/alerts.yml +++ b/tdrs-backend/plg/prometheus/alerts.yml @@ -32,7 +32,7 @@ groups: summary: "The {{ $labels.service }} service is down." description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 5 minutes." - alert: StagingBackendDown - expr: last_over_time(up{job=~"tdp-backend-staging""}[1m]) == 0 + expr: last_over_time(up{job=~"tdp-backend-staging"}[1m]) == 0 labels: severity: ERROR annotations: diff --git a/tdrs-backend/plg/prometheus/prometheus.yml b/tdrs-backend/plg/prometheus/prometheus.yml index 97ab67252..88a3d1b38 100644 --- a/tdrs-backend/plg/prometheus/prometheus.yml +++ b/tdrs-backend/plg/prometheus/prometheus.yml @@ -14,7 +14,7 @@ alerting: # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. rule_files: - "django-rules.yml" - - "alert.yml" + - "alerts.yml" scrape_configs: # The job name is added as a label `job=` to any timeseries scraped from this config. From 883699cfce87a6a05f5c5863fce84cab8090b053 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 09:44:40 -0500 Subject: [PATCH 08/25] - re-enable tests --- .circleci/build-and-test/workflows.yml | 112 ++++++++++++------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml index 84ea7776d..99d7c4fff 100644 --- a/.circleci/build-and-test/workflows.yml +++ b/.circleci/build-and-test/workflows.yml @@ -3,15 +3,15 @@ when: << pipeline.parameters.build_and_test_all >> jobs: - secrets-check - # - test-backend: - # requires: - # - secrets-check - # - test-frontend: - # requires: - # - secrets-check - # - test-e2e: - # requires: - # - secrets-check + - test-backend: + requires: + - secrets-check + - test-frontend: + requires: + - secrets-check + - test-e2e: + requires: + - secrets-check ci-build-and-test-all: jobs: @@ -22,52 +22,52 @@ - main - master - /^release.*/ - # - test-backend: - # filters: - # branches: - # only: - # - main - # - master - # - /^release.*/ - # requires: - # - secrets-check - # - test-frontend: - # filters: - # branches: - # only: - # - main - # - master - # - /^release.*/ - # requires: - # - secrets-check - # - test-e2e: - # filters: - # branches: - # only: - # - main - # - master - # - /^release.*/ - # requires: - # - secrets-check - # - make_erd: # from ../util folder - # filters: - # branches: - # only: - # - develop - # - master + - test-backend: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - test-frontend: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - test-e2e: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - make_erd: # from ../util folder + filters: + branches: + only: + - develop + - master - # build-and-test-backend: - # when: << pipeline.parameters.build_and_test_backend >> - # jobs: - # - secrets-check - # - test-backend: - # requires: - # - secrets-check + build-and-test-backend: + when: << pipeline.parameters.build_and_test_backend >> + jobs: + - secrets-check + - test-backend: + requires: + - secrets-check - # build-and-test-frontend: - # when: << pipeline.parameters.build_and_test_frontend >> - # jobs: - # - secrets-check - # - test-frontend: - # requires: - # - secrets-check + build-and-test-frontend: + when: << pipeline.parameters.build_and_test_frontend >> + jobs: + - secrets-check + - test-frontend: + requires: + - secrets-check From 824038f72e63a6c35278523a419b067a2bdbc84f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 10:29:20 -0500 Subject: [PATCH 09/25] - Move all networking to separate functions for convenience --- tdrs-backend/plg/deploy.sh | 68 ++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 17fa80b65..8faf6c0ab 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -58,20 +58,6 @@ deploy_grafana() { cf push --no-route -f $MANIFEST -t 180 --strategy rolling cf map-route grafana apps.internal --hostname grafana - # Add policy to allow grafana to talk to prometheus and loki - cf add-network-policy grafana prometheus --protocol tcp --port 8080 - cf add-network-policy grafana loki --protocol tcp --port 8080 - - # Add network policies to allow grafana to talk to all frontend apps in all environments - for app in ${DEV_FRONTEND_APPS[@]}; do - cf add-network-policy grafana $app -s tanf-dev --protocol tcp --port 80 - done - for app in ${STAGING_FRONTEND_APPS[@]}; do - cf add-network-policy grafana $app -s tanf-staging --protocol tcp --port 80 - done - cf add-network-policy grafana $PROD_FRONTEND --protocol tcp --port 80 - cf add-network-policy $PROD_FRONTEND grafana -s tanf-prod --protocol tcp --port 8080 - rm $DATASOURCES rm $MANIFEST popd @@ -81,15 +67,6 @@ deploy_prometheus() { pushd prometheus cf push --no-route -f manifest.yml -t 180 --strategy rolling cf map-route prometheus apps.internal --hostname prometheus - - # Add network policies to allow prometheus to talk to all backend apps in all environments - for app in ${DEV_BACKEND_APPS[@]}; do - cf add-network-policy prometheus $app -s tanf-dev --protocol tcp --port 8080 - done - for app in ${STAGING_BACKEND_APPS[@]}; do - cf add-network-policy prometheus $app -s tanf-staging --protocol tcp --port 8080 - done - cf add-network-policy prometheus $PROD_BACKEND --protocol tcp --port 8080 popd } @@ -97,7 +74,6 @@ deploy_loki() { pushd loki cf push --no-route -f manifest.yml -t 180 --strategy rolling cf map-route loki apps.internal --hostname loki - cf add-network-policy $PROD_BACKEND loki -s tanf-prod --protocol tcp --port 8080 popd } @@ -105,28 +81,52 @@ deploy_alertmanager() { pushd alertmanager CONFIG=alertmanager.prod.yml cp alertmanager.yml $CONFIG - SENDGRID_API_KEY=$(cf env tdp-backend-prod | grep SENDGRID | cut -d " " -f2-) + SENDGRID_API_KEY=$(cf env tdp-backend-raft | grep SENDGRID | cut -d " " -f2-) yq eval -i ".global.smtp_auth_password = \"$SENDGRID_API_KEY\"" $CONFIG cf push --no-route -f manifest.yml -t 180 --strategy rolling cf map-route alertmanager apps.internal --hostname alertmanager + rm $CONFIG + popd +} - # Allow prometheus to talk to alertmanager +setup_prod_net_pols() { + # Let grafana talk to prometheus and loki + cf add-network-policy grafana prometheus --protocol tcp --port 8080 + cf add-network-policy grafana loki --protocol tcp --port 8080 + + # Let prometheus talk to alertmanager and the prod backend cf add-network-policy prometheus alertmanager --protocol tcp --port 8080 + cf add-network-policy prometheus $PROD_BACKEND --protocol tcp --port 8080 - # Add network policies to allow alertmanager to talk to all frontend apps in all environments + # Let alertmanager/grafana talk to the prod frontend and vice versa + cf add-network-policy alertmanager $PROD_FRONTEND --protocol tcp --port 80 + cf add-network-policy grafana $PROD_FRONTEND --protocol tcp --port 80 + cf add-network-policy $PROD_FRONTEND alertmanager -s tanf-prod --protocol tcp --port 8080 + cf add-network-policy $PROD_FRONTEND grafana -s tanf-prod --protocol tcp --port 8080 + + # Let prod backend send logs to loki + cf add-network-policy $PROD_BACKEND loki -s tanf-prod --protocol tcp --port 8080 + + # Add network policies to allow alertmanager/grafana to talk to all frontend apps for app in ${DEV_FRONTEND_APPS[@]}; do cf add-network-policy alertmanager $app -s "tanf-dev" --protocol tcp --port 80 + cf add-network-policy grafana $app -s tanf-dev --protocol tcp --port 80 done for app in ${STAGING_FRONTEND_APPS[@]}; do cf add-network-policy alertmanager $app -s "tanf-staging" --protocol tcp --port 80 + cf add-network-policy grafana $app -s tanf-staging --protocol tcp --port 80 + done + + # Add network policies to allow prometheus to talk to all backend apps in all environments + for app in ${DEV_BACKEND_APPS[@]}; do + cf add-network-policy prometheus $app -s tanf-dev --protocol tcp --port 8080 + done + for app in ${STAGING_BACKEND_APPS[@]}; do + cf add-network-policy prometheus $app -s tanf-staging --protocol tcp --port 8080 done - cf add-network-policy alertmanager $PROD_FRONTEND --protocol tcp --port 80 - cf add-network-policy $PROD_FRONTEND alertmanager -s tanf-prod --protocol tcp --port 8080 - rm $CONFIG - popd } -setup_extra_net_pols() { +setup_dev_staging_net_pols() { # Add network policies to handle routing traffic from lower envs to the prod env cf target -o hhs-acf-ofa -s tanf-dev for i in ${!DEV_BACKEND_APPS[@]}; do @@ -141,6 +141,7 @@ setup_extra_net_pols() { cf add-network-policy ${STAGING_BACKEND_APPS[$i]} loki -s tanf-prod --protocol tcp --port 8080 cf add-network-policy ${STAGING_FRONTEND_APPS[$i]} alertmanager -s tanf-prod --protocol tcp --port 8080 done + cf target -o hhs-acf-ofa -s tanf-prod } err_help_exit() { @@ -185,7 +186,8 @@ if [ "$DEPLOY" == "plg" ]; then deploy_loki deploy_grafana $DB_SERVICE_NAME deploy_alertmanager - setup_extra_net_pols + setup_prod_net_pols + setup_dev_staging_net_pols fi if [ "$DEPLOY" == "pg-exporter" ]; then if [ "$DB_URI" == "" ]; then From da8c19d540ffadc9b9a21fb7800bd07d7aeedc66 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 10:31:23 -0500 Subject: [PATCH 10/25] - update key name location --- .gitconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitconfig b/.gitconfig index 2569bbbc9..c3ff81668 100644 --- a/.gitconfig +++ b/.gitconfig @@ -15,4 +15,4 @@ allowed = .gitconfig:.* allowed = .*DJANGO_SECRET_KEY=.* allowed = ./tdrs-backend/plg/loki/manifest.yml:* - allowed = ./tdrs-backend/plg/deploy.sh:108 + allowed = ./tdrs-backend/plg/deploy.sh:84 From fcb11563aacf770586169fafb5a0f3a3a9741fe7 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 10:36:36 -0500 Subject: [PATCH 11/25] - add missing prometheus net pols --- tdrs-backend/plg/deploy.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 8faf6c0ab..1616d235a 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -94,9 +94,11 @@ setup_prod_net_pols() { cf add-network-policy grafana prometheus --protocol tcp --port 8080 cf add-network-policy grafana loki --protocol tcp --port 8080 - # Let prometheus talk to alertmanager and the prod backend + # Let prometheus talk to alertmanager/grafana/loki/prod backend cf add-network-policy prometheus alertmanager --protocol tcp --port 8080 cf add-network-policy prometheus $PROD_BACKEND --protocol tcp --port 8080 + cf add-network-policy prometheus grafana --protocol tcp --port 8080 + cf add-network-policy prometheus loki --protocol tcp --port 8080 # Let alertmanager/grafana talk to the prod frontend and vice versa cf add-network-policy alertmanager $PROD_FRONTEND --protocol tcp --port 80 From 410b40b7de601d8693530fbdfeef3f711b031470 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 10:44:36 -0500 Subject: [PATCH 12/25] - target prod just in case --- tdrs-backend/plg/deploy.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 1616d235a..861a61e66 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -90,6 +90,9 @@ deploy_alertmanager() { } setup_prod_net_pols() { + # Target prod environment just in case + cf target -o hhs-acf-ofa -s tanf-prod + # Let grafana talk to prometheus and loki cf add-network-policy grafana prometheus --protocol tcp --port 8080 cf add-network-policy grafana loki --protocol tcp --port 8080 From 32e9e7a8c849ca0836efdeac36d4bc4e5e6d8915 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 15:04:55 -0500 Subject: [PATCH 13/25] - Add simple README for deploying PLG --- tdrs-backend/plg/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tdrs-backend/plg/README.md diff --git a/tdrs-backend/plg/README.md b/tdrs-backend/plg/README.md new file mode 100644 index 000000000..8b2f100a3 --- /dev/null +++ b/tdrs-backend/plg/README.md @@ -0,0 +1,27 @@ +# TDP PLG Stack +Before attempting to deploy the PLG stack or an postgres exporter you MUST have access to the production space in cloud.gov. + +## Deploying PLG +Before deploying the PLG stack you must have the `ADMIN_EMAILS` and `DEV_EMAILS` variables defined in your shell environment. The variables should be a comma separated string of emails, eg: `ADMIN_EMAILS="email1@email.com, email2@email.com, email3@email.com"` and `DEV_EMAILS="email4@email.com, email5@email.com, email6@email.com"`. + +Once both of the above items have been confirmed, you can target the production environment with the CF CLI and run the command below. + +``` +./deploy.sh -a -d tdp-db-prod +``` + +The command will deploy the entire PLG stack to the production environment and setup all appropriate network policies and routes. + +## Deploying a Postgres Exporter +Before deploying a postgres exporter, you need to acquire the AWS RDS database URI for the RDS instance in the environment you are deploying the exporter to. + +``` +cf env +``` + +From the output of this command find the `VCAP_SERVICES` variable. Within this variable is a JSON list of services the app you provided is bound to. Find the `aws-rds` key and copy the `uri` value to your clipboard from the `credentials` key. Then you can deploy your exporter with the command below. + +``` +./deploy.sh -p -d -u +``` +where `` MUST be one of `[dev, staging, production]`, and `` is the uri you just copied from the app's `VCAP_SERVICES` environment variable. This command also handles all of the necessary networking configuration. From cdbab0ef56864241c00ecfdc6e9c9ca1facc5742 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 18 Nov 2024 15:27:28 -0500 Subject: [PATCH 14/25] - Added email template resolution - Added email checker function --- tdrs-backend/plg/deploy.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 861a61e66..1c362333b 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -83,6 +83,8 @@ deploy_alertmanager() { cp alertmanager.yml $CONFIG SENDGRID_API_KEY=$(cf env tdp-backend-raft | grep SENDGRID | cut -d " " -f2-) yq eval -i ".global.smtp_auth_password = \"$SENDGRID_API_KEY\"" $CONFIG + yq eval -i ".receivers[0].email_configs[0].to = \"${ADMIN_EMAILS}\"" $CONFIG + yq eval -i ".receivers[1].email_configs[0].to = \"${DEV_EMAILS}\"" $CONFIG cf push --no-route -f manifest.yml -t 180 --strategy rolling cf map-route alertmanager apps.internal --hostname alertmanager rm $CONFIG @@ -149,6 +151,16 @@ setup_dev_staging_net_pols() { cf target -o hhs-acf-ofa -s tanf-prod } +check_email_vars() { + if [ "${ADMIN_EMAILS}" != "" ] && [ "${DEV_EMAILS}" != "" ]; then + echo "${ADMIN_EMAILS}" + echo "${DEV_EMAILS}" + else + echo "Missing definitions for ADMIN_EMAILS or DEV_EMAILS or both." + exit 1 + fi +} + err_help_exit() { echo $1 echo @@ -183,6 +195,8 @@ if [ "$#" -eq 0 ]; then exit fi +check_email_vars + if [ "$DB_SERVICE_NAME" == "" ]; then err_help_exit "Error: you must include a database service name." fi From 544c3116619caa99cb88ca0ab77cfe669679a5e9 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 19 Nov 2024 10:04:05 -0500 Subject: [PATCH 15/25] - Convert commadn back to prod --- tdrs-backend/plg/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 1c362333b..1d5648312 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -81,7 +81,7 @@ deploy_alertmanager() { pushd alertmanager CONFIG=alertmanager.prod.yml cp alertmanager.yml $CONFIG - SENDGRID_API_KEY=$(cf env tdp-backend-raft | grep SENDGRID | cut -d " " -f2-) + SENDGRID_API_KEY=$(cf env tdp-backend-prod | grep SENDGRID | cut -d " " -f2-) yq eval -i ".global.smtp_auth_password = \"$SENDGRID_API_KEY\"" $CONFIG yq eval -i ".receivers[0].email_configs[0].to = \"${ADMIN_EMAILS}\"" $CONFIG yq eval -i ".receivers[1].email_configs[0].to = \"${DEV_EMAILS}\"" $CONFIG From 5d5d3b10b990f4240a88b74ed1174e459325ff20 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 19 Nov 2024 11:33:12 -0500 Subject: [PATCH 16/25] - fix bug which didn't allow prometheus to load alert rules --- tdrs-backend/plg/prometheus/prometheus.local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/plg/prometheus/prometheus.local.yml b/tdrs-backend/plg/prometheus/prometheus.local.yml index 66576951b..8b0a4517d 100644 --- a/tdrs-backend/plg/prometheus/prometheus.local.yml +++ b/tdrs-backend/plg/prometheus/prometheus.local.yml @@ -15,7 +15,7 @@ alerting: # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. rule_files: - "django-rules.yml" - - "alerts.local.yml" + - "alerts.yml" # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. From 88f0028507e074a3b557d57ceb2e7174edfab35b Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 19 Nov 2024 14:06:01 -0500 Subject: [PATCH 17/25] - updated config to send resolved messages --- tdrs-backend/plg/alertmanager/alertmanager.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdrs-backend/plg/alertmanager/alertmanager.yml b/tdrs-backend/plg/alertmanager/alertmanager.yml index 9414062ae..66daa33cd 100644 --- a/tdrs-backend/plg/alertmanager/alertmanager.yml +++ b/tdrs-backend/plg/alertmanager/alertmanager.yml @@ -65,7 +65,9 @@ receivers: - name: 'admin-team-emails' email_configs: - to: '{{ admin_team_emails }}' + send_resolved: true - name: 'dev-team-emails' email_configs: - to: '{{ dev_team_emails }}' + send_resolved: true From aeb50b3f98f05d0aea559aba45a0eb25aea31df3 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Nov 2024 14:43:55 -0500 Subject: [PATCH 18/25] - Updated STTApiView to STTApiViewSet --- tdrs-backend/tdpservice/stts/urls.py | 8 +++++++- tdrs-backend/tdpservice/stts/views.py | 6 ++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/stts/urls.py b/tdrs-backend/tdpservice/stts/urls.py index 9fb01c24e..ae1edbb1a 100644 --- a/tdrs-backend/tdpservice/stts/urls.py +++ b/tdrs-backend/tdpservice/stts/urls.py @@ -1,10 +1,16 @@ """Routing for STTs.""" +from rest_framework.routers import DefaultRouter from django.urls import path from . import views +router = DefaultRouter() + +router.register("", views.STTApiViewSet) + urlpatterns = [ path("by_region", views.RegionAPIView.as_view(), name="stts-by-region"), path("alpha", views.STTApiAlphaView.as_view(), name="stts-alpha"), - path("", views.STTApiView.as_view(), name="stts"), ] + +urlpatterns += router.urls diff --git a/tdrs-backend/tdpservice/stts/views.py b/tdrs-backend/tdpservice/stts/views.py index 83a589d3c..5b1986089 100644 --- a/tdrs-backend/tdpservice/stts/views.py +++ b/tdrs-backend/tdpservice/stts/views.py @@ -2,7 +2,7 @@ import logging from django.db.models import Prefetch -from rest_framework import generics +from rest_framework import generics, mixins, viewsets from rest_framework.permissions import IsAuthenticated from tdpservice.stts.models import Region, STT from .serializers import RegionSerializer, STTSerializer @@ -30,7 +30,9 @@ class STTApiAlphaView(generics.ListAPIView): serializer_class = STTSerializer -class STTApiView(generics.ListAPIView): +class STTApiViewSet(mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet): """Simple view to get all STTs.""" pagination_class = None From dcda51abc8e80ed1f4ba49f3dfa5a273852893e5 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Nov 2024 15:16:51 -0500 Subject: [PATCH 19/25] - overriding retrieve method to key off of stt name instead of pk --- tdrs-backend/tdpservice/stts/views.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/stts/views.py b/tdrs-backend/tdpservice/stts/views.py index 5b1986089..bc0607e62 100644 --- a/tdrs-backend/tdpservice/stts/views.py +++ b/tdrs-backend/tdpservice/stts/views.py @@ -2,8 +2,9 @@ import logging from django.db.models import Prefetch -from rest_framework import generics, mixins, viewsets +from rest_framework import generics, mixins, status, viewsets from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response from tdpservice.stts.models import Region, STT from .serializers import RegionSerializer, STTSerializer @@ -39,3 +40,14 @@ class STTApiViewSet(mixins.ListModelMixin, permission_classes = [IsAuthenticated] queryset = STT.objects serializer_class = STTSerializer + + def retrieve(self, request, pk=None): + """Return a specific user.""" + try: + stt = self.queryset.get(name=pk) + self.check_object_permissions(request, stt) + serializer = self.get_serializer_class()(stt) + return Response(serializer.data) + except Exception: + logger.exception(f"Caught exception trying to get STT with name {pk}.") + return Response(status=status.HTTP_404_NOT_FOUND) From f1529689e4c1fb291d7b6a117738348f91d25f3c Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Nov 2024 15:47:27 -0500 Subject: [PATCH 20/25] - MVP of limiting submission boxes based on what an stt should be submitting --- tdrs-frontend/src/actions/reports.js | 9 +++++++-- .../SubmissionHistory/SubmissionHistory.jsx | 5 +++-- .../src/components/UploadReport/UploadReport.jsx | 7 ++++++- tdrs-frontend/src/reducers/reports.js | 11 ++++++----- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tdrs-frontend/src/actions/reports.js b/tdrs-frontend/src/actions/reports.js index 8ecb8839e..dcb459e89 100644 --- a/tdrs-frontend/src/actions/reports.js +++ b/tdrs-frontend/src/actions/reports.js @@ -269,8 +269,13 @@ export const SET_SELECTED_YEAR = 'SET_SELECTED_YEAR' export const SET_SELECTED_QUARTER = 'SET_SELECTED_QUARTER' export const SET_FILE_TYPE = 'SET_FILE_TYPE' -export const setStt = (stt) => (dispatch) => { - dispatch({ type: SET_SELECTED_STT, payload: { stt } }) +export const setStt = (stt) => async (dispatch) => { + const URL = `${process.env.REACT_APP_BACKEND_URL}/stts/${stt}` + const { data } = await axiosInstance.get(URL, { + withCredentials: true, + }) + const newUploadSections = Object.keys(data.filenames) + dispatch({ type: SET_SELECTED_STT, payload: { stt, newUploadSections } }) } export const setYear = (year) => (dispatch) => { dispatch({ type: SET_SELECTED_YEAR, payload: { year } }) diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index a1e28b7c0..5ddfb770f 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -1,8 +1,6 @@ import React from 'react' import PropTypes from 'prop-types' -import classNames from 'classnames' import { useDispatch, useSelector } from 'react-redux' -import { fileUploadSections } from '../../reducers/reports' import Paginator from '../Paginator' import { getAvailableFileList } from '../../actions/reports' import { useEffect } from 'react' @@ -64,6 +62,9 @@ const SubmissionHistory = ({ filterValues }) => { const dispatch = useDispatch() const [hasFetchedFiles, setHasFetchedFiles] = useState(false) const { files } = useSelector((state) => state.reports) + const fileUploadSections = useSelector( + (state) => state.reports.fileUploadSections + ) useEffect(() => { if (!hasFetchedFiles) { diff --git a/tdrs-frontend/src/components/UploadReport/UploadReport.jsx b/tdrs-frontend/src/components/UploadReport/UploadReport.jsx index 9e51c11a7..fa5b085a5 100644 --- a/tdrs-frontend/src/components/UploadReport/UploadReport.jsx +++ b/tdrs-frontend/src/components/UploadReport/UploadReport.jsx @@ -8,7 +8,6 @@ import Button from '../Button' import FileUpload from '../FileUpload' import { submit } from '../../actions/reports' import { useEventLogger } from '../../utils/eventLogger' -import { fileUploadSections } from '../../reducers/reports' function UploadReport({ handleCancel, stt }) { // The currently selected year from the reportingYears dropdown @@ -20,6 +19,12 @@ function UploadReport({ handleCancel, stt }) { // The set of uploaded files in our Redux state const files = useSelector((state) => state.reports.submittedFiles) + + // The set of sections the STT can report for + const fileUploadSections = useSelector( + (state) => state.reports.fileUploadSections + ) + // The logged in user in our Redux state const user = useSelector((state) => state.auth.user) diff --git a/tdrs-frontend/src/reducers/reports.js b/tdrs-frontend/src/reducers/reports.js index 9de986716..9f83843c1 100644 --- a/tdrs-frontend/src/reducers/reports.js +++ b/tdrs-frontend/src/reducers/reports.js @@ -23,7 +23,7 @@ const getFile = (files, section) => .sort((a, b) => b.id - a.id) .find((currentFile) => currentFile.section.includes(section)) -export const fileUploadSections = [ +export const defaultFileUploadSections = [ 'Active Case Data', 'Closed Case Data', 'Aggregate Data', @@ -73,7 +73,8 @@ export const serializeApiDataFile = (dataFile) => ({ const initialState = { files: [], - submittedFiles: fileUploadSections.map((section) => ({ + fileUploadSections: defaultFileUploadSections, + submittedFiles: defaultFileUploadSections.map((section) => ({ section, fileName: null, error: null, @@ -116,7 +117,7 @@ const reports = (state = initialState, action) => { ...state, isLoadingCurrentSubmission: false, currentSubmissionError: null, - submittedFiles: fileUploadSections.map((section) => { + submittedFiles: state.fileUploadSections.map((section) => { const file = getFile(data, section) if (file) { return serializeApiDataFile(file) @@ -201,8 +202,8 @@ const reports = (state = initialState, action) => { return { ...state, year } } case SET_SELECTED_STT: { - const { stt } = payload - return { ...state, stt } + const { stt, newUploadSections } = payload + return { ...state, stt, fileUploadSections: newUploadSections } } case SET_SELECTED_QUARTER: { const { quarter } = payload From 866f9b92268c47436657d045bad9b53fb7aa066a Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Nov 2024 15:58:06 -0500 Subject: [PATCH 21/25] Revert "- MVP of limiting submission boxes based on what an stt should be submitting" This reverts commit f1529689e4c1fb291d7b6a117738348f91d25f3c. --- tdrs-frontend/src/actions/reports.js | 9 ++------- .../SubmissionHistory/SubmissionHistory.jsx | 5 ++--- .../src/components/UploadReport/UploadReport.jsx | 7 +------ tdrs-frontend/src/reducers/reports.js | 11 +++++------ 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/tdrs-frontend/src/actions/reports.js b/tdrs-frontend/src/actions/reports.js index dcb459e89..8ecb8839e 100644 --- a/tdrs-frontend/src/actions/reports.js +++ b/tdrs-frontend/src/actions/reports.js @@ -269,13 +269,8 @@ export const SET_SELECTED_YEAR = 'SET_SELECTED_YEAR' export const SET_SELECTED_QUARTER = 'SET_SELECTED_QUARTER' export const SET_FILE_TYPE = 'SET_FILE_TYPE' -export const setStt = (stt) => async (dispatch) => { - const URL = `${process.env.REACT_APP_BACKEND_URL}/stts/${stt}` - const { data } = await axiosInstance.get(URL, { - withCredentials: true, - }) - const newUploadSections = Object.keys(data.filenames) - dispatch({ type: SET_SELECTED_STT, payload: { stt, newUploadSections } }) +export const setStt = (stt) => (dispatch) => { + dispatch({ type: SET_SELECTED_STT, payload: { stt } }) } export const setYear = (year) => (dispatch) => { dispatch({ type: SET_SELECTED_YEAR, payload: { year } }) diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index 5ddfb770f..a1e28b7c0 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -1,6 +1,8 @@ import React from 'react' import PropTypes from 'prop-types' +import classNames from 'classnames' import { useDispatch, useSelector } from 'react-redux' +import { fileUploadSections } from '../../reducers/reports' import Paginator from '../Paginator' import { getAvailableFileList } from '../../actions/reports' import { useEffect } from 'react' @@ -62,9 +64,6 @@ const SubmissionHistory = ({ filterValues }) => { const dispatch = useDispatch() const [hasFetchedFiles, setHasFetchedFiles] = useState(false) const { files } = useSelector((state) => state.reports) - const fileUploadSections = useSelector( - (state) => state.reports.fileUploadSections - ) useEffect(() => { if (!hasFetchedFiles) { diff --git a/tdrs-frontend/src/components/UploadReport/UploadReport.jsx b/tdrs-frontend/src/components/UploadReport/UploadReport.jsx index fa5b085a5..9e51c11a7 100644 --- a/tdrs-frontend/src/components/UploadReport/UploadReport.jsx +++ b/tdrs-frontend/src/components/UploadReport/UploadReport.jsx @@ -8,6 +8,7 @@ import Button from '../Button' import FileUpload from '../FileUpload' import { submit } from '../../actions/reports' import { useEventLogger } from '../../utils/eventLogger' +import { fileUploadSections } from '../../reducers/reports' function UploadReport({ handleCancel, stt }) { // The currently selected year from the reportingYears dropdown @@ -19,12 +20,6 @@ function UploadReport({ handleCancel, stt }) { // The set of uploaded files in our Redux state const files = useSelector((state) => state.reports.submittedFiles) - - // The set of sections the STT can report for - const fileUploadSections = useSelector( - (state) => state.reports.fileUploadSections - ) - // The logged in user in our Redux state const user = useSelector((state) => state.auth.user) diff --git a/tdrs-frontend/src/reducers/reports.js b/tdrs-frontend/src/reducers/reports.js index 9f83843c1..9de986716 100644 --- a/tdrs-frontend/src/reducers/reports.js +++ b/tdrs-frontend/src/reducers/reports.js @@ -23,7 +23,7 @@ const getFile = (files, section) => .sort((a, b) => b.id - a.id) .find((currentFile) => currentFile.section.includes(section)) -export const defaultFileUploadSections = [ +export const fileUploadSections = [ 'Active Case Data', 'Closed Case Data', 'Aggregate Data', @@ -73,8 +73,7 @@ export const serializeApiDataFile = (dataFile) => ({ const initialState = { files: [], - fileUploadSections: defaultFileUploadSections, - submittedFiles: defaultFileUploadSections.map((section) => ({ + submittedFiles: fileUploadSections.map((section) => ({ section, fileName: null, error: null, @@ -117,7 +116,7 @@ const reports = (state = initialState, action) => { ...state, isLoadingCurrentSubmission: false, currentSubmissionError: null, - submittedFiles: state.fileUploadSections.map((section) => { + submittedFiles: fileUploadSections.map((section) => { const file = getFile(data, section) if (file) { return serializeApiDataFile(file) @@ -202,8 +201,8 @@ const reports = (state = initialState, action) => { return { ...state, year } } case SET_SELECTED_STT: { - const { stt, newUploadSections } = payload - return { ...state, stt, fileUploadSections: newUploadSections } + const { stt } = payload + return { ...state, stt } } case SET_SELECTED_QUARTER: { const { quarter } = payload From d761dad331c16a189d996f4bd4a1a0b6e2398ca7 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Nov 2024 15:58:16 -0500 Subject: [PATCH 22/25] Revert "- overriding retrieve method to key off of stt name instead of pk" This reverts commit dcda51abc8e80ed1f4ba49f3dfa5a273852893e5. --- tdrs-backend/tdpservice/stts/views.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tdrs-backend/tdpservice/stts/views.py b/tdrs-backend/tdpservice/stts/views.py index bc0607e62..5b1986089 100644 --- a/tdrs-backend/tdpservice/stts/views.py +++ b/tdrs-backend/tdpservice/stts/views.py @@ -2,9 +2,8 @@ import logging from django.db.models import Prefetch -from rest_framework import generics, mixins, status, viewsets +from rest_framework import generics, mixins, viewsets from rest_framework.permissions import IsAuthenticated -from rest_framework.response import Response from tdpservice.stts.models import Region, STT from .serializers import RegionSerializer, STTSerializer @@ -40,14 +39,3 @@ class STTApiViewSet(mixins.ListModelMixin, permission_classes = [IsAuthenticated] queryset = STT.objects serializer_class = STTSerializer - - def retrieve(self, request, pk=None): - """Return a specific user.""" - try: - stt = self.queryset.get(name=pk) - self.check_object_permissions(request, stt) - serializer = self.get_serializer_class()(stt) - return Response(serializer.data) - except Exception: - logger.exception(f"Caught exception trying to get STT with name {pk}.") - return Response(status=status.HTTP_404_NOT_FOUND) From 5d959ef97fdccb88a71c68e5ee9a32b82ff47904 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 20 Nov 2024 15:58:26 -0500 Subject: [PATCH 23/25] Revert "- Updated STTApiView to STTApiViewSet" This reverts commit aeb50b3f98f05d0aea559aba45a0eb25aea31df3. --- tdrs-backend/tdpservice/stts/urls.py | 8 +------- tdrs-backend/tdpservice/stts/views.py | 6 ++---- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tdrs-backend/tdpservice/stts/urls.py b/tdrs-backend/tdpservice/stts/urls.py index ae1edbb1a..9fb01c24e 100644 --- a/tdrs-backend/tdpservice/stts/urls.py +++ b/tdrs-backend/tdpservice/stts/urls.py @@ -1,16 +1,10 @@ """Routing for STTs.""" -from rest_framework.routers import DefaultRouter from django.urls import path from . import views -router = DefaultRouter() - -router.register("", views.STTApiViewSet) - urlpatterns = [ path("by_region", views.RegionAPIView.as_view(), name="stts-by-region"), path("alpha", views.STTApiAlphaView.as_view(), name="stts-alpha"), + path("", views.STTApiView.as_view(), name="stts"), ] - -urlpatterns += router.urls diff --git a/tdrs-backend/tdpservice/stts/views.py b/tdrs-backend/tdpservice/stts/views.py index 5b1986089..83a589d3c 100644 --- a/tdrs-backend/tdpservice/stts/views.py +++ b/tdrs-backend/tdpservice/stts/views.py @@ -2,7 +2,7 @@ import logging from django.db.models import Prefetch -from rest_framework import generics, mixins, viewsets +from rest_framework import generics from rest_framework.permissions import IsAuthenticated from tdpservice.stts.models import Region, STT from .serializers import RegionSerializer, STTSerializer @@ -30,9 +30,7 @@ class STTApiAlphaView(generics.ListAPIView): serializer_class = STTSerializer -class STTApiViewSet(mixins.ListModelMixin, - mixins.RetrieveModelMixin, - viewsets.GenericViewSet): +class STTApiView(generics.ListAPIView): """Simple view to get all STTs.""" pagination_class = None From 57075c3cdbe07ca07539dad27e0244003c184f3e Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 9 Dec 2024 15:18:10 -0500 Subject: [PATCH 24/25] - Added extra matchers to ensure dev and admins are getting basically all the alerts for now --- tdrs-backend/plg/alertmanager/alertmanager.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/plg/alertmanager/alertmanager.yml b/tdrs-backend/plg/alertmanager/alertmanager.yml index 66daa33cd..77c981eb8 100644 --- a/tdrs-backend/plg/alertmanager/alertmanager.yml +++ b/tdrs-backend/plg/alertmanager/alertmanager.yml @@ -39,13 +39,22 @@ route: # The child route trees. routes: - # This routes performs a regular expression match on alert labels to - # catch alerts that are related to a list of services. + # Only alert dev team of uptime issues - matchers: - alertname=~"UpTime" receiver: dev-team-emails group_wait: 30m + # Send all severity CRITICAL/ERROR alerts to OFA admins and TDP Devs + - matchers: + - severity=~"ERROR|CRITICAL" + receiver: admin-team-emails + continue: true + - matchers: + - severity=~"ERROR|CRITICAL" + receiver: dev-team-emails + continue: true + # Inhibition rules allow to mute a set of alerts given that another alert is # firing. # We use this to mute any warning-level notifications if the same alert is From 8f5a4527c5c5e3e52910af3cd4084b81c5fa96cd Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 9 Dec 2024 15:48:07 -0500 Subject: [PATCH 25/25] - Add convenience flag for prometheus deploy - use correct grafana route in prometheus scrape config --- tdrs-backend/plg/prometheus/manifest.yml | 2 +- tdrs-backend/plg/prometheus/prometheus.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/plg/prometheus/manifest.yml b/tdrs-backend/plg/prometheus/manifest.yml index da68fdbd3..d5c72d72f 100644 --- a/tdrs-backend/plg/prometheus/manifest.yml +++ b/tdrs-backend/plg/prometheus/manifest.yml @@ -11,6 +11,6 @@ applications: mv ./prometheus-2.54.1.linux-amd64/prometheus ./prometheus && mv ./prometheus-2.54.1.linux-amd64/promtool ./promtool && rm -rf ./prometheus-2.54.1.linux-amd64 && rm -rf prometheus-2.54.1.linux-amd64.tar.gz && - ./prometheus --config.file=/home/vcap/app/prometheus.yml --storage.tsdb.path=/home/vcap/app/prometheus-data --storage.tsdb.retention.time=30d --storage.tsdb.retention.size=6GB --web.listen-address="0.0.0.0:8080" + ./prometheus --config.file=/home/vcap/app/prometheus.yml --storage.tsdb.path=/home/vcap/app/prometheus-data --storage.tsdb.retention.time=30d --storage.tsdb.retention.size=6GB --web.listen-address="0.0.0.0:8080" --web.enable-lifecycle buildpacks: - https://github.com/cloudfoundry/binary-buildpack diff --git a/tdrs-backend/plg/prometheus/prometheus.yml b/tdrs-backend/plg/prometheus/prometheus.yml index 88a3d1b38..55241934b 100644 --- a/tdrs-backend/plg/prometheus/prometheus.yml +++ b/tdrs-backend/plg/prometheus/prometheus.yml @@ -93,7 +93,7 @@ scrape_configs: - job_name: grafana metrics_path: /grafana/metrics static_configs: - - targets: ["grafana.app.cloud.gov:8080"] + - targets: ["grafana.apps.internal:8080"] labels: service: "grafana" env: "production"