From 24f3884a28d3aeae3c3ee25becb30ddd93398b50 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 12 Nov 2024 15:44:03 -0500 Subject: [PATCH 01/18] - Update networking in deploy.sh - Update .ini to use internal domain - Add nginx config for proxying prod Grafana --- tdrs-backend/plg/deploy.sh | 11 +++--- tdrs-backend/plg/grafana/custom.ini | 2 +- tdrs-backend/plg/grafana/manifest.yml | 4 +-- tdrs-frontend/nginx/cloud.gov/locations.conf | 36 ++++++++++++++++++++ 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 11adaebdd..65b90c416 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -19,6 +19,7 @@ deploy_pg_exporter() { cp manifest.yml $MANIFEST APP_NAME="pg-exporter-$1" + EXPORTER_SPACE=$(cf target | grep -Eo "tanf(.*)") yq eval -i ".applications[0].name = \"$APP_NAME\"" $MANIFEST yq eval -i ".applications[0].env.DATA_SOURCE_NAME = \"$2\"" $MANIFEST @@ -27,9 +28,10 @@ deploy_pg_exporter() { cf push --no-route -f $MANIFEST -t 180 --strategy rolling cf map-route $APP_NAME apps.internal --hostname $APP_NAME - # Add policy to allow prometheus to talk to pg-exporter + # Add policy to allow prometheus to talk to pg-exporter regardless of environment # TODO: this logic needs to be updated to allow routing accross spaces based on where we want PLG to live. - cf add-network-policy prometheus $APP_NAME -s "tanf-dev" --protocol tcp --port 9187 + cf target -o hhs-acf-ofa -s tanf-prod + cf add-network-policy prometheus $APP_NAME -s "$EXPORTER_SPACE" --protocol tcp --port 9187 rm $MANIFEST popd } @@ -47,9 +49,9 @@ deploy_grafana() { yq eval -i ".applications[0].services[0] = \"$1\"" $MANIFEST cf push --no-route -f $MANIFEST -t 180 --strategy rolling - # cf map-route $APP_NAME apps.internal --hostname $APP_NAME + cf map-route $APP_NAME apps.internal --hostname $APP_NAME # Give Grafana a public route for now. Might be able to swap to internal route later. - cf map-route "$APP_NAME" app.cloud.gov --hostname "${APP_NAME}" + # cf map-route "$APP_NAME" app.cloud.gov --hostname "${APP_NAME}" # Add policy to allow grafana to talk to prometheus and loki cf add-network-policy $APP_NAME prometheus --protocol tcp --port 8080 @@ -97,6 +99,7 @@ while getopts ":hap:u:d:" option; do DB_SERVICE_NAME=$OPTARG;; \?) # Invalid option echo "Error: Invalid option" + help exit;; esac done diff --git a/tdrs-backend/plg/grafana/custom.ini b/tdrs-backend/plg/grafana/custom.ini index 7d8be7d57..f6a9f81e1 100644 --- a/tdrs-backend/plg/grafana/custom.ini +++ b/tdrs-backend/plg/grafana/custom.ini @@ -40,7 +40,7 @@ http_addr = http_port = 8080 # The public facing domain name used to access grafana from a browser -domain = app.cloud.gov +domain = apps.internal # Redirect to correct domain if host header does not match domain # Prevents DNS rebinding attacks diff --git a/tdrs-backend/plg/grafana/manifest.yml b/tdrs-backend/plg/grafana/manifest.yml index 2f796535f..1d6be5f3a 100644 --- a/tdrs-backend/plg/grafana/manifest.yml +++ b/tdrs-backend/plg/grafana/manifest.yml @@ -5,12 +5,12 @@ applications: disk_quota: 2G instances: 1 env: - GF_PATHS_PROVISIONING: "/conf/provisioning" + GF_PATHS_PROVISIONING: "conf/provisioning" GF_PATHS_CONFIG: "/home/vcap/app/custom.ini" GF_PATHS_HOME: "/home/vcap/app/grafana-v11.2.0" GF_PATHS_DATA: "/home/vcap/app/data" GF_PATHS_LOGS: "/home/vcap/app/logs" - GF_PATHS_PLUGINS: "/conf/provisioning/plugins" + GF_PATHS_PLUGINS: "conf/provisioning/plugins" GF_SERVER_HTTP_PORT: 8080 GF_DATABASE_TYPE: postgres GF_DATABASE_SSL_MODE: require diff --git a/tdrs-frontend/nginx/cloud.gov/locations.conf b/tdrs-frontend/nginx/cloud.gov/locations.conf index 592063439..83b964bfc 100644 --- a/tdrs-frontend/nginx/cloud.gov/locations.conf +++ b/tdrs-frontend/nginx/cloud.gov/locations.conf @@ -60,6 +60,42 @@ location = /kibana_auth_check { send_timeout 900; } +location /grafana/ { + auth_request /grafana_auth_check; + auth_request_set $auth_status $upstream_status; + + set $grafana http://grafana.apps.internal:8080$request_uri; + proxy_pass $grafana; + proxy_set_header Host $host:3000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + + proxy_connect_timeout 300; + proxy_read_timeout 300; + proxy_send_timeout 300; + send_timeout 900; + proxy_buffer_size 4k; +} + +location = /grafana_auth_check { + internal; + set $endpoint http://{{env "BACKEND_HOST"}}.apps.internal::8080/grafana_auth_check/; + proxy_pass $endpoint$1$is_args$args; + proxy_set_header Host $host:3000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + + proxy_connect_timeout 300; + proxy_read_timeout 300; + proxy_send_timeout 300; + send_timeout 900; + proxy_pass_header x-csrftoken; +} + if ($request_method ~ ^(TRACE|OPTION)$) { return 405; } From 269b59e53d49d26caaab5bcfa21161b5f0a7be53 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 12 Nov 2024 15:45:36 -0500 Subject: [PATCH 02/18] - add missing arg for grafana deploy --- tdrs-backend/plg/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 65b90c416..9676a6966 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -116,7 +116,7 @@ fi if [ "$DEPLOY" == "plg" ]; then deploy_prometheus deploy_loki - deploy_grafana + deploy_grafana $DB_SERVICE_NAME fi if [ "$DEPLOY" == "pg-exporter" ]; then if [ "$DB_URI" == "" ]; then From b40c243dabae3c3ce3b26b7a460b423e2d5dec61 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 12 Nov 2024 16:01:11 -0500 Subject: [PATCH 03/18] - Update deploy script networking to test in dev initially --- scripts/deploy-backend.sh | 9 ++++++++- scripts/deploy-frontend.sh | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 3cf5f1030..6347b4070 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -123,7 +123,14 @@ update_plg_networking() { cf target -o hhs-acf-ofa -s "$CF_SPACE" # Promtial needs to send logs to Loki - cf add-network-policy "$CGAPPNAME_BACKEND" loki -s "tanf-dev" --protocol tcp --port 8080 + cf add-network-policy "$CGAPPNAME_BACKEND" loki -s "tanf-prod" --protocol tcp --port 8080 + + # Add network policy allowing Grafana to talk to the backend and to allow the backend to talk to Grafana + # TODO: to avoid having to target the prod space, move the network policies from grafana to an app to plg/deploy.sh + # TODO: make sure netpols tunnel to grafana in poduction after dev testing + # cf add-network-policy "$CGAPPNAME_BACKEND" "grafana" --protocol tcp --port 8080 + cf add-network-policy "$CGAPPNAME_FRONTEND" "grafana" --protocol tcp --port 8080 + cf add-network-policy "grafana" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80 } update_backend() diff --git a/scripts/deploy-frontend.sh b/scripts/deploy-frontend.sh index fd7206929..ddb0bcdea 100755 --- a/scripts/deploy-frontend.sh +++ b/scripts/deploy-frontend.sh @@ -13,7 +13,6 @@ CF_SPACE=${5} ENVIRONMENT=${6} env=${CF_SPACE#"tanf-"} -frontend_app_name=$(echo $CGHOSTNAME_FRONTEND | cut -d"-" -f3) # Update the Kibana name to include the environment KIBANA_BASE_URL="${CGAPPNAME_KIBANA}-${env}.apps.internal" From 6fd3a9b532c96cd99dec65aeab470ee8e67b4db5 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 13 Nov 2024 10:11:35 -0500 Subject: [PATCH 04/18] - remove prod tunnel - turn off tests --- .circleci/build-and-test/workflows.yml | 112 ++++++++++++------------- scripts/deploy-backend.sh | 2 +- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml index 99d7c4fff..1081c40e7 100644 --- a/.circleci/build-and-test/workflows.yml +++ b/.circleci/build-and-test/workflows.yml @@ -3,15 +3,15 @@ when: << pipeline.parameters.build_and_test_all >> jobs: - secrets-check - - test-backend: - requires: - - secrets-check - - test-frontend: - requires: - - secrets-check - - test-e2e: - requires: - - secrets-check + # - test-backend: + # requires: + # - secrets-check + # - test-frontend: + # requires: + # - secrets-check + # - test-e2e: + # requires: + # - secrets-check ci-build-and-test-all: jobs: @@ -22,52 +22,52 @@ - main - master - /^release.*/ - - test-backend: - filters: - branches: - only: - - main - - master - - /^release.*/ - requires: - - secrets-check - - test-frontend: - filters: - branches: - only: - - main - - master - - /^release.*/ - requires: - - secrets-check - - test-e2e: - filters: - branches: - only: - - main - - master - - /^release.*/ - requires: - - secrets-check - - make_erd: # from ../util folder - filters: - branches: - only: - - develop - - master + # - test-backend: + # filters: + # branches: + # only: + # - main + # - master + # - /^release.*/ + # requires: + # - secrets-check + # - test-frontend: + # filters: + # branches: + # only: + # - main + # - master + # - /^release.*/ + # requires: + # - secrets-check + # - test-e2e: + # filters: + # branches: + # only: + # - main + # - master + # - /^release.*/ + # requires: + # - secrets-check + # - make_erd: # from ../util folder + # filters: + # branches: + # only: + # - develop + # - master - build-and-test-backend: - when: << pipeline.parameters.build_and_test_backend >> - jobs: - - secrets-check - - test-backend: - requires: - - secrets-check + # build-and-test-backend: + # when: << pipeline.parameters.build_and_test_backend >> + # jobs: + # - secrets-check + # - test-backend: + # requires: + # - secrets-check - build-and-test-frontend: - when: << pipeline.parameters.build_and_test_frontend >> - jobs: - - secrets-check - - test-frontend: - requires: - - secrets-check + # build-and-test-frontend: + # when: << pipeline.parameters.build_and_test_frontend >> + # jobs: + # - secrets-check + # - test-frontend: + # requires: + # - secrets-check diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 6347b4070..ea5b607b2 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -123,7 +123,7 @@ update_plg_networking() { cf target -o hhs-acf-ofa -s "$CF_SPACE" # Promtial needs to send logs to Loki - cf add-network-policy "$CGAPPNAME_BACKEND" loki -s "tanf-prod" --protocol tcp --port 8080 + cf add-network-policy "$CGAPPNAME_BACKEND" loki --protocol tcp --port 8080 # Add network policy allowing Grafana to talk to the backend and to allow the backend to talk to Grafana # TODO: to avoid having to target the prod space, move the network policies from grafana to an app to plg/deploy.sh From f9e3b7aebaff8250016918aa3476ac2abb472edb Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 13 Nov 2024 11:16:33 -0500 Subject: [PATCH 05/18] - fix domain - fix location error --- tdrs-backend/plg/grafana/custom.ini | 2 +- tdrs-frontend/nginx/cloud.gov/locations.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/plg/grafana/custom.ini b/tdrs-backend/plg/grafana/custom.ini index f6a9f81e1..ff2b035a5 100644 --- a/tdrs-backend/plg/grafana/custom.ini +++ b/tdrs-backend/plg/grafana/custom.ini @@ -40,7 +40,7 @@ http_addr = http_port = 8080 # The public facing domain name used to access grafana from a browser -domain = apps.internal +domain = grafana.apps.internal # Redirect to correct domain if host header does not match domain # Prevents DNS rebinding attacks diff --git a/tdrs-frontend/nginx/cloud.gov/locations.conf b/tdrs-frontend/nginx/cloud.gov/locations.conf index 83b964bfc..37bd413e2 100644 --- a/tdrs-frontend/nginx/cloud.gov/locations.conf +++ b/tdrs-frontend/nginx/cloud.gov/locations.conf @@ -80,7 +80,7 @@ location /grafana/ { location = /grafana_auth_check { internal; - set $endpoint http://{{env "BACKEND_HOST"}}.apps.internal::8080/grafana_auth_check/; + set $endpoint http://{{env "BACKEND_HOST"}}.apps.internal:8080/grafana_auth_check/; proxy_pass $endpoint$1$is_args$args; proxy_set_header Host $host:3000; proxy_set_header X-Real-IP $remote_addr; From 4e50d9d9227c30dffe59cbf5cb7f6cfd5852dc3c Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 13 Nov 2024 15:20:16 -0500 Subject: [PATCH 06/18] - Remove prod networking from deploy-backend.sh - update plg/deploy.sh to handle all prod based networking since net-pols are persistent - Remove unnecessary scrape config from prometheus config --- scripts/deploy-backend.sh | 20 ++++--------- tdrs-backend/plg/deploy.sh | 33 ++++++++++++++++++++-- tdrs-backend/plg/prometheus/prometheus.yml | 4 --- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index ea5b607b2..5b5fcb243 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -116,21 +116,11 @@ prepare_promtail() { } update_plg_networking() { - # Need to switch the space after deploy since we're not always in dev space to handle specific networking from dev - # PLG apps to the correct backend app. - cf target -o hhs-acf-ofa -s tanf-dev - cf add-network-policy prometheus "$CGAPPNAME_BACKEND" -s "$CF_SPACE" --protocol tcp --port 8080 - cf target -o hhs-acf-ofa -s "$CF_SPACE" - - # Promtial needs to send logs to Loki - cf add-network-policy "$CGAPPNAME_BACKEND" loki --protocol tcp --port 8080 - - # Add network policy allowing Grafana to talk to the backend and to allow the backend to talk to Grafana - # TODO: to avoid having to target the prod space, move the network policies from grafana to an app to plg/deploy.sh - # TODO: make sure netpols tunnel to grafana in poduction after dev testing - # cf add-network-policy "$CGAPPNAME_BACKEND" "grafana" --protocol tcp --port 8080 - cf add-network-policy "$CGAPPNAME_FRONTEND" "grafana" --protocol tcp --port 8080 - cf add-network-policy "grafana" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80 + # Promtail needs to send logs to Loki + cf add-network-policy "$CGAPPNAME_BACKEND" loki -s tanf-prod --protocol tcp --port 8080 + + # Add network policy allowing frontend to talk to Grafana + cf add-network-policy "$CGAPPNAME_FRONTEND" "grafana" -s tanf-prod --protocol tcp --port 8080 } update_backend() diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 9676a6966..b69a678fd 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -1,13 +1,21 @@ #!/bin/bash set -e +DEV_BACKEND_APPS = ("tdp-backend-raft" "tdp-backend-qasp" "tdp-backend-a11y") +STAGING_BACKEND_APPS = ("tdp-backend-develop" "tdp-backend-staging") +PROD_BACKEND = "tdp-backend-prod" + +DEV_FRONTEND_APPS = ("tdp-frontend-raft" "tdp-frontend-qasp" "tdp-frontend-a11y") +STAGING_FRONTEND_APPS = ("tdp-frontend-develop" "tdp-frontend-staging") +PROD_FRONTEND = "tdp-frontend-prod" + help() { echo "Deploy the PLG stack or a Postgres exporter to the Cloud Foundry space you're currently authenticated in." echo "Syntax: deploy.sh [-h|a|p|u|d]" echo "Options:" echo "h Print this help message." echo "a Deploy the entire PLG stack." - echo "p Deploy a postgres exporter. Requires -u and -d" + echo "p Deploy a postgres exporter, expects the environment name (dev, staging, prod) to be passed with switch. Requires -u and -d" echo "u Requires -p. The database URI the exporter should connect with." echo "d The Cloud Foundry service name of the RDS instance. Should be included with all deployments." echo @@ -32,6 +40,7 @@ deploy_pg_exporter() { # TODO: this logic needs to be updated to allow routing accross spaces based on where we want PLG to live. cf target -o hhs-acf-ofa -s tanf-prod cf add-network-policy prometheus $APP_NAME -s "$EXPORTER_SPACE" --protocol tcp --port 9187 + cf target -o hhs-acf-ofa -s "$EXPORTER_SPACE" rm $MANIFEST popd } @@ -50,12 +59,20 @@ deploy_grafana() { cf push --no-route -f $MANIFEST -t 180 --strategy rolling cf map-route $APP_NAME apps.internal --hostname $APP_NAME - # Give Grafana a public route for now. Might be able to swap to internal route later. - # cf map-route "$APP_NAME" app.cloud.gov --hostname "${APP_NAME}" # Add policy to allow grafana to talk to prometheus and loki cf add-network-policy $APP_NAME prometheus --protocol tcp --port 8080 cf add-network-policy $APP_NAME loki --protocol tcp --port 8080 + + # Add network policies to allow grafana to talk to all frontend apps in all environments + for app in ${DEV_FRONTEND_APPS[@]}; do + cf add-network-policy "grafana" "$app" -s "tanf-dev" --protocol tcp --port 80 + done + for app in ${STAGING_FRONTEND_APPS[@]}; do + cf add-network-policy "grafana" "$app" -s "tanf-staging" --protocol tcp --port 80 + done + cf add-network-policy "grafana" "$PROD_FRONTEND" --protocol tcp --port 80 + rm $DATASOURCES rm $MANIFEST popd @@ -65,6 +82,16 @@ deploy_prometheus() { pushd prometheus cf push --no-route -f manifest.yml -t 180 --strategy rolling cf map-route prometheus apps.internal --hostname prometheus + + # Add network policies to allow prometheus to talk to all backend apps in all environments + for app in ${DEV_BACKEND_APPS[@]}; do + cf add-network-policy prometheus "$app" -s "tanf-dev" --protocol tcp --port 8080 + done + for app in ${STAGING_BACKEND_APPS[@]}; do + cf add-network-policy prometheus "$app" -s "tanf-staging" --protocol tcp --port 8080 + done + cf add-network-policy prometheus "$PROD_BACKEND" --protocol tcp --port 8080 + popd } diff --git a/tdrs-backend/plg/prometheus/prometheus.yml b/tdrs-backend/plg/prometheus/prometheus.yml index a8afaaa38..8052ff189 100644 --- a/tdrs-backend/plg/prometheus/prometheus.yml +++ b/tdrs-backend/plg/prometheus/prometheus.yml @@ -45,10 +45,6 @@ scrape_configs: static_configs: - targets: ["tdp-backend-prod.apps.internal:8080"] - - job_name: "celery-exporter-raft" - static_configs: - - targets: ["celery-exporter-raft.apps.internal:9540"] - - job_name: postgres-dev static_configs: - targets: ["pg-exporter-dev.apps.internal:9187"] From 6e5632e5ce19ccb8ea4ff2c20974fc65f9d61191 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 14 Nov 2024 10:11:20 -0500 Subject: [PATCH 07/18] - Updated syntax errors in script - Gave loki 1GB memory --- tdrs-backend/plg/deploy.sh | 26 +++++++++++++------------- tdrs-backend/plg/loki/manifest.yml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index b69a678fd..9c269f915 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -1,13 +1,13 @@ #!/bin/bash set -e -DEV_BACKEND_APPS = ("tdp-backend-raft" "tdp-backend-qasp" "tdp-backend-a11y") -STAGING_BACKEND_APPS = ("tdp-backend-develop" "tdp-backend-staging") -PROD_BACKEND = "tdp-backend-prod" +DEV_BACKEND_APPS=("tdp-backend-raft" "tdp-backend-qasp" "tdp-backend-a11y") +STAGING_BACKEND_APPS=("tdp-backend-develop" "tdp-backend-staging") +PROD_BACKEND="tdp-backend-prod" -DEV_FRONTEND_APPS = ("tdp-frontend-raft" "tdp-frontend-qasp" "tdp-frontend-a11y") -STAGING_FRONTEND_APPS = ("tdp-frontend-develop" "tdp-frontend-staging") -PROD_FRONTEND = "tdp-frontend-prod" +DEV_FRONTEND_APPS=("tdp-frontend-raft" "tdp-frontend-qasp" "tdp-frontend-a11y") +STAGING_FRONTEND_APPS=("tdp-frontend-develop" "tdp-frontend-staging") +PROD_FRONTEND="tdp-frontend-prod" help() { echo "Deploy the PLG stack or a Postgres exporter to the Cloud Foundry space you're currently authenticated in." @@ -66,12 +66,12 @@ deploy_grafana() { # Add network policies to allow grafana to talk to all frontend apps in all environments for app in ${DEV_FRONTEND_APPS[@]}; do - cf add-network-policy "grafana" "$app" -s "tanf-dev" --protocol tcp --port 80 + cf add-network-policy "grafana" $app -s "tanf-dev" --protocol tcp --port 80 done for app in ${STAGING_FRONTEND_APPS[@]}; do - cf add-network-policy "grafana" "$app" -s "tanf-staging" --protocol tcp --port 80 + cf add-network-policy "grafana" $app -s "tanf-staging" --protocol tcp --port 80 done - cf add-network-policy "grafana" "$PROD_FRONTEND" --protocol tcp --port 80 + cf add-network-policy "grafana" $PROD_FRONTEND --protocol tcp --port 80 rm $DATASOURCES rm $MANIFEST @@ -85,12 +85,12 @@ deploy_prometheus() { # Add network policies to allow prometheus to talk to all backend apps in all environments for app in ${DEV_BACKEND_APPS[@]}; do - cf add-network-policy prometheus "$app" -s "tanf-dev" --protocol tcp --port 8080 + cf add-network-policy prometheus $app -s "tanf-dev" --protocol tcp --port 8080 done for app in ${STAGING_BACKEND_APPS[@]}; do - cf add-network-policy prometheus "$app" -s "tanf-staging" --protocol tcp --port 8080 + cf add-network-policy prometheus $app -s "tanf-staging" --protocol tcp --port 8080 done - cf add-network-policy prometheus "$PROD_BACKEND" --protocol tcp --port 8080 + cf add-network-policy prometheus $PROD_BACKEND --protocol tcp --port 8080 popd } @@ -137,7 +137,7 @@ if [ "$#" -eq 0 ]; then fi pushd "$(dirname "$0")" -if [ "$DB_URI" == "" ] || [ "$DB_SERVICE_NAME" == "" ]; then +if [ "$DB_SERVICE_NAME" == "" ]; then err_help_exit "Error: you must include a database service name." fi if [ "$DEPLOY" == "plg" ]; then diff --git a/tdrs-backend/plg/loki/manifest.yml b/tdrs-backend/plg/loki/manifest.yml index ab0d5d532..3f747daf4 100644 --- a/tdrs-backend/plg/loki/manifest.yml +++ b/tdrs-backend/plg/loki/manifest.yml @@ -1,7 +1,7 @@ version: 1 applications: - name: loki - memory: 512M + memory: 1G disk_quota: 7G instances: 1 command: | From 57ad0ccb1217cb39101c1e5b3ac31be6da6d0f29 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 14 Nov 2024 12:28:34 -0500 Subject: [PATCH 08/18] - update comment - remove todo --- tdrs-backend/plg/deploy.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 9c269f915..01c62c6a7 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -15,7 +15,7 @@ help() { echo "Options:" echo "h Print this help message." echo "a Deploy the entire PLG stack." - echo "p Deploy a postgres exporter, expects the environment name (dev, staging, prod) to be passed with switch. Requires -u and -d" + echo "p Deploy a postgres exporter, expects the environment name (dev, staging, production) to be passed with switch. Requires -u and -d" echo "u Requires -p. The database URI the exporter should connect with." echo "d The Cloud Foundry service name of the RDS instance. Should be included with all deployments." echo @@ -37,7 +37,6 @@ deploy_pg_exporter() { cf map-route $APP_NAME apps.internal --hostname $APP_NAME # Add policy to allow prometheus to talk to pg-exporter regardless of environment - # TODO: this logic needs to be updated to allow routing accross spaces based on where we want PLG to live. cf target -o hhs-acf-ofa -s tanf-prod cf add-network-policy prometheus $APP_NAME -s "$EXPORTER_SPACE" --protocol tcp --port 9187 cf target -o hhs-acf-ofa -s "$EXPORTER_SPACE" From e36ddf90b7be19cbc18f309264261b8bf4da98de Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 14 Nov 2024 12:37:27 -0500 Subject: [PATCH 09/18] - Update grafana session settings --- tdrs-backend/plg/grafana/custom.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/plg/grafana/custom.ini b/tdrs-backend/plg/grafana/custom.ini index ff2b035a5..fef040207 100644 --- a/tdrs-backend/plg/grafana/custom.ini +++ b/tdrs-backend/plg/grafana/custom.ini @@ -553,10 +553,10 @@ login_cookie_name = grafana_session disable_login = false # The maximum lifetime (duration) an authenticated user can be inactive before being required to login at next visit. Default is 7 days (7d). This setting should be expressed as a duration, e.g. 5m (minutes), 6h (hours), 10d (days), 2w (weeks), 1M (month). The lifetime resets at each successful token rotation (token_rotation_interval_minutes). -login_maximum_inactive_lifetime_duration = +login_maximum_inactive_lifetime_duration = 30m # The maximum lifetime (duration) an authenticated user can be logged in since login time before being required to login. Default is 30 days (30d). This setting should be expressed as a duration, e.g. 5m (minutes), 6h (hours), 10d (days), 2w (weeks), 1M (month). -login_maximum_lifetime_duration = +login_maximum_lifetime_duration = 1d # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. token_rotation_interval_minutes = 10 From 08a8604294c64a3b7c8ded835919cfc52dd5a441 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 14 Nov 2024 12:40:40 -0500 Subject: [PATCH 10/18] - remove very annoying log messages --- tdrs-backend/tdpservice/users/api/authorization_check.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/users/api/authorization_check.py b/tdrs-backend/tdpservice/users/api/authorization_check.py index 60da3a17b..c070027dc 100644 --- a/tdrs-backend/tdpservice/users/api/authorization_check.py +++ b/tdrs-backend/tdpservice/users/api/authorization_check.py @@ -66,10 +66,9 @@ def get(self, request, *args, **kwargs): user_in_valid_group = user.is_ofa_sys_admin or user.is_digit_team if (user.hhs_id is not None and user_in_valid_group) or settings.BYPASS_OFA_AUTH: - logger.debug(f"User: {user} has correct authentication credentials. Allowing access to Kibana.") return HttpResponse(status=200) else: - logger.debug(f"User: {user} has incorrect authentication credentials. Not allowing access to Kibana.") + logger.warning(f"User: {user} has incorrect authentication credentials. Not allowing access to Kibana.") return HttpResponse(status=401) class GrafanaAuthorizationCheck(APIView): @@ -86,8 +85,7 @@ def get(self, request, *args, **kwargs): user_in_valid_group = user.is_ofa_sys_admin or user.is_developer if user_in_valid_group: - logger.debug(f"User: {user} has correct authentication credentials. Allowing access to Grafana.") return HttpResponse(status=200) else: - logger.debug(f"User: {user} has incorrect authentication credentials. Not allowing access to Grafana.") + logger.warning(f"User: {user} has incorrect authentication credentials. Not allowing access to Grafana.") return HttpResponse(status=401) From b3f57ad42a875018dbde2f0e98bc75c795bcae17 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 14 Nov 2024 13:05:19 -0500 Subject: [PATCH 11/18] - Move all PLG networking to plg deploy script --- scripts/deploy-backend.sh | 11 ----------- tdrs-backend/plg/deploy.sh | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 5b5fcb243..c416e595c 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -115,14 +115,6 @@ prepare_promtail() { popd } -update_plg_networking() { - # Promtail needs to send logs to Loki - cf add-network-policy "$CGAPPNAME_BACKEND" loki -s tanf-prod --protocol tcp --port 8080 - - # Add network policy allowing frontend to talk to Grafana - cf add-network-policy "$CGAPPNAME_FRONTEND" "grafana" -s tanf-prod --protocol tcp --port 8080 -} - update_backend() { cd tdrs-backend || exit @@ -161,9 +153,6 @@ update_backend() # Add network policy to allow frontend to access backend cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080 - # Add PLG routing - update_plg_networking - if [ "$CF_SPACE" = "tanf-prod" ]; then # Add network policy to allow backend to access tanf-prod services cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index 01c62c6a7..c411f5457 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -101,6 +101,25 @@ deploy_loki() { popd } +setup_extra_net_pols() { + # Add network policies to allow frontend/backend to talk to grafana/loki + cf target -o hhs-acf-ofa -s tanf-dev + for i in ${!DEV_BACKEND_APPS[@]}; do + cf add-network-policy ${DEV_FRONTEND_APPS[$i]} grafana -s tanf-prod --protocol tcp --port 8080 + cf add-network-policy ${DEV_BACKEND_APPS[$i]} loki -s tanf-prod --protocol tcp --port 8080 + done + + cf target -o hhs-acf-ofa -s tanf-staging + for i in ${!STAGING_BACKEND_APPS[@]}; do + cf add-network-policy ${STAGING_FRONTEND_APPS[$i]} grafana -s tanf-prod --protocol tcp --port 8080 + cf add-network-policy ${STAGING_BACKEND_APPS[$i]} loki -s tanf-prod --protocol tcp --port 8080 + done + + cf target -o hhs-acf-ofa -s tanf-prod + cf add-network-policy $PROD_FRONTEND grafana -s tanf-prod --protocol tcp --port 8080 + cf add-network-policy $PROD_BACKEND loki -s tanf-prod --protocol tcp --port 8080 +} + err_help_exit() { echo $1 echo @@ -143,6 +162,7 @@ if [ "$DEPLOY" == "plg" ]; then deploy_prometheus deploy_loki deploy_grafana $DB_SERVICE_NAME + setup_extra_net_pols fi if [ "$DEPLOY" == "pg-exporter" ]; then if [ "$DB_URI" == "" ]; then From 44d0923db4b7f68b76731afa2def198eeb6ef970 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 14 Nov 2024 14:50:41 -0500 Subject: [PATCH 12/18] - re add tests --- .circleci/build-and-test/workflows.yml | 112 ++++++++++++------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml index 1081c40e7..99d7c4fff 100644 --- a/.circleci/build-and-test/workflows.yml +++ b/.circleci/build-and-test/workflows.yml @@ -3,15 +3,15 @@ when: << pipeline.parameters.build_and_test_all >> jobs: - secrets-check - # - test-backend: - # requires: - # - secrets-check - # - test-frontend: - # requires: - # - secrets-check - # - test-e2e: - # requires: - # - secrets-check + - test-backend: + requires: + - secrets-check + - test-frontend: + requires: + - secrets-check + - test-e2e: + requires: + - secrets-check ci-build-and-test-all: jobs: @@ -22,52 +22,52 @@ - main - master - /^release.*/ - # - test-backend: - # filters: - # branches: - # only: - # - main - # - master - # - /^release.*/ - # requires: - # - secrets-check - # - test-frontend: - # filters: - # branches: - # only: - # - main - # - master - # - /^release.*/ - # requires: - # - secrets-check - # - test-e2e: - # filters: - # branches: - # only: - # - main - # - master - # - /^release.*/ - # requires: - # - secrets-check - # - make_erd: # from ../util folder - # filters: - # branches: - # only: - # - develop - # - master + - test-backend: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - test-frontend: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - test-e2e: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - make_erd: # from ../util folder + filters: + branches: + only: + - develop + - master - # build-and-test-backend: - # when: << pipeline.parameters.build_and_test_backend >> - # jobs: - # - secrets-check - # - test-backend: - # requires: - # - secrets-check + build-and-test-backend: + when: << pipeline.parameters.build_and_test_backend >> + jobs: + - secrets-check + - test-backend: + requires: + - secrets-check - # build-and-test-frontend: - # when: << pipeline.parameters.build_and_test_frontend >> - # jobs: - # - secrets-check - # - test-frontend: - # requires: - # - secrets-check + build-and-test-frontend: + when: << pipeline.parameters.build_and_test_frontend >> + jobs: + - secrets-check + - test-frontend: + requires: + - secrets-check From 47fb1c3f435f735d80b3f7bc583a00825e1806ec Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 22 Nov 2024 11:30:11 -0500 Subject: [PATCH 13/18] - dummy change --- tdrs-backend/plg/prometheus/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/plg/prometheus/prometheus.yml b/tdrs-backend/plg/prometheus/prometheus.yml index 66e35c519..9cd14eea8 100644 --- a/tdrs-backend/plg/prometheus/prometheus.yml +++ b/tdrs-backend/plg/prometheus/prometheus.yml @@ -95,4 +95,4 @@ scrape_configs: - targets: ["grafana.app.cloud.gov:9400"] labels: service: "grafana" - env: "production" + env: "production" \ No newline at end of file From a24e826047747e5fd4fdd3cfd899ebaf0718c368 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 22 Nov 2024 11:30:37 -0500 Subject: [PATCH 14/18] - revert --- tdrs-backend/plg/prometheus/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/plg/prometheus/prometheus.yml b/tdrs-backend/plg/prometheus/prometheus.yml index 9cd14eea8..66e35c519 100644 --- a/tdrs-backend/plg/prometheus/prometheus.yml +++ b/tdrs-backend/plg/prometheus/prometheus.yml @@ -95,4 +95,4 @@ scrape_configs: - targets: ["grafana.app.cloud.gov:9400"] labels: service: "grafana" - env: "production" \ No newline at end of file + env: "production" From 503c644d9286f2c608bce6f5bc21be74666e5ffb Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 25 Nov 2024 10:01:40 -0500 Subject: [PATCH 15/18] - Updated nginx conf to use correct auth check endpoint --- tdrs-frontend/nginx/cloud.gov/locations.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tdrs-frontend/nginx/cloud.gov/locations.conf b/tdrs-frontend/nginx/cloud.gov/locations.conf index 37bd413e2..2e14fc69f 100644 --- a/tdrs-frontend/nginx/cloud.gov/locations.conf +++ b/tdrs-frontend/nginx/cloud.gov/locations.conf @@ -61,7 +61,7 @@ location = /kibana_auth_check { } location /grafana/ { - auth_request /grafana_auth_check; + auth_request /plg_auth_check; auth_request_set $auth_status $upstream_status; set $grafana http://grafana.apps.internal:8080$request_uri; @@ -78,9 +78,9 @@ location /grafana/ { proxy_buffer_size 4k; } -location = /grafana_auth_check { +location = /plg_auth_check { internal; - set $endpoint http://{{env "BACKEND_HOST"}}.apps.internal:8080/grafana_auth_check/; + set $endpoint http://{{env "BACKEND_HOST"}}.apps.internal:8080/plg_auth_check/; proxy_pass $endpoint$1$is_args$args; proxy_set_header Host $host:3000; proxy_set_header X-Real-IP $remote_addr; From a5fce6b80097efca684d66ea80325b825ad60b7f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 26 Nov 2024 13:33:43 -0500 Subject: [PATCH 16/18] - Updated dashboard configs - added extra promtail config to prove filtering works --- .../dashboards/dev_logs_dashboard.json | 113 +++++++++++++++++ .../grafana/dashboards/logs_dashboard.json | 115 +----------------- tdrs-backend/plg/promtail/config.local.yml | 17 ++- 3 files changed, 127 insertions(+), 118 deletions(-) create mode 100644 tdrs-backend/plg/grafana/dashboards/dev_logs_dashboard.json diff --git a/tdrs-backend/plg/grafana/dashboards/dev_logs_dashboard.json b/tdrs-backend/plg/grafana/dashboards/dev_logs_dashboard.json new file mode 100644 index 000000000..7fc27f8d5 --- /dev/null +++ b/tdrs-backend/plg/grafana/dashboards/dev_logs_dashboard.json @@ -0,0 +1,113 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard allowing log visualization", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "gridPos": { + "h": 28, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": true, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{job=~\"$job\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Job Logs", + "type": "logs" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "Django", + "Logs", + "Loki" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "definition": "", + "description": "Filter logs by job.", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "label": "job", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + }, + "refresh": 1, + "regex": "^(?!.*[-]prod$).*$", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Dev Logs", + "uid": "cdyz6flmh0ttsy", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/tdrs-backend/plg/grafana/dashboards/logs_dashboard.json b/tdrs-backend/plg/grafana/dashboards/logs_dashboard.json index 5b34ecf3a..ef2c34f56 100644 --- a/tdrs-backend/plg/grafana/dashboards/logs_dashboard.json +++ b/tdrs-backend/plg/grafana/dashboards/logs_dashboard.json @@ -21,93 +21,6 @@ "graphTooltip": 0, "links": [], "panels": [ - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "dark-red", - "value": 80 - }, - { - "color": "light-red", - "value": 85 - }, - { - "color": "#EAB839", - "value": 90 - }, - { - "color": "semi-dark-green", - "value": 95 - }, - { - "color": "dark-green", - "value": 100 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "100 * avg_over_time(up{job=~\"$app\"}[$__range])", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "App Uptime", - "type": "stat" - }, { "datasource": { "type": "loki", @@ -117,7 +30,7 @@ "h": 28, "w": 24, "x": 0, - "y": 14 + "y": 0 }, "id": 1, "options": { @@ -157,7 +70,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "All", "value": "$__all" }, @@ -184,30 +97,6 @@ "skipUrlSync": false, "sort": 0, "type": "query" - }, - { - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "definition": "query_result(up)", - "hide": 0, - "includeAll": true, - "label": "App", - "multi": false, - "name": "app", - "options": [], - "query": { - "qryType": 3, - "query": "query_result(up)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "/.*job=\"([^\"]+).*/", - "skipUrlSync": false, - "sort": 0, - "type": "query" } ] }, diff --git a/tdrs-backend/plg/promtail/config.local.yml b/tdrs-backend/plg/promtail/config.local.yml index 9cb617c11..dc6eb0da4 100644 --- a/tdrs-backend/plg/promtail/config.local.yml +++ b/tdrs-backend/plg/promtail/config.local.yml @@ -15,19 +15,26 @@ scrape_configs: - targets: - localhost labels: - job: varlogs + job: varlogs-local __path__: /var/log/*log - - job_name: django + - job_name: backend-local static_configs: - targets: - localhost labels: - job: django + job: backend-local __path__: /logs/django.log - - job_name: nginx + - job_name: backend-prod static_configs: - targets: - localhost labels: - job: nginx + job: backend-prod + __path__: /logs/django.log + - job_name: frontend-local + static_configs: + - targets: + - localhost + labels: + job: frontend-local __path__: /var/log/nginx/*log From c051bec6d8fdfec6c907cc9fc58c9daff234da31 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 27 Nov 2024 09:33:18 -0500 Subject: [PATCH 17/18] - Made simple README outlining grafana's rbac/auth --- tdrs-backend/plg/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tdrs-backend/plg/README.md diff --git a/tdrs-backend/plg/README.md b/tdrs-backend/plg/README.md new file mode 100644 index 000000000..ac38627a9 --- /dev/null +++ b/tdrs-backend/plg/README.md @@ -0,0 +1,6 @@ +### Grafana Auth and RBAC Config +Grafana is accessible by any frontend app on a private route to users who have the correct role. The Grafana UI is not be accessible to any user or application unless they are routed to it via a frontend app. Grafana is configured to require user and password authentication. Having the extra layer of authentication is required because the roles defined in Grafana are not in alignment with the roles TDP defines. Assigning users to appropriate role and teams in Grafana allows for least privilege access to any information that Grafana might be able to display. + +Grafana has three roles: `Admin`, `Editor`, and `Viewer`. We have also defined two teams (groups) in Grafana: `OFA` and `TDP` and several users. The teams are how we manage least privilege to Grafana's resources. Upon creation, all users are given one of the base roles. All TDP dev user accounts are given read only access (`Viewer`) to Grafana and OFA has a user account(s) associated with each of the roles. All users who are outside of OFA should always be assigned the `Viewer` role to maintain least privilege. All dashboards in Grafana are viewable by team as opposed to individual users/roles. Dashboard permissions are configured per dashboard and each team is given read only access to the appropriate dashboards. The `ofa-admin` user is the only direct user given access to resources. This account is given exclusive admin rights to all of Grafana. + +All Grafana administration is handled under the `Administration` drop down in the hamburger menu which is only accessible to `Admin` users. Users can be created, assigned a role, and then associated with a team. As new dashboards are added to Grafana their permissions need to be configured for least privilege by going to Dashboards->->Settings->Permissions. The admin can use other dashboard permission configurations to help finish the configuration. From 264714433dae525771733118e19b372eec3db174 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 27 Nov 2024 11:30:03 -0500 Subject: [PATCH 18/18] - Use correct team names --- tdrs-backend/plg/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/plg/README.md b/tdrs-backend/plg/README.md index ac38627a9..f0438e8f4 100644 --- a/tdrs-backend/plg/README.md +++ b/tdrs-backend/plg/README.md @@ -1,6 +1,6 @@ ### Grafana Auth and RBAC Config Grafana is accessible by any frontend app on a private route to users who have the correct role. The Grafana UI is not be accessible to any user or application unless they are routed to it via a frontend app. Grafana is configured to require user and password authentication. Having the extra layer of authentication is required because the roles defined in Grafana are not in alignment with the roles TDP defines. Assigning users to appropriate role and teams in Grafana allows for least privilege access to any information that Grafana might be able to display. -Grafana has three roles: `Admin`, `Editor`, and `Viewer`. We have also defined two teams (groups) in Grafana: `OFA` and `TDP` and several users. The teams are how we manage least privilege to Grafana's resources. Upon creation, all users are given one of the base roles. All TDP dev user accounts are given read only access (`Viewer`) to Grafana and OFA has a user account(s) associated with each of the roles. All users who are outside of OFA should always be assigned the `Viewer` role to maintain least privilege. All dashboards in Grafana are viewable by team as opposed to individual users/roles. Dashboard permissions are configured per dashboard and each team is given read only access to the appropriate dashboards. The `ofa-admin` user is the only direct user given access to resources. This account is given exclusive admin rights to all of Grafana. +Grafana has three roles: `Admin`, `Editor`, and `Viewer`. We have also defined two teams (groups) in Grafana: `OFA` and `Raft` and several users. The teams are how we manage least privilege to Grafana's resources. Upon creation, all users are given one of the base roles. All Raft dev user accounts are given read only access (`Viewer`) to Grafana and OFA has a user account(s) associated with each of the roles. All users who are outside of OFA should always be assigned the `Viewer` role to maintain least privilege. All dashboards in Grafana are viewable by team as opposed to individual users/roles. Dashboard permissions are configured per dashboard and each team is given read only access to the appropriate dashboards. The `ofa-admin` user is the only direct user given access to resources. This account is given exclusive admin rights to all of Grafana. All Grafana administration is handled under the `Administration` drop down in the hamburger menu which is only accessible to `Admin` users. Users can be created, assigned a role, and then associated with a team. As new dashboards are added to Grafana their permissions need to be configured for least privilege by going to Dashboards->->Settings->Permissions. The admin can use other dashboard permission configurations to help finish the configuration.