diff --git a/.github/ISSUE_TEMPLATE/dev-issue-template.md b/.github/ISSUE_TEMPLATE/dev-issue-template.md index 530a6db49..8857901a5 100644 --- a/.github/ISSUE_TEMPLATE/dev-issue-template.md +++ b/.github/ISSUE_TEMPLATE/dev-issue-template.md @@ -9,6 +9,7 @@ assignees: '' **Description:** _Provide a brief background and justification for this issue_ +_OFA & UX team: Please include bullets capturing the value to STT end users, related guidance, and/or talking points that should be communicated to STTs in release notes_ **Acceptance Criteria:** diff --git a/docs/How-We-Work/team-meetings.md b/docs/How-We-Work/team-meetings.md index 8b78dcc59..ff770ed7a 100644 --- a/docs/How-We-Work/team-meetings.md +++ b/docs/How-We-Work/team-meetings.md @@ -68,12 +68,12 @@ A typical sprint schedule is described in the table below. - External factors outside of the author spending time building the ticket (ie need external team's input, see how a feature develops, etc.) - Ex. Waiting on X ticket to finish the scope of said unrefined ticket, problem found / unsure how big it is and knows other work will unearth it - If we know the ACs but not the tasks, then its unrefined + - Release Notes summary is empty or incomplete as applicable, to be provided by UX/Product by default. - Refined: Ticket is complete and is ready to be executed. - Refined & Ready to Go (Next Sprint) - "Earmarked" work for the upcoming sprint. - **Labelling:** - WIP - - Author knows the 5 W's or darn near (90%) - Drafted ticket – either still on the author the finish their part or a short team conversation is needed. - Administrative in nature - Ex. Stub, ticket that doesn't feel there's enough to warrant an introduction diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index 243d54c20..dacf21a29 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -58,6 +58,7 @@ cerberus = "==1.3.4" xlsxwriter = "==3.1.9" openpyxl = "==3.1.2" sendgrid = "==6.10.0" +django_prometheus = "==2.3.1" sentry-sdk = "==2.11.0" [requires] diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index 77b69e0a1..b0b0abaa3 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8af9f7341ff2b7ce87be8c21c58b6649cb9463fdf999f4e996452a549038caf5" + "sha256": "0c999e2844fdfba8e44241e735fc3c192024ac1e8d9c823c431b12334f17f2fe" }, "pipfile-spec": 6, "requires": { @@ -373,6 +373,14 @@ ], "version": "==0.2.7" }, + "django-prometheus": { + "hashes": [ + "sha256:cf9b26f7ba2e4568f08f8f91480a2882023f5908579681bcf06a4d2465f12168", + "sha256:f9c8b6c780c9419ea01043c63a437d79db2c33353451347894408184ad9c3e1e" + ], + "index": "pypi", + "version": "==2.3.1" + }, "django-storages": { "hashes": [ "sha256:204a99f218b747c46edbfeeb1310d357f83f90fa6a6024d8d0a3f422570cee84", @@ -764,7 +772,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0.post0" }, "python-http-client": { @@ -878,7 +886,7 @@ "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875", "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412" ], - "markers": "platform_python_implementation == 'CPython' and python_version < '3.13'", + "markers": "python_version < '3.13' and platform_python_implementation == 'CPython'", "version": "==0.2.8" }, "s3transfer": { @@ -920,7 +928,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "sqlparse": { @@ -1540,7 +1548,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0.post0" }, "pyyaml": { @@ -1631,7 +1639,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "snowballstemmer": { diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 81d7065c4..1feed80e4 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -69,6 +69,71 @@ services: volumes: - elastic_data:/usr/share/elasticsearch/data + grafana: + restart: always + image: grafana/grafana:11.2.0 + ports: + - 9400:9400 + volumes: + - ./plg/grafana/datasources.yml:/etc/grafana/provisioning/datasources/default.yml + - ./plg/grafana/dashboards/provider.yml:/etc/grafana/provisioning/dashboards/default.yml + - ./plg/grafana/dashboards:/var/lib/grafana/provisioning/dashboards + - ./plg/grafana/custom.ini:/etc/grafana/grafana.ini + - grafana_data:/var/lib/grafana + + prometheus: + restart: always + image: prom/prometheus:v2.54.1 + ports: + - 9090:9090 + volumes: + - ./plg/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - ./plg/prometheus/django_rules.yml:/etc/prometheus/prom_django_rules.yml + - prometheus_data:/prometheus + depends_on: + - web + - celery-exporter + - postgres-exporter + + promtail: + restart: always + image: grafana/promtail:3.0.1 + ports: + - 9080:9080 + volumes: + - ./plg/promtail/config.yml:/etc/promtail/config.yml + - ~/tdp-logs/nginx:/var/log/nginx + - logs:/logs + command: -config.file=/etc/promtail/config.yml + + loki: + restart: always + image: grafana/loki:3.0.1 + ports: + - 3100:3100 + volumes: + - loki_data:/loki + + celery-exporter: + restart: always + image: ovalmoney/celery-exporter:1.5.1 + ports: + - 9540:9540 + environment: + - CELERY_EXPORTER_BROKER_URL=redis://redis-server:6379/0 + depends_on: + - web + + postgres-exporter: + restart: always + image: prometheuscommunity/postgres-exporter:v0.15.0 + ports: + - 9187:9187 + environment: + - DATA_SOURCE_NAME=postgres://tdpuser:something_secure@postgres:5432/tdrs_test?sslmode=disable + depends_on: + - web + web: restart: always environment: @@ -97,7 +162,7 @@ services: - DJANGO_DEBUG - SENDGRID_API_KEY - GENERATE_TRAILER_ERRORS=True - - BYPASS_KIBANA_AUTH + - BYPASS_OFA_AUTH - ELASTICSEARCH_REINDEX_THREAD_COUNT - ELASTICSEARCH_REINDEX_CHUNK_SIZE - ELASTICSEARCH_REINDEX_REQUEST_TIMEOUT @@ -111,6 +176,7 @@ services: - ELASTICSEARCH_LOG_INDEX_SLOW_LEVEL volumes: - .:/tdpapp + - logs:/logs image: tdp build: . command: > @@ -141,6 +207,10 @@ volumes: localstack_data: postgres_data: elastic_data: + prometheus_data: + grafana_data: + loki_data: + logs: networks: diff --git a/tdrs-backend/plg/grafana/custom.ini b/tdrs-backend/plg/grafana/custom.ini new file mode 100644 index 000000000..b8124cdeb --- /dev/null +++ b/tdrs-backend/plg/grafana/custom.ini @@ -0,0 +1,1975 @@ +##################### Grafana Configuration Defaults ##################### +# +# Do not modify this file in grafana installs +# + +# possible values : production, development +app_mode = production + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +instance_name = ${HOSTNAME} + +#################################### Paths ############################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +data = data + +# Temporary files in `data` directory older than given duration will be removed +temp_data_lifetime = 24h + +# Directory where grafana can store logs +logs = data/log + +# Directory where grafana will automatically scan and look for plugins +plugins = data/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +provisioning = conf/provisioning + +#################################### Server ############################## +[server] +# Protocol (http, https, h2, socket) +protocol = http + +# Minimum TLS version allowed. By default, this value is empty. Accepted values are: TLS1.2, TLS1.3. If nothing is set TLS1.2 would be taken +min_tls_version = "" + +# The ip address to bind to, empty will bind to all interfaces +http_addr = + +# The http port to use +http_port = 9400 + +# The public facing domain name used to access grafana from a browser +domain = localhost + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +enforce_domain = false + +# The full public facing url +root_url = %(protocol)s://%(domain)s:%(http_port)s/grafana/ + +# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +serve_from_sub_path = true + +# Log web requests +router_logging = false + +# the path relative working path +static_root_path = public + +# enable gzip +enable_gzip = false + +# https certs & key file +cert_file = +cert_key = +cert_pass = + +# Certificates file watch interval +certs_watch_interval = + +# Unix socket gid +# Changing the gid of a file without privileges requires that the target group is in the group of the process and that the process is the file owner +# It is recommended to set the gid as http server user gid +# Not set when the value is -1 +socket_gid = -1 + +# Unix socket mode +socket_mode = 0660 + +# Unix socket path +socket = /tmp/grafana.sock + +# CDN Url +cdn_url = + +# Sets the maximum time in minutes before timing out read of an incoming request and closing idle connections. +# `0` means there is no timeout for reading the request. +read_timeout = 0 + +# This setting enables you to specify additional headers that the server adds to HTTP(S) responses. +[server.custom_response_headers] +#exampleHeader1 = exampleValue1 +#exampleHeader2 = exampleValue2 + +[environment] +# Sets whether the local file system is available for Grafana to use. Default is true for backward compatibility. +local_file_system_available = true + +#################################### GRPC Server ######################### +[grpc_server] +network = "tcp" +address = "127.0.0.1:10000" +use_tls = false +cert_file = +key_file = +# this will log the request and response for each unary gRPC call +enable_logging = false + +# Maximum size of a message that can be received in bytes. If not set, uses the gRPC default (4MiB). +max_recv_msg_size = + +# Maximum size of a message that can be sent in bytes. If not set, uses the gRPC default (unlimited). +max_send_msg_size = + +#################################### Database ############################ +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url property. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +type = sqlite3 +host = 127.0.0.1:3306 +name = grafana +user = root +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +password = +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +url = + +# Max idle conn setting default is 2 +max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +log_queries = + +# For "postgres", use either "disable", "require" or "verify-full" +# For "mysql", use either "true", "false", or "skip-verify". +ssl_mode = disable + +# For "postgres", use either "1" to enable or "0" to disable SNI +ssl_sni = + +# Database drivers may support different transaction isolation levels. +# Currently, only "mysql" driver supports isolation levels. +# If the value is empty - driver's default isolation level is applied. +# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". +isolation_level = + +ca_cert_path = +client_key_path = +client_cert_path = +server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +path = grafana.db + +# For "sqlite3" only. cache mode setting used for connecting to the database +cache_mode = private + +# For "sqlite3" only. Enable/disable Write-Ahead Logging, https://sqlite.org/wal.html. Default is false. +wal = false + +# For "mysql" and "postgres". Lock the database for the migrations, default is true. +migration_locking = true + +# For "mysql" and "postgres" only if migrationLocking is set. How many seconds to wait before failing to lock the database for the migrations, default is 0. +locking_attempt_timeout_sec = 0 + +# For "sqlite" only. How many times to retry query in case of database is locked failures. Default is 0 (disabled). +query_retries = 0 + +# For "sqlite" only. How many times to retry transaction in case of database is locked failures. Default is 5. +transaction_retries = 5 + +# Set to true to add metrics and tracing for database queries. +instrument_queries = false + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +connstr = + +# prefix prepended to all the keys in the remote cache +prefix = + +# This enables encryption of values stored in the remote cache +encryption = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +logging = false + +# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +timeout = 30 + +# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. +dialTimeout = 10 + +# How many seconds the data proxy waits before sending a keepalive request. +keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +expect_continue_timeout_seconds = 1 + +# Optionally limits the total number of connections per host, including connections in the dialing, +# active, and idle states. On limit violation, dials will block. +# A value of zero (0) means no limit. +max_conns_per_host = 0 + +# The maximum number of idle connections that Grafana will keep alive. +max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request. +send_user_header = false + +# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. +response_limit = 0 + +# Limits the number of rows that Grafana will process from SQL data sources. +row_limit = 1000000 + +# Sets a custom value for the `User-Agent` header for outgoing data proxy requests. If empty, the default value is `Grafana/` (for example `Grafana/9.0.0`). +user_agent = + +#################################### Analytics ########################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +reporting_enabled = true + +# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs +reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com/api/grafana/versions/stable to get the latest version. +check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +google_analytics_ua_id = + +# Google Analytics 4 tracking code, only enabled if you specify an id here +google_analytics_4_id = + +# When Google Analytics 4 Enhanced event measurement is enabled, we will try to avoid sending duplicate events and let Google Analytics 4 detect navigation changes, etc. +google_analytics_4_send_manual_page_views = false + +# Google Tag Manager ID, only enabled if you specify an id here +google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +rudderstack_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +rudderstack_config_url = + +# Rudderstack Integrations URL, optional. Only valid if you pass the SDK version 1.1 or higher +rudderstack_integrations_url = + +# Intercom secret, optional, used to hash user_id before passing to Intercom via Rudderstack +intercom_secret = + +# Application Insights connection string. Specify an URL string to enable this feature. +application_insights_connection_string = + +# Optional. Specifies an Application Insights endpoint URL where the endpoint string is wrapped in backticks ``. +application_insights_endpoint_url = + +# Controls if the UI contains any links to user feedback forms +feedback_links_enabled = true + +# Static context that is being added to analytics events +reporting_static_context = + +#################################### Security ############################ +[security] +# disable creation of admin user on first start of grafana +disable_initial_admin_creation = false + +# default admin user, created on startup +admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +admin_password = admin + +# default admin email, created on startup +admin_email = admin@localhost + +# used for signing +secret_key = SW2YcwTIb9zpOOhoPsMm + +# current key provider used for envelope encryption, default to static value specified by secret_key +encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +available_encryption_providers = + +# disable gravatar profile images +disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +data_source_proxy_whitelist = + +# disable protection against brute force login attempts +disable_brute_force_login_protection = false + +# set to true if you host Grafana behind HTTPS. default is false. +cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,