diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml index f9b602adf..a40d1568f 100644 --- a/.circleci/build-and-test/jobs.yml +++ b/.circleci/build-and-test/jobs.yml @@ -3,7 +3,7 @@ steps: - checkout - docker-compose-check - - docker-compose-up-with-elastic-backend + - docker-compose-up-backend - run: name: Run Unit Tests And Create Code Coverage Report command: | @@ -46,7 +46,7 @@ steps: - checkout - docker-compose-check - - docker-compose-up-with-elastic-backend + - docker-compose-up-backend - docker-compose-up-frontend - install-nodejs-machine - disable-npm-audit diff --git a/.circleci/util/commands.yml b/.circleci/util/commands.yml index 09d175b69..ebbdfb7e1 100644 --- a/.circleci/util/commands.yml +++ b/.circleci/util/commands.yml @@ -11,12 +11,6 @@ name: Build and spin-up Django API service command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build - docker-compose-up-with-elastic-backend: - steps: - - run: - name: Build and spin-up Django API service - command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build - cf-check: steps: - run: diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf index bec070813..bd75526c5 100644 --- a/tdrs-backend/clamav-router/nginx.conf +++ b/tdrs-backend/clamav-router/nginx.conf @@ -2,23 +2,43 @@ events { worker_connections 1024; } # This opens a route to clamav prod -http{ - resolver {{nameservers}} valid=10s; +http{ + charset utf-8; + log_format cloudfoundry 'NginxLog "$request" $status $body_bytes_sent'; + access_log /dev/stdout cloudfoundry; + + resolver {{nameservers}} valid=5s; + + log_format compression '$remote_addr - $remote_user [$time_local] ' + '"proxy_host and upstream_addr": $proxy_host $upstream_addr, ' + ' "request": $request, ' + '"body_bytes_sent" : $body_bytes_sent, ' + '"request_body": $request_body, ' + '"http_x_forwarded_for": $http_x_forwarded_for, ' + '"host": $host, ' + ' "status": $status, ' + '"proxy_add_x_forwarded_for": $proxy_add_x_forwarded_for, ' + '"http_referer": $http_referer, ' + '"http_user_agent": $http_user_agent, ' + '"cookies=$http_cookie;" "server=$server_name" "http_host=$http_host"' + ' Proxy: "$proxy_host" "$upstream_addr"'; + server { - client_max_body_size 100m; listen {{port}}; client_max_body_size 100m; - location /scan { - proxy_pass http://tanf-prod-clamav-rest.apps.internal:9000/scan; + location ~* ^/scan(.*)$ { + set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan; + proxy_pass $clamav$1$is_args$args; proxy_pass_request_headers on; } } + server { - client_max_body_size 100m; listen 9000; client_max_body_size 100m; - location /scan { - proxy_pass http://tanf-prod-clamav-rest.apps.internal:9000/scan; + location ~* ^/scan(.*)$ { + set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan; + proxy_pass $clamav$1$is_args$args; proxy_pass_request_headers on; } } diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 53a70eccb..570c9f019 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -46,7 +46,7 @@ services: - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh kibana: - image: docker.elastic.co/kibana/kibana-oss:7.4.2 + image: docker.elastic.co/kibana/kibana-oss:7.10.2 ports: - 5601:5601 environment: @@ -59,11 +59,10 @@ services: - elastic elastic: - image: elasticsearch:7.17.6 + image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2 environment: - discovery.type=single-node - logger.discovery.level=debug - - xpack.security.enabled=false ports: - 9200:9200 - 9300:9300 diff --git a/tdrs-backend/elastic_setup/Dockerfile b/tdrs-backend/elastic_setup/Dockerfile deleted file mode 100644 index 32e6429f6..000000000 --- a/tdrs-backend/elastic_setup/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG ELASTIC_VERSION - -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} - -COPY . / - -RUN ["chmod", "+x", "/entrypoint.sh"] -RUN ["chmod", "+x", "/util.sh"] - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/tdrs-backend/elastic_setup/entrypoint.sh b/tdrs-backend/elastic_setup/entrypoint.sh deleted file mode 100644 index 6073b0540..000000000 --- a/tdrs-backend/elastic_setup/entrypoint.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash - -set -eu -set -o pipefail - -source "${BASH_SOURCE[0]%/*}"/util.sh - - -# -------------------------------------------------------- -# Users declarations - -declare -A users_passwords -users_passwords=( - [kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}" - [ofa_admin]="${OFA_ADMIN_PASSWORD:-}" -) - -declare -A users_roles -users_roles=( - [kibana_system]='kibana_system' - [ofa_admin]='kibana_admin' -) - -# -------------------------------------------------------- -# Roles declarations for custom roles - -declare -A roles_files -roles_files=( - -) - -# -------------------------------------------------------- - - -log 'Waiting for availability of Elasticsearch. This can take several minutes.' - -declare -i exit_code=0 -wait_for_elasticsearch || exit_code=$? - -if ((exit_code)); then - case $exit_code in - 6) - suberr 'Could not resolve host. Is Elasticsearch running?' - ;; - 7) - suberr 'Failed to connect to host. Is Elasticsearch healthy?' - ;; - 28) - suberr 'Timeout connecting to host. Is Elasticsearch healthy?' - ;; - *) - suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}" - ;; - esac - - exit $exit_code -fi - -sublog 'Elasticsearch is running' - -log 'Waiting for initialization of built-in users' - -wait_for_builtin_users || exit_code=$? - -if ((exit_code)); then - suberr 'Timed out waiting for condition' - exit $exit_code -fi - -sublog 'Built-in users were initialized' - -for role in "${!roles_files[@]}"; do - log "Role '$role'" - - declare body_file - body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}" - if [[ ! -f "${body_file:-}" ]]; then - sublog "No role body found at '${body_file}', skipping" - continue - fi - - sublog 'Creating/updating' - ensure_role "$role" "$(<"${body_file}")" -done - -for user in "${!users_passwords[@]}"; do - log "User '$user'" - if [[ -z "${users_passwords[$user]:-}" ]]; then - sublog 'No password defined, skipping' - continue - fi - - declare -i user_exists=0 - user_exists="$(check_user_exists "$user")" - - if ((user_exists)); then - sublog 'User exists, setting password' - set_user_password "$user" "${users_passwords[$user]}" - else - if [[ -z "${users_roles[$user]:-}" ]]; then - suberr ' No role defined, skipping creation' - continue - fi - - sublog 'User does not exist, creating' - create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}" - fi -done - -log "Elastic setup completed. Exiting with code: $?" diff --git a/tdrs-backend/elastic_setup/util.sh b/tdrs-backend/elastic_setup/util.sh deleted file mode 100644 index 045110249..000000000 --- a/tdrs-backend/elastic_setup/util.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env bash - -# Log a message. -function log { - echo "[+] $1" -} - -# Log a message at a sub-level. -function sublog { - echo " ⠿ $1" -} - -# Log an error. -function err { - echo "[x] $1" >&2 -} - -# Log an error at a sub-level. -function suberr { - echo " ⠍ $1" >&2 -} - -# Poll the 'elasticsearch' service until it responds with HTTP code 200. -function wait_for_elasticsearch { - local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}" - - local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" ) - - if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then - args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) - fi - - local -i result=1 - local output - - # retry for max 300s (60*5s) - for _ in $(seq 1 60); do - local -i exit_code=0 - output="$(curl "${args[@]}")" || exit_code=$? - - if ((exit_code)); then - result=$exit_code - fi - - if [[ "${output: -3}" -eq 200 ]]; then - result=0 - break - fi - - sleep 5 - done - - if ((result)) && [[ "${output: -3}" -ne 000 ]]; then - echo -e "\n${output::-3}" - fi - - return $result -} - -# Poll the Elasticsearch users API until it returns users. -function wait_for_builtin_users { - local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}" - - local -a args=( '-s' '-D-' '-m15' "http://${elasticsearch_host}:9200/_security/user?pretty" ) - - if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then - args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) - fi - - local -i result=1 - - local line - local -i exit_code - local -i num_users - - # retry for max 30s (30*1s) - for _ in $(seq 1 30); do - num_users=0 - - # read exits with a non-zero code if the last read input doesn't end - # with a newline character. The printf without newline that follows the - # curl command ensures that the final input not only contains curl's - # exit code, but causes read to fail so we can capture the return value. - # Ref. https://unix.stackexchange.com/a/176703/152409 - while IFS= read -r line || ! exit_code="$line"; do - if [[ "$line" =~ _reserved.+true ]]; then - (( num_users++ )) - fi - done < <(curl "${args[@]}"; printf '%s' "$?") - - if ((exit_code)); then - result=$exit_code - fi - - # we expect more than just the 'elastic' user in the result - if (( num_users > 1 )); then - result=0 - break - fi - - sleep 1 - done - - return $result -} - -# Verify that the given Elasticsearch user exists. -function check_user_exists { - local username=$1 - - local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}" - - local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' - "http://${elasticsearch_host}:9200/_security/user/${username}" - ) - - if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then - args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) - fi - - local -i result=1 - local -i exists=0 - local output - - output="$(curl "${args[@]}")" - if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then - result=0 - fi - if [[ "${output: -3}" -eq 200 ]]; then - exists=1 - fi - - if ((result)); then - echo -e "\n${output::-3}" - else - echo "$exists" - fi - - return $result -} - -# Set password of a given Elasticsearch user. -function set_user_password { - local username=$1 - local password=$2 - - local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}" - - local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' - "http://${elasticsearch_host}:9200/_security/user/${username}/_password" - '-X' 'POST' - '-H' 'Content-Type: application/json' - '-d' "{\"password\" : \"${password}\"}" - ) - - if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then - args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) - fi - - local -i result=1 - local output - - output="$(curl "${args[@]}")" - if [[ "${output: -3}" -eq 200 ]]; then - result=0 - fi - - if ((result)); then - echo -e "\n${output::-3}\n" - fi - - return $result -} - -# Create the given Elasticsearch user. -function create_user { - local username=$1 - local password=$2 - local role=$3 - - local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}" - - local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' - "http://${elasticsearch_host}:9200/_security/user/${username}" - '-X' 'POST' - '-H' 'Content-Type: application/json' - '-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}" - ) - - if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then - args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) - fi - - local -i result=1 - local output - - output="$(curl "${args[@]}")" - if [[ "${output: -3}" -eq 200 ]]; then - result=0 - fi - - if ((result)); then - echo -e "\n${output::-3}\n" - fi - - return $result -} - -# Ensure that the given Elasticsearch role is up-to-date, create it if required. -function ensure_role { - local name=$1 - local body=$2 - - local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}" - - local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' - "http://${elasticsearch_host}:9200/_security/role/${name}" - '-X' 'POST' - '-H' 'Content-Type: application/json' - '-d' "$body" - ) - - if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then - args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) - fi - - local -i result=1 - local output - - output="$(curl "${args[@]}")" - if [[ "${output: -3}" -eq 200 ]]; then - result=0 - fi - - if ((result)); then - echo -e "\n${output::-3}\n" - fi - - return $result -} \ No newline at end of file diff --git a/tdrs-frontend/nginx/README.md b/tdrs-frontend/nginx/README.md index e6400eb37..d5beefc6b 100644 --- a/tdrs-frontend/nginx/README.md +++ b/tdrs-frontend/nginx/README.md @@ -2,7 +2,7 @@ ## 1. Installation -The Nginx uses a master process and mutiple workers to efficiently manage requests. The master process reads and evaluates config file and manages the workers. +The Nginx uses a master process and mutiple workers to efficiently manage requests. The master process reads and evaluates config file and manages the workers. In TDP, all config and associated files with it are placed in ```/tdrs-frontend/nginx``` folder. There are various differences between the local and deployed versions. Due to the fact that cloud.gov uses buildpacks and local version uses containerized docker instance, there are two separate directories for local and deployed versions. @@ -35,7 +35,35 @@ location ^~ /v1/ { The Nginx configuration files are located in: ```*'/frontend/nginx/'```* -## Backend +### Dynamic locations +In cloud environments, DNS information for applications is in a state of constant flux. To avoid having to restart Nginx each time an app's DNS information changes, Nginx provides features to resolve a host after a TTL has expired. The http block and location block below demonstrate the config updates to support dynamic locations. Note the `...` in the blocks below represents other config options in the blocks that we don't need to worry about. + +``` +http { + include mime.types; + + resolver 127.0.0.11 ipv6=off valid=5s; + ... +} +``` + +``` +location ~* ^/kibana/(.*)$ { + auth_request /kibana_auth_check; + auth_request_set $auth_status $upstream_status; + + set $kibana http://${KIBANA}:5601/; + proxy_pass $kibana$1$is_args$args; + ... +} +``` + +In the `http` block we need to include the `resolver` directive. This tells Nginx where our desired nameserver is and any other options we want to configure, such as the TTL option. +The configuration above indicates the nameserver is at `127.0.0.11`, ipv6 is disabled, and the TTL for a hostname is five seconds. However, for Nginx to re-resolve a host +we have to update the location, i.e. the `proxy_pass` in the location to use variable resolution. When Nginx resolves the variable given to the `proxy_pass` directive it will also determine if the TTL has expired for the host that the variable resolves to and will then resolve the DNS info if it has expired. If `proxy_pass` is not given a variable, Nginx will never resolve the host given to the `proxy_pass` directive no matter how short the TTL option is. + + +## 3. Backend ### Nginx @@ -43,7 +71,7 @@ The frontend then sends processing requests to the *backend* Django server, whic ### Gunicorn -Gunicorn is WSGI HTTP server based on python and uses **worker [Worker Processes](https://docs.gunicorn.org/en/stable/design.html#choosing-a-worker-type)**. The number of workers is relative to server request load. +Gunicorn is WSGI HTTP server based on python and uses **worker [Worker Processes](https://docs.gunicorn.org/en/stable/design.html#choosing-a-worker-type)**. The number of workers is relative to server request load. With having workers responding to requests in Gunicorn, it is importnat to have Nginx in front to handle requests first, otherwise DDOS attacks would consume the server (See Nginx config). @@ -55,7 +83,7 @@ workers = 2 There are two config files: one for development and one for deployed applications. Gunicorn is started from ```gunicorn_start.sh``` which starts either version of config files based on environment. -## Security +## 4. Security ### Whitelist IPs A list of IP addresses has been added to ```ip_whitelist.conf```. This means any request from an ip address not in the subnets included in this file will be rejected. This list is created manually and needs to be maintained to whitelist and include user IP subnets. @@ -63,9 +91,9 @@ A list of IP addresses has been added to ```ip_whitelist.conf```. This means any ### Security Headers All security headers are following best practices from [Mozilla](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers) and [OWASP](https://owasp.org/www-project-secure-headers/) and are added with comments on the config files. -### CORS +### CORS -Cross-Origin Resource Sharing (CORS) header allows a server to indicate any origin such as domain or port other than its own from which a browser can load resources. By adding HTTP headers that let server know which origins are permitted to read that information from the web browser. It should be noted that request might be passed without implications on CORS, this includes most form requests. +Cross-Origin Resource Sharing (CORS) header allows a server to indicate any origin such as domain or port other than its own from which a browser can load resources. By adding HTTP headers that let server know which origins are permitted to read that information from the web browser. It should be noted that request might be passed without implications on CORS, this includes most form requests. Since the frontend has to send requests to the backend server, the security headers are being set when serving the frontend requests as well as when Nginx acts as proxy server for the backend API. This is to ensure the sercurity headers cannot be tampered with and are always set to the correct values. diff --git a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf index 514010873..5898a8a5c 100644 --- a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf +++ b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf @@ -34,7 +34,7 @@ http { limit_req_zone $binary_remote_addr zone=limitreqsbyaddr:20m rate=1000r/s; limit_req_status 444; - resolver {{nameservers}} valid=10s; + resolver {{nameservers}} valid=5s; server { root public; diff --git a/tdrs-frontend/nginx/cloud.gov/locations.conf b/tdrs-frontend/nginx/cloud.gov/locations.conf index b7cd5517f..152535f50 100644 --- a/tdrs-frontend/nginx/cloud.gov/locations.conf +++ b/tdrs-frontend/nginx/cloud.gov/locations.conf @@ -22,11 +22,12 @@ location ~ ^/(v1|admin|static/admin|swagger|redocs) { add_header Access-Control-Allow-Origin 's3-us-gov-west-1.amazonaws.com'; } -location /kibana/ { +location ~* ^/kibana/(.*)$ { auth_request /kibana_auth_check; auth_request_set $auth_status $upstream_status; - proxy_pass http://{{env "KIBANA_BASE_URL"}}:5601/; + set $kibana http://{{env "KIBANA_BASE_URL"}}:5601/; + proxy_pass $kibana$1$is_args$args; proxy_pass_header x-csrftoken; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template index 8cf7b79ab..360968015 100644 --- a/tdrs-frontend/nginx/local/default.conf.template +++ b/tdrs-frontend/nginx/local/default.conf.template @@ -8,6 +8,8 @@ events { http { include mime.types; + resolver 127.0.0.11 ipv6=off valid=5s; + server { listen 80; client_max_body_size 100m; @@ -99,7 +101,8 @@ http { '"http_x_forwarded_for": "$http_x_forwarded_for",' '"host": "$host",' '"$http_referer" "$http_user_agent" "cookies=$http_cookie;" "server=$server_name" "http_host=$http_host"' - '"HTTP_ORIGIN = $http_origin"'; + '"HTTP_ORIGIN = $http_origin"' + ' Proxy Pass: "$proxy_host" "$upstream_addr"'; access_log /dev/stdout compression; #access_log stderr compression; diff --git a/tdrs-frontend/nginx/local/locations.conf b/tdrs-frontend/nginx/local/locations.conf index f19cac64b..69843a419 100644 --- a/tdrs-frontend/nginx/local/locations.conf +++ b/tdrs-frontend/nginx/local/locations.conf @@ -21,11 +21,12 @@ location ~ ^/(v1|admin|static/admin|static/drf-yasg|swagger|redocs) { proxy_pass_header x-csrftoken; } -location /kibana/ { +location ~* ^/kibana/(.*)$ { auth_request /kibana_auth_check; auth_request_set $auth_status $upstream_status; - proxy_pass http://${KIBANA}:5601/; + set $kibana http://${KIBANA}:5601/; + proxy_pass $kibana$1$is_args$args; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade";