Merge branch 'develop' into fix/pentest-high-pri-xss

raft-tech · Feb 16, 2024 · 1312fc2 · 1312fc2
2 parents 05982f3 + 9b62b1c
commit 1312fc2
Show file tree

Hide file tree

Showing 40 changed files with 295 additions and 131 deletions.
diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml
@@ -79,19 +79,33 @@
       frontend-appname:
         default: tdp-frontend
         type: string
+      kibana-appname:
+        default: tdp-kibana
+        type: string
+      proxy-appname:
+        default: tdp-elastic-proxy
+        type: string
       cf-space:
         default: tanf-dev
         type: string
     steps:
       - get-app-deploy-strategy:
           appname: <<parameters.backend-appname>>
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt update
+            sudo add-apt-repository ppa:rmescandon/yq
+            sudo apt-get install yq
       - run:
           name: Deploy backend application
           command: |
             bash ./scripts/deploy-backend.sh \
               $DEPLOY_STRATEGY \
               <<parameters.frontend-appname>> \
               <<parameters.backend-appname>> \
+              <<parameters.kibana-appname>> \
+              <<parameters.proxy-appname>> \
               <<parameters.cf-space>>
 
   deploy-clamav:
@@ -115,6 +129,9 @@
       frontend-appname:
         default: tdp-frontend
         type: string
+      kibana-appname:
+        default: tdp-kibana
+        type: string
 # So the frontend knows what space its in for the banner.
 # I am unclear if the domain is a reliable metric to make this function
 # It seems like it might not be working
@@ -136,6 +153,7 @@
               $DEPLOY_STRATEGY \
               <<parameters.frontend-appname>> \
               <<parameters.backend-appname>> \
+              <<parameters.kibana-appname>> \
               <<parameters.cf-space>> \
               <<parameters.environment>>
 

diff --git a/.gitconfig b/.gitconfig
@@ -7,6 +7,7 @@
 	allowed = [A-Z]+_KEY=..echo \".{S3_CREDENTIALS}\" [|] jq -r .+
 	allowed = ./tdrs-backend/.env.example:.*
 	allowed = ./tdrs-backend/docker-compose.yml:57:.*
+	allowed = ./tdrs-backend/manifest.proxy.yml:*
 	allowed = regexes.json:.*
 	allowed = ./scripts/copy-login-gov-keypair.sh:14:JWT_KEY=.*
 	allowed = scripts/deploy-backend.sh:.+:DJANGO_SECRET_KEY=..python -c .from secrets import token_urlsafe. print.token_urlsafe..*

diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh
@@ -10,7 +10,9 @@ DEPLOY_STRATEGY=${1}
 #The application name  defined via the manifest yml for the frontend
 CGAPPNAME_FRONTEND=${2}
 CGAPPNAME_BACKEND=${3}
-CF_SPACE=${4}
+CGAPPNAME_KIBANA=${4}
+CGAPPNAME_PROXY=${5}
+CF_SPACE=${6}
 
 strip() {
     # Usage: strip "string" "pattern"
@@ -20,8 +22,14 @@ strip() {
 env=$(strip $CF_SPACE "tanf-")
 backend_app_name=$(echo $CGAPPNAME_BACKEND | cut -d"-" -f3)
 
+# Update the Kibana and Elastic proxy names to include the environment
+CGAPPNAME_KIBANA="${CGAPPNAME_KIBANA}-${backend_app_name}"
+CGAPPNAME_PROXY="${CGAPPNAME_PROXY}-${backend_app_name}"
+
 echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY"
 echo BACKEND_HOST: "$CGAPPNAME_BACKEND"
+echo KIBANA_HOST: "$CGAPPNAME_KIBANA"
+echo ELASTIC_PROXY_HOST: "$CGAPPNAME_PROXY"
 echo CF_SPACE: "$CF_SPACE"
 echo env: "$env"
 echo backend_app_name: "$backend_app_name"
@@ -49,6 +57,7 @@ set_cf_envs()
   "DJANGO_SETTINGS_MODULE"
   "DJANGO_SU_NAME"
   "FRONTEND_BASE_URL"
+  "KIBANA_BASE_URL"
   "LOGGING_LEVEL"
   "REDIS_URI"
   "JWT_KEY"
@@ -86,6 +95,36 @@ generate_jwt_cert()
     cf set-env "$CGAPPNAME_BACKEND" JWT_KEY "$(cat key.pem)"
 }
 
+update_kibana()
+{
+  cd tdrs-backend || exit
+
+  # Run template evaluation on manifest
+  yq eval -i ".applications[0].services[0] = \"es-${backend_app_name}\""  manifest.proxy.yml
+  yq eval -i ".applications[0].env.CGAPPNAME_PROXY = \"${CGAPPNAME_PROXY}\""  manifest.kibana.yml
+
+  if [ "$1" = "rolling" ] ; then
+        # Do a zero downtime deploy.  This requires enough memory for
+        # two apps to exist in the org/space at one time.
+        cf push "$CGAPPNAME_PROXY" --no-route -f manifest.proxy.yml -t 180 --strategy rolling || exit 1
+        cf push "$CGAPPNAME_KIBANA" --no-route -f manifest.kibana.yml -t 180 --strategy rolling || exit 1
+    else
+        cf push "$CGAPPNAME_PROXY" --no-route -f manifest.proxy.yml -t 180
+        cf push "$CGAPPNAME_KIBANA" --no-route -f manifest.kibana.yml -t 180
+    fi
+
+    cf map-route "$CGAPPNAME_PROXY" apps.internal --hostname "$CGAPPNAME_PROXY"
+    cf map-route "$CGAPPNAME_KIBANA" apps.internal --hostname "$CGAPPNAME_KIBANA"
+
+    # Add network policy allowing Kibana to talk to the proxy and to allow the backend to talk to Kibana
+    cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_PROXY" --protocol tcp --port 8080
+    cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
+    cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
+    cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80
+
+    cd ..
+}
+
 update_backend()
 {
     cd tdrs-backend || exit
@@ -189,6 +228,8 @@ else
   FRONTEND_BASE_URL="$DEFAULT_FRONTEND_ROUTE"
 fi
 
+KIBANA_BASE_URL="http://$CGAPPNAME_KIBANA.apps.internal"
+
 # Dynamically generate a new DJANGO_SECRET_KEY
 DJANGO_SECRET_KEY=$(python3 -c "from secrets import token_urlsafe; print(token_urlsafe(50))")
 
@@ -208,6 +249,7 @@ if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then
     # Perform a rolling update for the backend and frontend deployments if
     # specified, otherwise perform a normal deployment
     update_backend 'rolling'
+    update_kibana 'rolling'
 elif [ "$DEPLOY_STRATEGY" = "bind" ] ; then
     # Bind the services the application depends on and restage the app.
     bind_backend_to_services
@@ -216,15 +258,20 @@ elif [ "$DEPLOY_STRATEGY" = "initial" ]; then
     # for it to work. the app will fail to start once, have the services bind,
     # and then get restaged.
     update_backend
+    update_kibana
     bind_backend_to_services
 elif [ "$DEPLOY_STRATEGY" = "rebuild" ]; then
     # You want to redeploy the instance under the same name
     # Delete the existing app (with out deleting the services)
     # and perform the initial deployment strategy.
     cf delete "$CGAPPNAME_BACKEND" -r -f
+    cf delete "$CGAPPNAME_KIBANA" -r -f
+    cf delete "$CGAPPNAME_PROXY" -r -f
     update_backend
+    update_kibana
     bind_backend_to_services
 else
     # No changes to deployment config, just deploy the changes and restart
     update_backend
+    update_kibana
 fi
diff --git a/scripts/deploy-frontend.sh b/scripts/deploy-frontend.sh
@@ -8,14 +8,21 @@ DEPLOY_STRATEGY=${1}
 #The application name  defined via the manifest yml for the frontend
 CGHOSTNAME_FRONTEND=${2}
 CGHOSTNAME_BACKEND=${3}
-CF_SPACE=${4}
-ENVIRONMENT=${5}
+CGAPPNAME_KIBANA=${4}
+CF_SPACE=${5}
+ENVIRONMENT=${6}
+
+backend_app_name=$(echo $CGHOSTNAME_BACKEND | cut -d"-" -f3)
+
+# Update the Kibana name to include the environment
+KIBANA_BASE_URL="${CGAPPNAME_KIBANA}-${backend_app_name}.apps.internal"
 
 update_frontend()
 {
     echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY"
     echo FRONTEND_HOST: "$CGHOSTNAME_FRONTEND"
     echo BACKEND_HOST: "$CGHOSTNAME_BACKEND"
+    echo KIBANA_BASE_URL: "$KIBANA_BASE_URL"
     cd tdrs-frontend || exit
 
     if [ "$CF_SPACE" = "tanf-prod" ]; then
@@ -44,6 +51,7 @@ update_frontend()
     fi
 
     cf set-env "$CGHOSTNAME_FRONTEND" BACKEND_HOST "$CGHOSTNAME_BACKEND"
+    cf set-env "$CGHOSTNAME_FRONTEND" KIBANA_BASE_URL "$KIBANA_BASE_URL"
 
     npm run build:$ENVIRONMENT
     unlink .env.production

diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf
@@ -1,8 +1,9 @@
-events { worker_connections 1024;
+events { 
+    worker_connections 1024;
 }
-
 # This opens a route to clamav prod
 http{ 
+    resolver {{nameservers}} valid=10s;
     server {
         client_max_body_size 100m;
         listen {{port}};
@@ -21,4 +22,4 @@ http{
             proxy_pass_request_headers      on;
         }
     }
-}
+}
diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
@@ -46,40 +46,15 @@ services:
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
   kibana:
-    image: elastic/kibana:7.17.10
+    image: docker.elastic.co/kibana/kibana-oss:7.4.2
     ports:
       - 5601:5601
     environment:
-      - xpack.security.encryptionKey=${KIBANA_ENCRYPTION_KEY:-something_at_least_32_characters}
-      - xpack.security.session.idleTimeout="1h"
-      - xpack.security.session.lifespan="30d"
-    volumes:
-      - ./kibana.yml:/usr/share/kibana/config/kibana.yml
-    depends_on:
-      - elastic
-
-  # This task only needs to be performed once, during the *initial* startup of
-  # the stack. Any subsequent run will reset the passwords of existing users to
-  # the values defined inside the '.env' file, and the built-in roles to their
-  # default permissions.
-  #
-  # By default, it is excluded from the services started by 'docker compose up'
-  # due to the non-default profile it belongs to. To run it, either provide the
-  # '--profile=elastic_setup' CLI flag to Compose commands, or "up" the service by name
-  # such as 'docker compose up elastic_setup'.
-  elastic_setup:
-    profiles:
-      - elastic_setup
-    build:
-      context: elastic_setup/
-      args:
-        ELASTIC_VERSION: "7.17.6"
-    init: true
-    environment:
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-changeme}
-      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-changeme}
-      OFA_ADMIN_PASSWORD: ${OFA_ADMIN_PASSWORD:-changeme}
-      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-elastic}
+      - ELASTICSEARCH_HOSTS="http://elastic:9200"
+      - SERVER_HOST=kibana
+      - SERVER_BASEPATH=/kibana
+      - SERVER_SECURITYRESPONSEHEADERS_REFERRERPOLICY=no-referrer
+      - CSP_WARNLEGACYBROWSERS=false
     depends_on:
       - elastic
 
@@ -88,12 +63,7 @@ services:
     environment:
       - discovery.type=single-node
       - logger.discovery.level=debug
-      - xpack.security.enabled=true
-      - xpack.security.authc.anonymous.username="ofa_admin"
-      - xpack.security.authc.anonymous.roles="ofa_admin"
-      - xpack.security.authc.anonymous.authz_exception=true
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-changeme}
-      - KIBANA_SYSTEM_PASSWORD=${KIBANA_SYSTEM_PASSWORD:-changeme}
+      - xpack.security.enabled=false
     ports:
       - 9200:9200
       - 9300:9300

diff --git a/tdrs-backend/kibana.yml b/tdrs-backend/kibana.yml
diff --git a/tdrs-backend/manifest.kibana.yml b/tdrs-backend/manifest.kibana.yml
@@ -0,0 +1,16 @@
+version: 1
+applications:
+  - name: tdp-kibana
+    memory: 2G
+    disk_quota: 2G
+    instances: 1
+    env:
+      CGAPPNAME_PROXY: {{ proxy_hostname }}
+      SERVER_BASEPATH: /kibana
+      SERVER_SECURITYRESPONSEHEADERS_REFERRERPOLICY: no-referrer
+      CSP_WARNLEGACYBROWSERS: false
+    docker:
+      image: docker.elastic.co/kibana/kibana-oss:7.4.2
+    command: |
+      export ELASTICSEARCH_HOSTS=http://$CGAPPNAME_PROXY.apps.internal:8080 &&
+      /usr/local/bin/dumb-init -- /usr/local/bin/kibana-docker
diff --git a/tdrs-backend/manifest.proxy.yml b/tdrs-backend/manifest.proxy.yml
@@ -0,0 +1,15 @@
+version: 1
+applications:
+- name: tdp-elastic-proxy
+  memory: 64M
+  disk_quota: 64M
+  instances: 1
+  services:
+    - {{ service_0 }}
+  docker:
+    image: elipe17/aws-es-proxy:latest
+  command: |
+    export ENDPOINT=$(echo $VCAP_SERVICES | grep -Eo 'host[^,]*' | grep -Eo '[^:]*$' | tr -d '"' | sed -e 's/^/https:\/\//') &&
+    export AWS_ACCESS_KEY_ID=$(echo $VCAP_SERVICES | grep -Eo 'access_key[^,]*' | grep -Eo '[^:]*$' | tr -d '"') &&
+    export AWS_SECRET_ACCESS_KEY=$(echo $VCAP_SERVICES | grep -Eo 'secret_key[^,]*' | grep -Eo '[^:]*$' | tr -d '"') &&
+    /usr/local/bin/aws-es-proxy -endpoint $ENDPOINT -listen 0.0.0.0:8080 -verbose -debug
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py
@@ -12,6 +12,7 @@
             document=SSP_M1DataSubmissionDocument(),
             preparsing_validators=[
                 validators.hasLength(150),
+                validators.notEmpty(8, 19)
             ],
             postparsing_validators=[
                 validators.if_then_validator(
@@ -125,7 +126,7 @@
                     startIndex=8,
                     endIndex=19,
                     required=True,
-                    validators=[validators.isAlphaNumeric()]
+                    validators=[validators.notEmpty()]
                 ),
                 Field(
                     item="2",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py
@@ -14,6 +14,7 @@
             document=SSP_M2DataSubmissionDocument(),
             preparsing_validators=[
                 validators.hasLength(150),
+                validators.notEmpty(8, 19)
             ],
             postparsing_validators=[
                 validators.validate__FAM_AFF__SSN(),
@@ -152,7 +153,7 @@
                     startIndex=8,
                     endIndex=19,
                     required=True,
-                    validators=[validators.isAlphaNumeric()]
+                    validators=[validators.notEmpty()]
                 ),
                 Field(
                     item="26",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py
@@ -11,6 +11,7 @@
     document=SSP_M3DataSubmissionDocument(),
     preparsing_validators=[
         validators.notEmpty(start=19, end=60),
+        validators.notEmpty(8, 19)
     ],
     postparsing_validators=[
         validators.if_then_validator(
@@ -118,7 +119,7 @@
             startIndex=8,
             endIndex=19,
             required=True,
-            validators=[validators.isAlphaNumeric()]
+            validators=[validators.notEmpty()]
         ),
         Field(
             item="60",
@@ -318,6 +319,7 @@
     quiet_preparser_errors=True,
     preparsing_validators=[
         validators.notEmpty(start=60, end=101),
+        validators.notEmpty(8, 19)
     ],
     postparsing_validators=[
         validators.if_then_validator(
@@ -425,7 +427,7 @@
             startIndex=8,
             endIndex=19,
             required=True,
-            validators=[validators.isAlphaNumeric()]
+            validators=[validators.notEmpty()]
         ),
         Field(
             item="60",