Merge branch 'develop' into 2794-case-number

.circleci/build-and-test/jobs.yml

@@ -4,7 +4,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-backend
+      - docker-compose-up-with-elastic-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
           command: |
@@ -47,7 +47,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-backend
+      - docker-compose-up-with-elastic-backend
       - docker-compose-up-frontend
       - install-nodejs-machine
       - disable-npm-audit
@@ -61,7 +61,7 @@
                wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
       - run:
           name: apply the migrations
-          command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate" 
+          command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate"
       - run:
           name: Remove existing cypress test users
           command: cd tdrs-backend; docker-compose exec web python manage.py delete_cypress_users -usernames [email protected] [email protected]

.circleci/util/commands.yml

@@ -11,6 +11,12 @@
           name: Build and spin-up Django API service
           command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
 
+  docker-compose-up-with-elastic-backend:
+    steps:
+      - run:
+          name: Build and spin-up Django API service
+          command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build
+
   cf-check:
     steps:
       - run:

docs/Sprint-Review/sprint-91-summary.md

@@ -0,0 +1,62 @@
+# Sprint 91 Summary
+
+01/17/2024 - 01/30/2024
+
+Velocity (Dev): 24
+
+## Sprint Goal
+* Dev:
+    * Continue parsing engine development and begin work on enhancement tickets
+        * #2536 Cat 4 validation 
+    * #1858 Secure OFA staff access to Kibana 
+        * Unblocks #1350 when complete 
+* DevOps:
+    * #2790 - Update deployment code to support Kibana and integrate with Standing Elastic instance
+* Design: 
+    * Tie up current documentation work
+    * Continue refinement of research roadmap
+
+
+## Tickets
+### Completed/Merged
+* [#2751 Resource Card updated with latest coding instructions](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2751)
+
+### Ready to Merge
+* [#2772 Elastic bulk document creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2772)
+* [#1350 Kibana access from TDP](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1350)
+* [#1858 Spike: Secure Kibana access](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1858)
+* [#2711 Catch report month / year mismatches](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2711)
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2790 Kibana Deployment](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2790)
+* [#2681 Section 1 Validation clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2681)
+
+
+
+### Closed (not merged)
+* N/A
+
+
+---
+
+## Moved to Next Sprint (In Progress, Blocked, Raft Review)
+### In Progress
+* [#2646 - Populate data file summary case aggregates differently per section](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2646)
+* [#2820 [bug] Uncaught exception re: parsing error preventing feedback report generation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2820)
+* [#2768 Fix production OWASP scan reporting](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2768)
+* [#2799 Generate error mismatching field rpt_month_year w/ header](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2799)
+* [#2781 As a developer,  I want to have documentation on django migration best practices](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2781)
+
+
+### Blocked
+* N/A
+
+### Raft Review
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+* [#2746 As an STT, I need to know if there are issues with the DOBs reported in my data files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2746)
+* [#2813 Reduce dev environment count](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2813)
+* [#2729 As a developer, I want to move migration commands in the pipeline to CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2729)

docs/Technical-Documentation/migration-best-practices.md

@@ -0,0 +1,80 @@
+# Project migrations and database best practices
+
+## Indexing
+
+- If some column is going to be queried repeatedly then, create the database indexes on that column.
+- Django migration has built-in support for creating indexes in database.
+- If multiple columns are going to be queried together then index_together can be used to create composite indexes.
+- Tools such as pgbadger can be used to analyse the query pattern on the production.
+- PostgreSQL has support for different kind of indexes which can outperform the performance in various conditions and workloads. Research about it for other databases and apply the best one for the application.
+
+## Naming
+
+Migrations have to be named as follows:
+
+{migration number}_{model_name}_{change being applied}
+
+E.g:
+
+```shell
+0002_datafilesumary_change_pk_to_uuid.py
+```
+
+
+## Data migration
+
+It is possible to create/add data in the migration. This is called data migration and is a feature available in Django, however, it has it's own shortcomings and should be avoided.
+
+## Version control
+
+All migrations shall be version controlled. Each PR should only include one migration. This ensures better control over the model changes and gives a better option to revert changes back to previous version.
+
+However, in cloud.gov the database has limited disk space. Since alter/add transactions in one migration need to cache the status in disk and then apply the changes, it is strongly suggested to limit the number of transactions and changes in one migration. 
+
+Considering notes above, we conclude that there is a tradeoff between the number of migrations and the number of changes in one migration. One should be careful with this tradeoff specifically in apps that have larger database tables such as search_indexes.
+
+## Back up before migration
+
+Before applying any new migration into production, take a full backup from the DB.
+
+## Check before migration
+
+Check django [migration transactions](https://docs.djangoproject.com/en/3.2/topics/migrations/#transactions) details for database and assess the behavior on what will happen in case of server network lost between the server running the migration and db, server went out of memory, server freeze etc behaviors.
+
+## Useful commands
+
+### List the existing migrations
+
+```python
+from django.db.migrations.recorder import MigrationRecorder
+
+existing_migration = MigrationRecorder.Migration.objects.all()
+for migration in existing_migration:
+    print(migration)
+```
+
+will output lines similar to:
+
+```
+.
+..
+Migration 0007_ssp_m1_ssp_m2_ssp_m3 for search_indexes                       
+Migration 0038_user_access_requested_date for users
+Migration 0008_auto_20230522_1850 for search_indexes
+Migration 0002_alter_parsererror_error_type for parsers
+Migration 0003_auto_20230518_1339 for parsers
+Migration 0004_parsererror_object_uuid for parsers
+Migration 0005_auto_20230601_1510 for parsers
+Migration 0006_alter_parsererror_item_number for parsers
+Migration 0009_auto_20230525_1959 for search_indexes
+Migration 0010_add_tmp_uuid for search_indexes
+Migration 0039_alter_user_options for users
+Migration 0011_gen_uuid for search_indexes
+Migration 0012_set_uuid_pk for search_indexes
+Migration 0013_rename_uuid for search_indexes
+Migration 0014_auto_20230707_1952 for search_indexes
+Migration 0006_auto_20230726_1448 for parsers
+Migration 0015_auto_20230724_1830 for search_indexes
+Migration 0016_auto_20230803_1721 for search_indexes
+Migration 0006_auto_20230810_1500 for parsers
+```

tdrs-backend/docker-compose.yml

@@ -50,20 +50,50 @@ services:
     ports:
       - 5601:5601
     environment:
-      - xpack.security.encryptionKey="something_at_least_32_characters"
+      - xpack.security.encryptionKey=${KIBANA_ENCRYPTION_KEY:-something_at_least_32_characters}
       - xpack.security.session.idleTimeout="1h"
       - xpack.security.session.lifespan="30d"
     volumes:
       - ./kibana.yml:/usr/share/kibana/config/kibana.yml
     depends_on:
       - elastic
 
+  # This task only needs to be performed once, during the *initial* startup of
+  # the stack. Any subsequent run will reset the passwords of existing users to
+  # the values defined inside the '.env' file, and the built-in roles to their
+  # default permissions.
+  #
+  # By default, it is excluded from the services started by 'docker compose up'
+  # due to the non-default profile it belongs to. To run it, either provide the
+  # '--profile=elastic_setup' CLI flag to Compose commands, or "up" the service by name
+  # such as 'docker compose up elastic_setup'.
+  elastic_setup:
+    profiles:
+      - elastic_setup
+    build:
+      context: elastic_setup/
+      args:
+        ELASTIC_VERSION: "7.17.6"
+    init: true
+    environment:
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-changeme}
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-changeme}
+      OFA_ADMIN_PASSWORD: ${OFA_ADMIN_PASSWORD:-changeme}
+      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-elastic}
+    depends_on:
+      - elastic
+
   elastic:
     image: elasticsearch:7.17.6
     environment:
       - discovery.type=single-node
       - logger.discovery.level=debug
-      - xpack.security.enabled=false
+      - xpack.security.enabled=true
+      - xpack.security.authc.anonymous.username="ofa_admin"
+      - xpack.security.authc.anonymous.roles="ofa_admin"
+      - xpack.security.authc.anonymous.authz_exception=true
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-changeme}
+      - KIBANA_SYSTEM_PASSWORD=${KIBANA_SYSTEM_PASSWORD:-changeme}
     ports:
       - 9200:9200
       - 9300:9300
@@ -101,6 +131,7 @@ services:
       - CYPRESS_TOKEN
       - DJANGO_DEBUG
       - SENDGRID_API_KEY
+      - BYPASS_KIBANA_AUTH
     volumes:
       - .:/tdpapp
     image: tdp

tdrs-backend/elastic_setup/Dockerfile

@@ -0,0 +1,10 @@
+ARG ELASTIC_VERSION
+
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+
+COPY . /
+
+RUN ["chmod", "+x", "/entrypoint.sh"]
+RUN ["chmod", "+x", "/util.sh"]
+
+ENTRYPOINT ["/entrypoint.sh"]

tdrs-backend/elastic_setup/entrypoint.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source "${BASH_SOURCE[0]%/*}"/util.sh
+
+
+# --------------------------------------------------------
+# Users declarations
+
+declare -A users_passwords
+users_passwords=(
+	[kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}"
+	[ofa_admin]="${OFA_ADMIN_PASSWORD:-}"
+)
+
+declare -A users_roles
+users_roles=(
+	[kibana_system]='kibana_system'
+    [ofa_admin]='kibana_admin'
+)
+
+# --------------------------------------------------------
+# Roles declarations for custom roles
+
+declare -A roles_files
+roles_files=(
+
+)
+
+# --------------------------------------------------------
+
+
+log 'Waiting for availability of Elasticsearch. This can take several minutes.'
+
+declare -i exit_code=0
+wait_for_elasticsearch || exit_code=$?
+
+if ((exit_code)); then
+	case $exit_code in
+		6)
+			suberr 'Could not resolve host. Is Elasticsearch running?'
+			;;
+		7)
+			suberr 'Failed to connect to host. Is Elasticsearch healthy?'
+			;;
+		28)
+			suberr 'Timeout connecting to host. Is Elasticsearch healthy?'
+			;;
+		*)
+			suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}"
+			;;
+	esac
+
+	exit $exit_code
+fi
+
+sublog 'Elasticsearch is running'
+
+log 'Waiting for initialization of built-in users'
+
+wait_for_builtin_users || exit_code=$?
+
+if ((exit_code)); then
+	suberr 'Timed out waiting for condition'
+	exit $exit_code
+fi
+
+sublog 'Built-in users were initialized'
+
+for role in "${!roles_files[@]}"; do
+	log "Role '$role'"
+
+	declare body_file
+	body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}"
+	if [[ ! -f "${body_file:-}" ]]; then
+		sublog "No role body found at '${body_file}', skipping"
+		continue
+	fi
+
+	sublog 'Creating/updating'
+	ensure_role "$role" "$(<"${body_file}")"
+done
+
+for user in "${!users_passwords[@]}"; do
+	log "User '$user'"
+	if [[ -z "${users_passwords[$user]:-}" ]]; then
+		sublog 'No password defined, skipping'
+		continue
+	fi
+
+	declare -i user_exists=0
+	user_exists="$(check_user_exists "$user")"
+
+	if ((user_exists)); then
+		sublog 'User exists, setting password'
+		set_user_password "$user" "${users_passwords[$user]}"
+	else
+		if [[ -z "${users_roles[$user]:-}" ]]; then
+			suberr '  No role defined, skipping creation'
+			continue
+		fi
+
+		sublog 'User does not exist, creating'
+		create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}"
+	fi
+done
+
+log "Elastic setup completed. Exiting with code: $?"