diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml index 634a3c29b..02d3c8f53 100644 --- a/.circleci/base_config.yml +++ b/.circleci/base_config.yml @@ -13,11 +13,11 @@ executors: machine-executor: machine: docker_layer_caching: false - image: ubuntu-2204:2024.01.1 + image: ubuntu-2204:2024.05.1 large-machine-executor: machine: docker_layer_caching: false - image: ubuntu-2204:2024.01.1 + image: ubuntu-2204:2024.05.1 resource_class: large parameters: diff --git a/.circleci/build-and-test/commands.yml b/.circleci/build-and-test/commands.yml index 52cfe7149..70ef3f98d 100644 --- a/.circleci/build-and-test/commands.yml +++ b/.circleci/build-and-test/commands.yml @@ -49,6 +49,13 @@ - run: name: Disable npm audit warnings in CI command: npm set audit false - + # This allows us to use the node orb to install packages within other commands install-nodejs-packages: node/install-packages + + docker-login: + steps: + - run: + name: Docker login + command: | + echo "$CIRCI_DOCKER_LOGIN" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-circi --password-stdin diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml index a40d1568f..469c92250 100644 --- a/.circleci/build-and-test/jobs.yml +++ b/.circleci/build-and-test/jobs.yml @@ -3,6 +3,7 @@ steps: - checkout - docker-compose-check + - docker-login - docker-compose-up-backend - run: name: Run Unit Tests And Create Code Coverage Report @@ -46,6 +47,7 @@ steps: - checkout - docker-compose-check + - docker-login - docker-compose-up-backend - docker-compose-up-frontend - install-nodejs-machine diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index af907351b..d1aa82b7d 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -1,4 +1,33 @@ # commands: + init-deploy: + steps: + - checkout + - sudo-check + - cf-check + + build-and-tag-images: + parameters: + backend-appname: + default: tdp-backend + type: string + frontend-appname: + default: tdp-frontend + type: string + steps: + - run: + name: Update Docker daemon + command: | + sudo echo '{"max-concurrent-uploads": 1}' | sudo tee /etc/docker/daemon.json + sudo service docker restart + - run: + name: Create builder + command: | + docker buildx create --name container-builder --driver docker-container --use --bootstrap + - run: + name: Build and tag images + command: | + ./scripts/build-and-tag-images.sh <> <> ./tdrs-backend ./tdrs-frontend $CIRCLE_BUILD_NUM $CIRCLE_SHA1 "$CIRCI_DOCKER_LOGIN" tdp-circi + deploy-cloud-dot-gov: parameters: environment: @@ -25,9 +54,6 @@ default: tdp-frontend type: string steps: - - checkout - - sudo-check - - cf-check - login-cloud-dot-gov: cf-password: <> cf-org: <> diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index 63d5bc070..ce163101f 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -1,3 +1,33 @@ + build-and-tag-develop: + executor: large-machine-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - build-and-tag-images: + backend-appname: tdp-backend-develop + frontend-appname: tdp-frontend-develop + + build-and-tag-staging: + executor: large-machine-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - build-and-tag-images: + backend-appname: tdp-backend-staging + frontend-appname: tdp-frontend-staging + + build-and-tag-production: + executor: large-machine-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - build-and-tag-images: + backend-appname: tdp-backend-production + frontend-appname: tdp-frontend-production + deploy-dev: parameters: target_env: @@ -5,6 +35,7 @@ executor: docker-executor working_directory: ~/tdp-deploy steps: + - init-deploy - deploy-cloud-dot-gov: backend-appname: tdp-backend-<< parameters.target_env >> frontend-appname: tdp-frontend-<< parameters.target_env >> @@ -13,6 +44,7 @@ executor: docker-executor working_directory: ~/tdp-deploy steps: + - init-deploy - deploy-cloud-dot-gov: backend-appname: tdp-backend-staging frontend-appname: tdp-frontend-staging @@ -24,6 +56,7 @@ executor: docker-executor working_directory: ~/tdp-deploy steps: + - init-deploy - deploy-cloud-dot-gov: backend-appname: tdp-backend-develop frontend-appname: tdp-frontend-develop @@ -133,6 +166,7 @@ executor: docker-executor working_directory: ~/tdp-deploy steps: + - init-deploy - deploy-cloud-dot-gov: environment: production backend-appname: tdp-backend-prod diff --git a/.circleci/deployment/workflows.yml b/.circleci/deployment/workflows.yml index 8a4269c04..a0de09f9e 100644 --- a/.circleci/deployment/workflows.yml +++ b/.circleci/deployment/workflows.yml @@ -93,27 +93,48 @@ - develop - main - master - - deploy-develop: + - build-and-tag-develop: requires: - deploy-infrastructure-staging filters: branches: only: - develop - - deploy-staging: + - deploy-develop: + requires: + - build-and-tag-develop + filters: + branches: + only: + - develop + - build-and-tag-staging: requires: - deploy-infrastructure-staging filters: branches: only: - main - - deploy-production: + - deploy-staging: + requires: + - build-and-tag-staging + filters: + branches: + only: + - main + - build-and-tag-production: requires: - deploy-infrastructure-production filters: branches: only: - master + - deploy-production: + requires: + - build-and-tag-production + filters: + branches: + only: + - master - test-deployment-e2e: requires: - deploy-develop diff --git a/.circleci/generate_config.sh b/.circleci/generate_config.sh old mode 100644 new mode 100755 diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml index 225758ef5..fdabb0a22 100644 --- a/.circleci/owasp/jobs.yml +++ b/.circleci/owasp/jobs.yml @@ -4,6 +4,7 @@ steps: - checkout - docker-compose-check + - docker-login - docker-compose-up-backend - docker-compose-up-frontend - run: @@ -26,6 +27,7 @@ steps: - checkout - docker-compose-check + - docker-login - docker-compose-up-backend - docker-compose-up-frontend - run: @@ -66,6 +68,7 @@ - sudo-check - cf-check - docker-compose-check + - docker-login - login-cloud-dot-gov: cf-password: <> cf-space: <> diff --git a/Taskfile.yml b/Taskfile.yml index 2c67784b9..ac4812394 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,6 +2,11 @@ version: '3' tasks: + upload-kibana-objs: + desc: Upload dashboards to Kibana server + cmds: + - curl -X POST localhost:5601/api/saved_objects/_import -H "kbn-xsrf: true" --form file=@tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson + create-network: desc: Create the external network cmds: @@ -29,7 +34,7 @@ tasks: desc: Create Sentry service dir: sentry cmds: - # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask : --cpuset-cpus 0 + # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask : --cpuset-cpus 0 - (docker run --privileged -p 9001:9000 -d --memory="8g" --memory-swap="8g" --name sentry docker:dind) || true - docker exec sentry sh -c "git clone https://github.com/getsentry/self-hosted.git || true" @@ -155,7 +160,7 @@ tasks: - docker rm $(docker ps -aq) || true - docker rmi $(docker images -q) || true - docker volume rm $(docker volume ls -q) || true - + clamav-up: desc: Start clamav service dir: tdrs-backend @@ -187,7 +192,7 @@ tasks: - task: frontend-up - task: clamav-up - + # need more work frontend-init: desc: Initialize the frontend project diff --git a/docs/Sprint-Review/sprint-107-summary.md b/docs/Sprint-Review/sprint-107-summary.md new file mode 100644 index 000000000..e6d1aa4d9 --- /dev/null +++ b/docs/Sprint-Review/sprint-107-summary.md @@ -0,0 +1,89 @@ +# sprint-107-summary + +8/28/2024 - 9/10/2024 + +### Priority Setting + +* Re-parsing epic +* Postgres db access +* UX research with DIGIT team +* Continuous communication with STTs about latest TDP features and updates + +### Sprint Goal + +**Dev:** + +_**Re-parsing, Admin Console Improvements, and Application Health Monitoring work**_ + +* \#3106 — Re-Parse Django Action +* \#3137 — \[bug] OFA unable to export data to csv by record type and fiscal period +* \#3074 — TDP Data Files page permissions for DIGIT & Sys Admin user groups +* \#3044 — Prometheus/Grafana - Local Environment +* \#3042 — Sentry in cloud.gov + +**DevOps:** +_**Successful deployments across environments and pipeline stability investments**_ + +* \#2965 — As tech lead, I want a database seed implemented for testing +* \#2458 — Integrate Nexus into CircleCI + +**Design:** + +_**Support reviews, In-app banner to support parsed data, Continue Error Audit (Cat 4)**_ + +* \#3156 — Release Notes Email Template +* \#3100 — \[Design Deliverable] Update stakeholders & personas document +* \#2968 — \[Design Deliverable] Update Error Audit for Cat 4 / QA + +## Tickets + +### Completed/Merged + +* [#2561 As a sys admin, I need TDP to automatically deactivate accounts that are inactive for 180 days](https://github.com/raft-tech/TANF-app/issues/2561) +* [#2792 \[Error Audit\] Category 3 error messages clean-up ](https://github.com/raft-tech/TANF-app/issues/2792) +* [#3043 Sentry: Local environment for Debugging](https://github.com/raft-tech/TANF-app/issues/3043) +* [#3064 Re-parse Meta Model](https://github.com/raft-tech/TANF-app/issues/3064) +* [#3065 Spike - Guarantee Sequential Execution of Re-parse Command](https://github.com/raft-tech/TANF-app/issues/3065) +* [#3074 TDP Data Files page permissions for DIGIT & Sys Admin user groups ](https://github.com/raft-tech/TANF-app/issues/3074) +* [#3076 Admin Filter Enhancements for Data Files Page ](https://github.com/raft-tech/TANF-app/issues/3076) +* [#3078 \[Research Synthesis\] DIGIT Admin Experience Improvements ](https://github.com/raft-tech/TANF-app/issues/3078) +* [#3087 Admin By Newest Filter Enhancements for Data Files Page ](https://github.com/raft-tech/TANF-app/issues/3087) +* [#3114 \[Design Spike\] In-app banner for submission history pages w/ data parsed before May 2024 ](https://github.com/raft-tech/TANF-app/issues/3114) +* [#3142 \[Research Spike\] Get more detail about Yun & DIGIT's data workflow and use cases ](https://github.com/raft-tech/TANF-app/issues/3142) + +### Submitted (QASP Review, OCIO Review) + +* + +### Ready to Merge + +* [#2883 Pre-Made Reporting Dashboards on Kibana ](https://github.com/raft-tech/TANF-app/issues/2883) +* [#3102 Admin Exp: Django Implement Multi-Select Fiscal Period Dropdown For Data Export ](https://github.com/raft-tech/TANF-app/issues/3102) + +### Closed (Not Merged) + +* [#3110 Spike - Investigate Custom Filter Integration ](https://github.com/raft-tech/TANF-app/issues/3110) +* [#3156 Release Notes Knowledge Center and Email Template ](https://github.com/raft-tech/TANF-app/issues/3156) + +### Moved to Next Sprint + +**In Progress** + +* [#2968 \[Design Deliverable\] Update Error Audit for Cat 4 / QA ](https://github.com/raft-tech/TANF-app/issues/2968) +* [#3060 As a TDP user, I need to stay logged in when I'm actively using the system ](https://github.com/raft-tech/TANF-app/issues/3060) +* [#3100 \[Design Deliverable\] Update stakeholders & personas document ](https://github.com/raft-tech/TANF-app/issues/3100) +* [#3106 Re-Parse Django Action ](https://github.com/raft-tech/TANF-app/issues/3106) +* [#3137 \[bug\] OFA unable to export data to csv by record type and fiscal period ](https://github.com/raft-tech/TANF-app/issues/3137) +* [#3164 \[Research Synthesis\] Yun & DIGIT's data workflow and use cases ](https://github.com/raft-tech/TANF-app/issues/3164) +* [#3170 Reparse Command Fails when Queryset is Large ](https://github.com/raft-tech/TANF-app/issues/3170) +* [#3179 Spike - How We Work / Hopes & Fears Workshop prep ](https://github.com/raft-tech/TANF-app/issues/3179) + +**Blocked** + +* + +**Raft Review** + +* [#2458 Integrate Nexus into CircleCI ](https://github.com/raft-tech/TANF-app/issues/2458) +* [#2965 As tech lead, I want a database seed implemented for testing ](https://github.com/raft-tech/TANF-app/issues/2965) +* [#3044 Prometheus/Grafana - Local Environment ](https://github.com/raft-tech/TANF-app/issues/3044) diff --git a/docs/Technical-Documentation/images/nexus-dev-admin-login.png b/docs/Technical-Documentation/images/nexus-dev-admin-login.png new file mode 100644 index 000000000..d3b00e903 Binary files /dev/null and b/docs/Technical-Documentation/images/nexus-dev-admin-login.png differ diff --git a/docs/Technical-Documentation/nexus-repo.md b/docs/Technical-Documentation/nexus-repo.md index 6f4a15bf5..5e504a384 100644 --- a/docs/Technical-Documentation/nexus-repo.md +++ b/docs/Technical-Documentation/nexus-repo.md @@ -40,7 +40,7 @@ After logging in as root for the first time, you will be taken to a page to set In order to use Nexus as a Docker repository, the DNS for the repo needs to be able to terminate https. We are currently using cloudflare to do this. -When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port. +When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port. In order to allow [Docker client login and connections](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats/docker-registry/docker-authentication) you must set up the Docker Bearer Token Realm in Settings -> Security -> Realms -> and move the Docker Bearer Token Realm over to Active. Also, any users will need nx-repository-view-docker-#{RepoName}-(browse && read) at a minimum and (add and edit) in order to push images. @@ -48,21 +48,86 @@ Also, any users will need nx-repository-view-docker-#{RepoName}-(browse && read) We have a separate endpoint to connect specifically to the docker repository. [https://tdp-docker.dev.raftlabs.tech](tdp-docker.dev.raftlabs.tech) -e.g. `docker login https://tdp-docker.dev.raftlabs.tech` +e.g. +``` +docker login https://tdp-docker.dev.raftlabs.tech +``` ### Pushing Images Before an image can be pushed to the nexus repository, it must be tagged for that repo: -`docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}` +``` +docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version} +``` then you can push: -`docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}` +``` +docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version} +``` ### Pulling Images -We have set up a proxy mirror to dockerhub that can pull and cache DockerHub images. -Then we have created a group docker repository that can be pulled from. If the container is in our hosted repo, the group will return that container. If not, it will see if we have a cached version of that container in our proxy repo and, if not, pull that from dockerhub, cache it and allow the docker pull to happen. +We do not allow anonymous access on our Nexus instance. With that said, if you have not [logged in with Docker](#docker-login) you will not be able to pull. If you are logged in: + +``` +docker pull tdp-docker.dev.raftlabs.tech/${ImageName}:${Version} +``` + +## Nexus Administration + +### UI Admin Login +To administer Nexus via the UI, you will need to access the service key in our dev cloud.gov environment. + +Log in with CloudFoundry +``` +cf login --sso +``` +Be sure to specify the space as `tanf-dev` -`docker pull https://tdp-docker-store.dev.raftlabs.tech/${ImageName}:${Version}` \ No newline at end of file +After you've authenticated you can grab the password from the key: +``` +cf service-key tanf-keys nexus-dev-admin +``` + +The key returns a username and a password: +``` +{ + "credentials": { + "password": REDACTED, + "username": REDACTED + } +} +``` +Copy the `password` to your clipboard and login into the Nexus UI with the `tdp-dev-admin` user. See below: + +![Nexus Dev Admin Login](./images/nexus-dev-admin-login.png) + +### VM Login +To access the VM running Nexus, you will need to gain access to the Raft internal network. To do this, you will need to install CloudFlare's WARP zero trust VPN. Follow the instructions [here](https://gorafttech-my.sharepoint.com/:w:/g/personal/tradin_teamraft_com/EZePOTv0dbdBguHITcoXQF0Bd5JAcqeLsJTlEOktTfIXHA?e=34WqB4) to get setup. From there, reach out to Eric Lipe or Connor Meehan for the IP, username, and password to access the VM. Once you have the credentials, you can login with SSH: +``` +ssh username@IP_Address +``` + +Once logged in, you can run `docker ps` or other docker commands to view and administer the Nexus container as necessary. You should also consider generating an ssh key to avoid having to enter the password each time you login. To do so, run the following commands on your local machine. +``` +ssh-keygen +``` + +``` +ssh-copy-id username@IP_Address +``` +Now you will no longer have to enter the password when logging in. + +## Local Docker Login +After logging into the `tanf-dev` space with the `cf` cli, execute the following commands to authenticate your local docker daemon +``` +export NEXUS_DOCKER_PASSWORD=`cf service-key tanf-keys nexus-dev | tail -n +2 | jq .credentials.password` +echo "$NEXUS_DOCKER_PASSWORD" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-dev --password-stdin +``` + +Sometimes the `docker login...` command above doesn't work. If that happens, just copy the content of `NEXUS_DOCKER_PASSWORD` to your clipboard and paste it when prompted for the password after executing the command below. +``` +docker login https://tdp-docker.dev.raftlabs.tech -u tdp-dev +``` diff --git a/scripts/apply-remote-migrations.sh b/scripts/apply-remote-migrations.sh index 897f4d04f..a067109bf 100644 --- a/scripts/apply-remote-migrations.sh +++ b/scripts/apply-remote-migrations.sh @@ -5,7 +5,7 @@ app=${1} cd ./tdrs-backend echo "Install dependencies..." -sudo apt install -y gcc +sudo apt-get install -y gcc && sudo apt-get install -y graphviz && sudo apt-get install -y graphviz-dev sudo apt install -y libpq-dev python3-dev python -m venv ./env diff --git a/scripts/build-and-tag-images.sh b/scripts/build-and-tag-images.sh new file mode 100755 index 000000000..679485d79 --- /dev/null +++ b/scripts/build-and-tag-images.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +if [ "$#" -ne 8 ]; then + echo "Error, this script expects 8 parameters." + echo "I.e: ./build-tag-images.sh BACKEND_APP_NAME FRONTEND_APP_NAME BACKEND_PATH FRONTEND_PATH BUILD_NUM COMMIT_HASH DOCKER_LOGIN DOCKER_USER" + exit 1 +fi + +BACKEND_APP_NAME=$1 +FRONTEND_APP_NAME=$2 +BACKEND_PATH=$3 +FRONTEND_PATH=$4 +BUILD_NUM=$5 +COMMIT_HASH=$6 +DOCKER_LOGIN=$7 +DOCKER_USER=$8 +BUILD_DATE=`date +%F` +TAG="${BUILD_DATE}_build-${BUILD_NUM}_${COMMIT_HASH}" + +export DOCKER_CLI_EXPERIMENTAL=enabled + +build_and_tag() { + echo "$DOCKER_LOGIN" | docker login https://tdp-docker.dev.raftlabs.tech -u $DOCKER_USER --password-stdin + docker buildx build --load --platform linux/amd64 -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:latest "$BACKEND_PATH" + docker buildx build --load --platform linux/arm64 -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:latest "$BACKEND_PATH" + docker push --all-tags tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME + docker buildx build --load --platform linux/amd64 -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:latest "$FRONTEND_PATH" + docker buildx build --load --platform linux/arm64 -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:latest "$FRONTEND_PATH" + docker push --all-tags tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME + docker logout +} + +echo "Building and Tagging images for $BACKEND_APP_NAME and $FRONTEND_APP_NAME" +build_and_tag diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index ebbce8243..a1e3f2583 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -99,6 +99,10 @@ update_kibana() cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601 cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601 cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80 + + # Upload dashboards to Kibana + CMD="curl -X POST $CGAPPNAME_KIBANA.apps.internal:5601/api/saved_objects/_import -H 'kbn-xsrf: true' --form file=@/home/vcap/app/tdpservice/search_indexes/kibana_saved_objs.ndjson" + cf run-task $CGAPPNAME_BACKEND --command "$CMD" --name kibana-obj-upload } update_backend() diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile index 443d8f070..492487ec7 100644 --- a/tdrs-backend/Dockerfile +++ b/tdrs-backend/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.8-slim-buster +FROM python:3.10.8-slim-bullseye ENV PYTHONUNBUFFERED 1 ARG user=tdpuser diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index fe6562492..3330ae493 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.4" services: zaproxy: - image: softwaresecurityproject/zap-stable:2.14.0 + image: tdp-docker.dev.raftlabs.tech/dependencies/softwaresecurityproject/zap-stable:2.14.0 command: sleep 3600 depends_on: - web @@ -12,7 +12,7 @@ services: - ../scripts/zap-hook.py:/zap/scripts/zap-hook.py:ro postgres: - image: postgres:15.7 + image: tdp-docker.dev.raftlabs.tech/dependencies/postgres:15.7 environment: - PGDATA=/var/lib/postgresql/data/ - POSTGRES_DB=tdrs_test @@ -25,14 +25,14 @@ services: - postgres_data:/var/lib/postgresql/data/:rw clamav-rest: - image: rafttech/clamav-rest:0.103.2 + image: tdp-docker.dev.raftlabs.tech/dependencies/rafttech/clamav-rest:0.103.2 environment: - MAX_FILE_SIZE=200M ports: - "9000:9000" localstack: - image: localstack/localstack:0.13.3 + image: tdp-docker.dev.raftlabs.tech/dependencies/localstack/localstack:0.13.3 environment: - SERVICES=s3 - DATA_DIR=/tmp/localstack/data @@ -46,7 +46,7 @@ services: - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh kibana: - image: docker.elastic.co/kibana/kibana-oss:7.10.2 + image: tdp-docker.dev.raftlabs.tech/dependencies/docker.elastic.co/kibana/kibana-oss:7.10.2 ports: - 5601:5601 environment: @@ -55,11 +55,13 @@ services: - SERVER_BASEPATH=/kibana - SERVER_SECURITYRESPONSEHEADERS_REFERRERPOLICY=no-referrer - CSP_WARNLEGACYBROWSERS=false + volumes: + - ./search_indexes/kibana_saved_objs.ndjson:/usr/share/kibana/kibana_saved_objs.ndjson depends_on: - elastic elastic: - image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2 + image: tdp-docker.dev.raftlabs.tech/dependencies/docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2 environment: - discovery.type=single-node - logger.discovery.level=debug @@ -177,7 +179,7 @@ services: volumes: - .:/tdpapp - logs:/tdpapp - image: tdp + image: tdp-backend build: . command: > bash -c "./wait_for_services.sh && @@ -196,7 +198,7 @@ services: - elastic redis-server: - image: "redis:alpine" + image: tdp-docker.dev.raftlabs.tech/dependencies/redis:alpine command: redis-server /tdpapp/redis.conf ports: - "6379:6379" diff --git a/tdrs-backend/manifest.kibana.yml b/tdrs-backend/manifest.kibana.yml index 181b29ec0..da77a16d4 100644 --- a/tdrs-backend/manifest.kibana.yml +++ b/tdrs-backend/manifest.kibana.yml @@ -10,7 +10,7 @@ applications: SERVER_SECURITYRESPONSEHEADERS_REFERRERPOLICY: no-referrer CSP_WARNLEGACYBROWSERS: false docker: - image: docker.elastic.co/kibana/kibana-oss:7.4.2 + image: docker.elastic.co/kibana/kibana-oss:7.10.2 command: | export ELASTICSEARCH_HOSTS=http://$CGAPPNAME_PROXY.apps.internal:8080 && /usr/local/bin/dumb-init -- /usr/local/bin/kibana-docker diff --git a/tdrs-backend/tdpservice/conftest.py b/tdrs-backend/tdpservice/conftest.py index 416a4a890..0437ef13b 100644 --- a/tdrs-backend/tdpservice/conftest.py +++ b/tdrs-backend/tdpservice/conftest.py @@ -395,6 +395,12 @@ def test_private_key(): yield get_private_key(key) +@pytest.fixture() +def system_user(): + """Create system user.""" + return UserFactory.create(username='system') + + # Register factories with pytest-factoryboy for automatic dependency injection # of model-related fixtures into tests. register(OwaspZapScanFactory) diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index c00541419..6fe5355e0 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -5,6 +5,7 @@ from io import StringIO from typing import Union +from django.conf import settings from django.contrib.admin.models import ADDITION, ContentType, LogEntry from django.core.files.base import File from django.db import models @@ -206,6 +207,10 @@ def submitted_by(self): """Return the author as a string for this data file.""" return self.user.get_full_name() + def admin_link(self): + """Return a link to the admin console for this file.""" + return f"{settings.FRONTEND_BASE_URL}/admin/data_files/datafile/?id={self.pk}" + @classmethod def create_new_version(self, data): """Create a new version of a data file with an incremented version.""" diff --git a/tdrs-backend/tdpservice/data_files/tasks.py b/tdrs-backend/tdpservice/data_files/tasks.py new file mode 100644 index 000000000..16e35de79 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/tasks.py @@ -0,0 +1,48 @@ +"""Celery shared tasks for use in scheduled jobs.""" + +from celery import shared_task +from datetime import timedelta +from django.utils import timezone +from django.contrib.auth.models import Group +from django.db.models import Q, Count +from tdpservice.users.models import AccountApprovalStatusChoices, User +from tdpservice.data_files.models import DataFile +from tdpservice.parsers.models import DataFileSummary +from tdpservice.email.helpers.data_file import send_stuck_file_email + + +def get_stuck_files(): + """Return a queryset containing files in a 'stuck' state.""" + stuck_files = DataFile.objects.annotate(reparse_count=Count('reparse_meta_models')).filter( + # non-reparse submissions over an hour old + Q( + reparse_count=0, + created_at__lte=timezone.now() - timedelta(hours=1), + ) | # OR + # reparse submissions past the timeout, where the reparse did not complete + Q( + reparse_count__gt=0, + reparse_meta_models__timeout_at__lte=timezone.now(), + reparse_meta_models__finished=False, + reparse_meta_models__success=False + ) + ).filter( + # where there is NO summary or the summary is in PENDING status + Q(summary=None) | Q(summary__status=DataFileSummary.Status.PENDING) + ) + + return stuck_files + + +@shared_task +def notify_stuck_files(): + """Find files stuck in 'Pending' and notify SysAdmins.""" + stuck_files = get_stuck_files() + + if stuck_files.count() > 0: + recipients = User.objects.filter( + account_approval_status=AccountApprovalStatusChoices.APPROVED, + groups=Group.objects.get(name='OFA System Admin') + ).values_list('username', flat=True).distinct() + + send_stuck_file_email(stuck_files, recipients) diff --git a/tdrs-backend/tdpservice/data_files/test/test_stuck_files.py b/tdrs-backend/tdpservice/data_files/test/test_stuck_files.py new file mode 100644 index 000000000..95f4f8f3a --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/test/test_stuck_files.py @@ -0,0 +1,252 @@ +"""Test the get_stuck_files function.""" + + +import pytest +from datetime import timedelta +from django.utils import timezone +from tdpservice.data_files.models import DataFile +from tdpservice.parsers.models import DataFileSummary +from tdpservice.data_files.tasks import get_stuck_files +from tdpservice.parsers.test.factories import ParsingFileFactory, DataFileSummaryFactory, ReparseMetaFactory + + +def _time_ago(hours=0, minutes=0, seconds=0): + return timezone.now() - timedelta(hours=hours, minutes=minutes, seconds=seconds) + + +def make_datafile(stt_user, stt, version): + """Create a test data file with default params.""" + datafile = ParsingFileFactory.create( + quarter=DataFile.Quarter.Q1, section=DataFile.Section.ACTIVE_CASE_DATA, + year=2023, version=version, user=stt_user, stt=stt + ) + return datafile + + +def make_summary(datafile, status): + """Create a test data file summary given a file and status.""" + return DataFileSummaryFactory.create( + datafile=datafile, + status=status, + ) + + +def make_reparse_meta(finished, success): + """Create a test reparse meta model.""" + return ReparseMetaFactory.create( + timeout_at=_time_ago(hours=1), + finished=finished, + success=success + ) + + +@pytest.mark.django_db +def test_find_pending_submissions__none_stuck(stt_user, stt): + """Finds no stuck files.""" + # an accepted standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.ACCEPTED) + + # an accepted reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.ACCEPTED) + rpm = make_reparse_meta(True, True) + df2.reparse_meta_models.add(rpm) + + # a pending standard submission, less than an hour old + df3 = make_datafile(stt_user, stt, 3) + df3.created_at = _time_ago(minutes=40) + df3.save() + make_summary(df3, DataFileSummary.Status.PENDING) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 0 + + +@pytest.mark.django_db +def test_find_pending_submissions__non_reparse_stuck(stt_user, stt): + """Finds standard upload/submission stuck in Pending.""" + # a pending standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.PENDING) + + # an accepted reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.ACCEPTED) + rpm = make_reparse_meta(True, True) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df1.pk + + +@pytest.mark.django_db +def test_find_pending_submissions__non_reparse_stuck__no_dfs(stt_user, stt): + """Finds standard upload/submission stuck in Pending.""" + # a standard submission with no summary + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + + # an accepted reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.ACCEPTED) + rpm = make_reparse_meta(True, True) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df1.pk + + +@pytest.mark.django_db +def test_find_pending_submissions__reparse_stuck(stt_user, stt): + """Finds a reparse submission stuck in pending, past the timeout.""" + # an accepted standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.ACCEPTED) + + # a pending reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.PENDING) + rpm = make_reparse_meta(False, False) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df2.pk + + +@pytest.mark.django_db +def test_find_pending_submissions__reparse_stuck__no_dfs(stt_user, stt): + """Finds a reparse submission stuck in pending, past the timeout.""" + # an accepted standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.ACCEPTED) + + # a reparse submission with no summary, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + rpm = make_reparse_meta(False, False) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df2.pk + + +@pytest.mark.django_db +def test_find_pending_submissions__reparse_and_non_reparse_stuck(stt_user, stt): + """Finds stuck submissions, both reparse and standard parse.""" + # a pending standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.PENDING) + + # a pending reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.PENDING) + rpm = make_reparse_meta(False, False) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 2 + for f in stuck_files: + assert f.pk in (df1.pk, df2.pk) + + +@pytest.mark.django_db +def test_find_pending_submissions__reparse_and_non_reparse_stuck_no_dfs(stt_user, stt): + """Finds stuck submissions, both reparse and standard parse.""" + # a pending standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + + # a pending reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + rpm = make_reparse_meta(False, False) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 2 + for f in stuck_files: + assert f.pk in (df1.pk, df2.pk) + + +@pytest.mark.django_db +def test_find_pending_submissions__old_reparse_stuck__new_not_stuck(stt_user, stt): + """Finds no stuck files, as the new parse is successful.""" + # a pending standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + dfs1 = make_summary(df1, DataFileSummary.Status.PENDING) + + # reparse fails the first time + rpm1 = make_reparse_meta(False, False) + df1.reparse_meta_models.add(rpm1) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + + # reparse again, succeeds this time + dfs1.delete() # reparse deletes the original dfs and creates the new one + make_summary(df1, DataFileSummary.Status.ACCEPTED) + + rpm2 = make_reparse_meta(True, True) + df1.reparse_meta_models.add(rpm2) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 0 + + +@pytest.mark.django_db +def test_find_pending_submissions__new_reparse_stuck__old_not_stuck(stt_user, stt): + """Finds files stuck from the new reparse, even though the old one was successful.""" + # file rejected on first upload + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + dfs1 = make_summary(df1, DataFileSummary.Status.REJECTED) + + # reparse succeeds + rpm1 = make_reparse_meta(True, True) + df1.reparse_meta_models.add(rpm1) + + # reparse again, fails this time + dfs1.delete() # reparse deletes the original dfs and creates the new one + DataFileSummary.objects.create( + datafile=df1, + status=DataFileSummary.Status.PENDING, + ) + + rpm2 = make_reparse_meta(False, False) + df1.reparse_meta_models.add(rpm2) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df1.pk diff --git a/tdrs-backend/tdpservice/data_files/util.py b/tdrs-backend/tdpservice/data_files/util.py index 17beb90aa..0d2d7a941 100644 --- a/tdrs-backend/tdpservice/data_files/util.py +++ b/tdrs-backend/tdpservice/data_files/util.py @@ -3,6 +3,7 @@ from io import BytesIO import xlsxwriter import calendar +from tdpservice.parsers.models import ParserErrorCategoryChoices def get_xls_serialized_file(data): @@ -48,6 +49,7 @@ def format_error_msg(x): ('item_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].values()])), ('internal_variable_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].keys()])), ('row_number', lambda x: x['row_number']), + ('error_type', lambda x: str(ParserErrorCategoryChoices(x['error_type']).label)), ] # write beta banner diff --git a/tdrs-backend/tdpservice/email/email_enums.py b/tdrs-backend/tdpservice/email/email_enums.py index 4527b6016..82e15e66d 100644 --- a/tdrs-backend/tdpservice/email/email_enums.py +++ b/tdrs-backend/tdpservice/email/email_enums.py @@ -15,3 +15,4 @@ class EmailType(Enum): ACCOUNT_DEACTIVATED = 'account-deactivated.html' ACCOUNT_DEACTIVATED_ADMIN = 'account-deactivated-admin.html' UPCOMING_SUBMISSION_DEADLINE = 'upcoming-submission-deadline.html' + STUCK_FILE_LIST = 'stuck-file-list.html' diff --git a/tdrs-backend/tdpservice/email/helpers/data_file.py b/tdrs-backend/tdpservice/email/helpers/data_file.py index 1ed966a87..3b9112b54 100644 --- a/tdrs-backend/tdpservice/email/helpers/data_file.py +++ b/tdrs-backend/tdpservice/email/helpers/data_file.py @@ -1,5 +1,6 @@ """Helper functions for sending data file submission emails.""" from django.conf import settings +from tdpservice.users.models import User from tdpservice.email.email_enums import EmailType from tdpservice.email.email import automated_email, log from tdpservice.parsers.util import get_prog_from_section @@ -69,3 +70,31 @@ def send_data_submitted_email( text_message=text_message, logger_context=logger_context ) + + +def send_stuck_file_email(stuck_files, recipients): + """Send an email to sys admins with details of files stuck in Pending.""" + logger_context = { + 'user_id': User.objects.get_or_create(username='system')[0].pk + } + + template_path = EmailType.STUCK_FILE_LIST.value + subject = 'List of submitted files with pending status after 1 hour' + text_message = 'The system has detected stuck files.' + + context = { + "subject": subject, + "url": settings.FRONTEND_BASE_URL, + "files": stuck_files, + } + + log(f'Emailing stuck files to SysAdmins: {list(recipients)}', logger_context=logger_context) + + automated_email( + email_path=template_path, + recipient_email=recipients, + subject=subject, + email_context=context, + text_message=text_message, + logger_context=logger_context + ) diff --git a/tdrs-backend/tdpservice/email/templates/stuck-file-list.html b/tdrs-backend/tdpservice/email/templates/stuck-file-list.html new file mode 100644 index 000000000..bfe5055a2 --- /dev/null +++ b/tdrs-backend/tdpservice/email/templates/stuck-file-list.html @@ -0,0 +1,40 @@ +{% extends 'base.html' %} +{% block content %} + + + + +

+

+ +

Hello,

+ +

The system has detected stuck data submissions.

+ + + + + + + + + + + + + {% for file in files %} + + + + + + + + {% endfor %} + +
SttSectionFiscal yearSubmitted onFile
{{ file.stt }}{{ file.section }}{{ file.fiscal_year }}{{ file.created_at }} {{ file.created_time_ago }} + View in Admin Console +
+ +{% endblock %} \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 1f14b6557..b2b9f0445 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -211,11 +211,10 @@ def rollback_records(unsaved_records, datafile): f"Encountered error while indexing datafile documents: \n{e}", "error" ) - logger.warn("Encountered an Elastic exception, enforcing DB cleanup.") + logger.warning("Encountered an Elastic exception, enforcing DB cleanup.") num_deleted, models = qset.delete() - logger.info("Succesfully performed DB cleanup after elastic failure.") log_parser_exception(datafile, - "Succesfully performed DB cleanup after elastic failure.", + "Succesfully performed DB cleanup after elastic failure in rollback_records.", "info" ) except DatabaseError as e: @@ -310,7 +309,7 @@ def delete_serialized_records(duplicate_manager, dfs): total_deleted += num_deleted dfs.total_number_of_records_created -= num_deleted log_parser_exception(dfs.datafile, - "Succesfully performed DB cleanup after elastic failure.", + "Succesfully performed DB cleanup after elastic failure in delete_serialized_records.", "info" ) except DatabaseError as e: diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 101f6c141..c0f50e85b 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -1,11 +1,29 @@ """Factories for generating test data for parsers.""" import factory +from django.utils import timezone from tdpservice.parsers.models import DataFileSummary, ParserErrorCategoryChoices from faker import Faker from tdpservice.data_files.test.factories import DataFileFactory from tdpservice.users.test.factories import UserFactory from tdpservice.stts.test.factories import STTFactory + +class ReparseMetaFactory(factory.django.DjangoModelFactory): + """Generate test reparse meta model.""" + + class Meta: + """Hardcoded meta data for factory.""" + + model = "search_indexes.ReparseMeta" + + timeout_at = timezone.now() + finished = False + success = False + num_files_to_reparse = 1 + files_completed = 1 + files_failed = 0 + + class ParsingFileFactory(factory.django.DjangoModelFactory): """Generate test data for data files.""" @@ -184,43 +202,43 @@ class Meta: EMPLOYMENT_STATUS = 1 WORK_ELIGIBLE_INDICATOR = "01" WORK_PART_STATUS = "01" - UNSUB_EMPLOYMENT = 1 - SUB_PRIVATE_EMPLOYMENT = 1 - SUB_PUBLIC_EMPLOYMENT = 1 - WORK_EXPERIENCE_HOP = 1 - WORK_EXPERIENCE_EA = 1 - WORK_EXPERIENCE_HOL = 1 - OJT = 1 - JOB_SEARCH_HOP = 1 - JOB_SEARCH_EA = 1 - JOB_SEARCH_HOL = 1 - COMM_SERVICES_HOP = 1 - COMM_SERVICES_EA = 1 - COMM_SERVICES_HOL = 1 - VOCATIONAL_ED_TRAINING_HOP = 1 - VOCATIONAL_ED_TRAINING_EA = 1 - VOCATIONAL_ED_TRAINING_HOL = 1 - JOB_SKILLS_TRAINING_HOP = 1 - JOB_SKILLS_TRAINING_EA = 1 - JOB_SKILLS_TRAINING_HOL = 1 - ED_NO_HIGH_SCHOOL_DIPL_HOP = 1 - ED_NO_HIGH_SCHOOL_DIPL_EA = 1 - ED_NO_HIGH_SCHOOL_DIPL_HOL = 1 - SCHOOL_ATTENDENCE_HOP = 1 - SCHOOL_ATTENDENCE_EA = 1 - SCHOOL_ATTENDENCE_HOL = 1 - PROVIDE_CC_HOP = 1 - PROVIDE_CC_EA = 1 - PROVIDE_CC_HOL = 1 - OTHER_WORK_ACTIVITIES = 1 - DEEMED_HOURS_FOR_OVERALL = 1 - DEEMED_HOURS_FOR_TWO_PARENT = 1 - EARNED_INCOME = 1 - UNEARNED_INCOME_TAX_CREDIT = 1 - UNEARNED_SOCIAL_SECURITY = 1 - UNEARNED_SSI = 1 - UNEARNED_WORKERS_COMP = 1 - OTHER_UNEARNED_INCOME = 1 + UNSUB_EMPLOYMENT = "01" + SUB_PRIVATE_EMPLOYMENT = "01" + SUB_PUBLIC_EMPLOYMENT = "01" + WORK_EXPERIENCE_HOP = "01" + WORK_EXPERIENCE_EA = "01" + WORK_EXPERIENCE_HOL = "01" + OJT = "01" + JOB_SEARCH_HOP = "01" + JOB_SEARCH_EA = "01" + JOB_SEARCH_HOL = "01" + COMM_SERVICES_HOP = "01" + COMM_SERVICES_EA = "01" + COMM_SERVICES_HOL = "01" + VOCATIONAL_ED_TRAINING_HOP = "01" + VOCATIONAL_ED_TRAINING_EA = "01" + VOCATIONAL_ED_TRAINING_HOL = "01" + JOB_SKILLS_TRAINING_HOP = "01" + JOB_SKILLS_TRAINING_EA = "01" + JOB_SKILLS_TRAINING_HOL = "01" + ED_NO_HIGH_SCHOOL_DIPL_HOP = "01" + ED_NO_HIGH_SCHOOL_DIPL_EA = "01" + ED_NO_HIGH_SCHOOL_DIPL_HOL = "01" + SCHOOL_ATTENDENCE_HOP = "01" + SCHOOL_ATTENDENCE_EA = "01" + SCHOOL_ATTENDENCE_HOL = "01" + PROVIDE_CC_HOP = "01" + PROVIDE_CC_EA = "01" + PROVIDE_CC_HOL = "01" + OTHER_WORK_ACTIVITIES = "01" + DEEMED_HOURS_FOR_OVERALL = "01" + DEEMED_HOURS_FOR_TWO_PARENT = "01" + EARNED_INCOME = "01" + UNEARNED_INCOME_TAX_CREDIT = "01" + UNEARNED_SOCIAL_SECURITY = "01" + UNEARNED_SSI = "01" + UNEARNED_WORKERS_COMP = "01" + OTHER_UNEARNED_INCOME = "01" class TanfT3Factory(factory.django.DjangoModelFactory): @@ -451,10 +469,10 @@ class Meta: CURRENT_MONTH_STATE_EXEMPT = 1 EMPLOYMENT_STATUS = 1 WORK_PART_STATUS = "01" - UNSUB_EMPLOYMENT = 1 - SUB_PRIVATE_EMPLOYMENT = 1 - SUB_PUBLIC_EMPLOYMENT = 1 - OJT = 1 + UNSUB_EMPLOYMENT = "01" + SUB_PRIVATE_EMPLOYMENT = "01" + SUB_PUBLIC_EMPLOYMENT = "01" + OJT = "01" JOB_SEARCH = '1' COMM_SERVICES = '1' VOCATIONAL_ED_TRAINING = '1' diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index def4240db..d01a44030 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2,6 +2,7 @@ import pytest +import os from django.contrib.admin.models import LogEntry from django.conf import settings from django.db.models import Q as Query @@ -1299,7 +1300,7 @@ def test_parse_tribal_section_2_file(tribal_section_2_file, dfs): t4 = t4_objs.first() t5 = t5_objs.last() - assert t4.CLOSURE_REASON == 8 + assert t4.CLOSURE_REASON == '15' assert t5.COUNTABLE_MONTH_FED_TIME == ' 8' @@ -1739,6 +1740,9 @@ def test_parse_duplicate(file, batch_size, model, record_type, num_errors, dfs, settings.BULK_CREATE_BATCH_SIZE = batch_size parse.parse_datafile(datafile, dfs) + + settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + parser_errors = ParserError.objects.filter(file=datafile, error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by('id') for e in parser_errors: @@ -1782,6 +1786,9 @@ def test_parse_partial_duplicate(file, batch_size, model, record_type, num_error settings.BULK_CREATE_BATCH_SIZE = batch_size parse.parse_datafile(datafile, dfs) + + settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + parser_errors = ParserError.objects.filter(file=datafile, error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by('id') for e in parser_errors: @@ -1806,6 +1813,8 @@ def test_parse_cat_4_edge_case_file(cat4_edge_case_file, dfs): parse.parse_datafile(cat4_edge_case_file, dfs) + settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + parser_errors = ParserError.objects.filter(file=cat4_edge_case_file).filter( error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index 6a3717817..893d10f97 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -12,7 +12,7 @@ def transform(value, **kwargs): month = transform_to_months(quarter)[month_index] except ValueError: return None - return f"{year}{month_to_int(month)}" + return int(f"{year}{month_to_int(month)}") return transform def tanf_ssn_decryption_func(value, **kwargs): diff --git a/tdrs-backend/tdpservice/parsers/validators/category3.py b/tdrs-backend/tdpservice/parsers/validators/category3.py index cb278e5e2..89f9547c8 100644 --- a/tdrs-backend/tdpservice/parsers/validators/category3.py +++ b/tdrs-backend/tdpservice/parsers/validators/category3.py @@ -386,7 +386,7 @@ def validate(record, row_schema): "Caught exception in validator: validate__WORK_ELIGIBLE_INDICATOR__HOH__AGE. " + f"With field values: {vals}." ) - logger.error(f'Exception: {e}') + logger.debug(f'Exception: {e}') # Per conversation with Alex on 03/26/2024, returning the true case during exception handling to avoid # confusing the STTs. return true_case diff --git a/tdrs-backend/tdpservice/scheduling/management/db_backup.py b/tdrs-backend/tdpservice/scheduling/management/db_backup.py index 11beceaed..0929a98e4 100644 --- a/tdrs-backend/tdpservice/scheduling/management/db_backup.py +++ b/tdrs-backend/tdpservice/scheduling/management/db_backup.py @@ -20,7 +20,10 @@ OS_ENV = os.environ -content_type = ContentType.objects.get_for_model(LogEntry) + +def get_content_type(): + """Get content type for log entry.""" + return ContentType.objects.get_for_model(LogEntry) def get_system_values(): """Return dict of keys and settings to use whether local or deployed.""" @@ -91,7 +94,7 @@ def backup_database(file_name, logger.info(msg) LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Executed Database Backup", action_flag=ADDITION, @@ -123,7 +126,7 @@ def restore_database(file_name, postgres_client, database_uri, system_user): msg = "Completed database creation." LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Executed Database create", action_flag=ADDITION, @@ -145,7 +148,7 @@ def restore_database(file_name, postgres_client, database_uri, system_user): msg = "Completed database restoration." LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Executed Database restore", action_flag=ADDITION, @@ -177,7 +180,7 @@ def upload_file(file_name, bucket, sys_values, system_user, object_name=None, re msg = "Successfully uploaded {} to s3://{}/{}.".format(file_name, bucket, object_name) LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Executed database backup S3 upload", action_flag=ADDITION, @@ -208,7 +211,7 @@ def download_file(bucket, msg = "Successfully downloaded s3 file {}/{} to {}.".format(bucket, object_name, file_name) LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Executed database backup S3 download", action_flag=ADDITION, @@ -293,7 +296,7 @@ def main(argv, sys_values, system_user): if arg_to_backup: LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Begining Database Backup", action_flag=ADDITION, @@ -316,7 +319,7 @@ def main(argv, sys_values, system_user): LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Finished Database Backup", action_flag=ADDITION, @@ -329,7 +332,7 @@ def main(argv, sys_values, system_user): elif arg_to_restore: LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Begining Database Restore", action_flag=ADDITION, @@ -352,7 +355,7 @@ def main(argv, sys_values, system_user): LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Finished Database Restore", action_flag=ADDITION, @@ -377,7 +380,7 @@ def run_backup(arg): logger.error(f"Caught Exception in run_backup. Exception: {e}.") LogEntry.objects.log_action( user_id=system_user.pk, - content_type_id=content_type.pk, + content_type_id=get_content_type().pk, object_id=None, object_repr="Exception in run_backup", action_flag=ADDITION, diff --git a/tdrs-backend/tdpservice/scheduling/test/test_db_backup.py b/tdrs-backend/tdpservice/scheduling/test/test_db_backup.py new file mode 100644 index 000000000..bdcdc727e --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/test/test_db_backup.py @@ -0,0 +1,69 @@ +"""Test cases for db_backup.py functions.""" + +import os +import pytest +from tdpservice.scheduling.management import db_backup +from django.contrib.admin.models import LogEntry + +@pytest.mark.django_db +def test_backup_database(system_user): + """Test backup functionality.""" + file_name = "/tmp/test_backup.pg" + ret = db_backup.backup_database(file_name, "", + "postgres://tdpuser:something_secure@postgres:5432/tdrs_test", + system_user) + + assert ret is True + assert os.path.getsize(file_name) > 0 + os.remove(file_name) + assert os.path.exists(file_name) is False + +@pytest.mark.django_db +def test_backup_database_fail_on_backup(system_user): + """Test backup fails on psql non-zero return code.""" + with pytest.raises(Exception) as e: + file_name = "/tmp/test_backup.pg" + db_backup.backup_database(file_name, "asdfasdfassfd", + "postgres://tdpuser:something_secure@postgres:5432/tdrs_test", + system_user) + + assert str(e.value) == "pg_dump command failed with a non zero exit code." + assert os.path.exists(file_name) is False + +@pytest.mark.django_db +def test_backup_database_fail_on_general_exception(): + """Test backup succeeds but raises exception on string user for log entry.""" + with pytest.raises(Exception) as e: + file_name = "/tmp/test_backup.pg" + db_backup.backup_database(file_name, "", + "postgres://tdpuser:something_secure@postgres:5432/tdrs_test", + "system_user") + + assert str(e.value) == "'str' object has no attribute 'pk'" + assert os.path.exists(file_name) is True + os.remove(file_name) + assert os.path.exists(file_name) is False + + +@pytest.mark.django_db +def test_get_database_credentials(): + """Test get credentials.""" + creds = db_backup.get_database_credentials("postgres://tdpuser:something_secure@postgres:5432/tdrs_test") + assert creds == ["tdpuser", "something_secure", "postgres", "5432", "tdrs_test"] + +@pytest.mark.django_db +def test_main_backup(mocker, system_user): + """Test call the main function.""" + mocker.patch( + 'tdpservice.scheduling.management.db_backup.upload_file', + return_value=True + ) + sys_vals = {"DATABASE_URI": "postgres://tdpuser:something_secure@postgres:5432", + "DATABASE_DB_NAME": "tdrs_test", + "POSTGRES_CLIENT_DIR": "", + "S3_BUCKET": "", + "S3_REGION": ""} + + db_backup.main(['-b', '-f', '/tmp/test_backup.pg'], sys_values=sys_vals, system_user=system_user) + assert LogEntry.objects.get(change_message="Begining database backup.").pk is not None + assert LogEntry.objects.get(change_message="Finished database backup.").pk is not None diff --git a/tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson b/tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson new file mode 100644 index 000000000..11a4be181 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson @@ -0,0 +1,22 @@ +{"attributes":{"fieldFormatMap":"{\"RPT_MONTH_YEAR_DATE\":{\"id\":\"date\",\"params\":{\"parsedUrl\":{\"origin\":\"http://localhost:3000\",\"pathname\":\"/kibana/app/home\",\"basePath\":\"/kibana\"},\"pattern\":\"YYYYMM\"}}}","fields":"[{\"count\":0,\"name\":\"AID_AGED_BLIND\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"AMOUNT_EARNED_INCOME\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"AMOUNT_EARNED_INCOME.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"AMOUNT_EARNED_INCOME\"}}},{\"count\":0,\"name\":\"AMOUNT_UNEARNED_INCOME\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"AMOUNT_UNEARNED_INCOME.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"AMOUNT_UNEARNED_INCOME\"}}},{\"count\":0,\"name\":\"AMT_FOOD_STAMP_ASSISTANCE\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"AMT_SUB_CC\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"ASSISTANCE\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CALENDAR_QUARTER\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CASE_NUMBER\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"CASE_NUMBER.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"CASE_NUMBER\"}}},{\"count\":0,\"name\":\"CASH_AMOUNT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CC_AMOUNT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CC_NBR_MONTHS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CHILDREN_COVERED\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CHILD_SUPPORT_AMT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CITIZENSHIP_STATUS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CLOSURE_REASON\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"CLOSURE_REASON.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"CLOSURE_REASON\"}}},{\"count\":0,\"name\":\"COMM_SERVICES_EA\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"COMM_SERVICES_EA.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"COMM_SERVICES_EA\"}}},{\"count\":0,\"name\":\"COMM_SERVICES_HOL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"COMM_SERVICES_HOL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"COMM_SERVICES_HOL\"}}},{\"count\":0,\"name\":\"COMM_SERVICES_HOP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"COMM_SERVICES_HOP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"COMM_SERVICES_HOP\"}}},{\"count\":0,\"name\":\"COOPERATION_CHILD_SUPPORT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"COUNTABLE_MONTHS_STATE_TRIBE\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"COUNTABLE_MONTHS_STATE_TRIBE.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"COUNTABLE_MONTHS_STATE_TRIBE\"}}},{\"count\":0,\"name\":\"COUNTABLE_MONTH_FED_TIME\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"COUNTABLE_MONTH_FED_TIME.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"COUNTABLE_MONTH_FED_TIME\"}}},{\"count\":0,\"name\":\"COUNTY_FIPS_CODE\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"COUNTY_FIPS_CODE.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"COUNTY_FIPS_CODE\"}}},{\"count\":0,\"name\":\"CURRENT_MONTH_STATE_EXEMPT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"DATE_OF_BIRTH\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"DATE_OF_BIRTH.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"DATE_OF_BIRTH\"}}},{\"count\":0,\"name\":\"DEEMED_HOURS_FOR_OVERALL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"DEEMED_HOURS_FOR_OVERALL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"DEEMED_HOURS_FOR_OVERALL\"}}},{\"count\":0,\"name\":\"DEEMED_HOURS_FOR_TWO_PARENT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"DEEMED_HOURS_FOR_TWO_PARENT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"DEEMED_HOURS_FOR_TWO_PARENT\"}}},{\"count\":0,\"name\":\"DISABLED_TITLE_XIVAPDT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"DISABLED_TITLE_XIVAPDT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"DISABLED_TITLE_XIVAPDT\"}}},{\"count\":0,\"name\":\"DISPOSITION\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"EARNED_INCOME\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"EARNED_INCOME.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"EARNED_INCOME\"}}},{\"count\":0,\"name\":\"EDUCATION_LEVEL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"EDUCATION_LEVEL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"EDUCATION_LEVEL\"}}},{\"count\":0,\"name\":\"ED_NO_HIGH_SCHOOL_DIPL_EA\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"ED_NO_HIGH_SCHOOL_DIPL_EA.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"ED_NO_HIGH_SCHOOL_DIPL_EA\"}}},{\"count\":0,\"name\":\"ED_NO_HIGH_SCHOOL_DIPL_HOL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"ED_NO_HIGH_SCHOOL_DIPL_HOL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"ED_NO_HIGH_SCHOOL_DIPL_HOL\"}}},{\"count\":0,\"name\":\"ED_NO_HIGH_SCHOOL_DIPL_HOP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"ED_NO_HIGH_SCHOOL_DIPL_HOP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"ED_NO_HIGH_SCHOOL_DIPL_HOP\"}}},{\"count\":0,\"name\":\"EMPLOYMENT_STATUS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FAILURE_TO_COMPLY\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FAMILY_AFFILIATION\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FAMILY_CAP\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FAMILY_CASH_RESOURCES\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FAMILY_EXEMPT_TIME_LIMITS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FAMILY_NEW_CHILD\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FAMILY_SANC_ADULT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FAMILY_TYPE\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FED_DISABILITY_STATUS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FED_OASDI_PROGRAM\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"FUNDING_STREAM\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"GENDER\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"JOB_SEARCH_EA\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"JOB_SEARCH_EA.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"JOB_SEARCH_EA\"}}},{\"count\":0,\"name\":\"JOB_SEARCH_HOL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"JOB_SEARCH_HOL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"JOB_SEARCH_HOL\"}}},{\"count\":0,\"name\":\"JOB_SEARCH_HOP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"JOB_SEARCH_HOP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"JOB_SEARCH_HOP\"}}},{\"count\":0,\"name\":\"JOB_SKILLS_TRAINING_EA\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"JOB_SKILLS_TRAINING_EA.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"JOB_SKILLS_TRAINING_EA\"}}},{\"count\":0,\"name\":\"JOB_SKILLS_TRAINING_HOL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"JOB_SKILLS_TRAINING_HOL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"JOB_SKILLS_TRAINING_HOL\"}}},{\"count\":0,\"name\":\"JOB_SKILLS_TRAINING_HOP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"JOB_SKILLS_TRAINING_HOP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"JOB_SKILLS_TRAINING_HOP\"}}},{\"count\":0,\"name\":\"MARITAL_STATUS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"MONTHS_FED_TIME_LIMIT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"MONTHS_FED_TIME_LIMIT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"MONTHS_FED_TIME_LIMIT\"}}},{\"count\":0,\"name\":\"MONTHS_STATE_TIME_LIMIT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"MONTHS_STATE_TIME_LIMIT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"MONTHS_STATE_TIME_LIMIT\"}}},{\"count\":0,\"name\":\"NBR_FAMILY_MEMBERS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NBR_MONTHS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NEEDS_OF_PREGNANT_WOMAN\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NEEDS_PREGNANT_WOMAN\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NEW_APPLICANT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NONCUSTODIAL_PARENT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NON_COOPERATION_CSE\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_1_PARENTS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_2_PARENTS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_ADULT_RECIPIENTS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_APPLICATIONS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_APPROVED\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_BIRTHS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_CHILD_RECIPIENTS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_CLOSED_CASES\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_DENIED\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_FAMILIES\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_NONCUSTODIALS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_NO_PARENTS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_OUTWEDLOCK_BIRTHS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NUM_RECIPIENTS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"OJT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"OJT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"OJT\"}}},{\"count\":0,\"name\":\"OTHER_AMOUNT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"OTHER_NBR_MONTHS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"OTHER_NON_SANCTION\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"OTHER_SANCTION\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"OTHER_TOTAL_REDUCTIONS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"OTHER_UNEARNED_INCOME\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"OTHER_UNEARNED_INCOME.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"OTHER_UNEARNED_INCOME\"}}},{\"count\":0,\"name\":\"OTHER_WORK_ACTIVITIES\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"OTHER_WORK_ACTIVITIES.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"OTHER_WORK_ACTIVITIES\"}}},{\"count\":0,\"name\":\"PARENT_MINOR_CHILD\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"PROVIDE_CC_EA\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"PROVIDE_CC_EA.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"PROVIDE_CC_EA\"}}},{\"count\":0,\"name\":\"PROVIDE_CC_HOL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"PROVIDE_CC_HOL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"PROVIDE_CC_HOL\"}}},{\"count\":0,\"name\":\"PROVIDE_CC_HOP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"PROVIDE_CC_HOP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"PROVIDE_CC_HOP\"}}},{\"count\":0,\"name\":\"RACE_AMER_INDIAN\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RACE_ASIAN\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RACE_BLACK\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RACE_HAWAIIAN\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RACE_HISPANIC\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RACE_WHITE\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RECEIVES_FOOD_STAMPS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RECEIVES_MED_ASSISTANCE\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RECEIVES_SUB_CC\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RECEIVES_SUB_HOUSING\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RECEIVE_NONSSA_BENEFITS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RECEIVE_SSI\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RECOUPMENT_PRIOR_OVRPMT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_AID_AGED_BLIND\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_AID_TOTALLY_DISABLED\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_FEDERAL_DISABILITY\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_FOOD_STAMPS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_MED_ASSIST\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_OASDI_INSURANCE\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_SSI\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_SUB_CC\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REC_SUB_HOUSING\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"REDUCTIONS_ON_RECEIPTS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RELATIONSHIP_HOH\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"RELATIONSHIP_HOH.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"RELATIONSHIP_HOH\"}}},{\"count\":0,\"name\":\"RPT_MONTH_YEAR\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RecordType\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"RecordType.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"RecordType\"}}},{\"count\":0,\"name\":\"SANC_REDUCTION_AMT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"SANC_TEEN_PARENT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"SCHOOL_ATTENDENCE_EA\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"SCHOOL_ATTENDENCE_EA.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"SCHOOL_ATTENDENCE_EA\"}}},{\"count\":0,\"name\":\"SCHOOL_ATTENDENCE_HOL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"SCHOOL_ATTENDENCE_HOL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"SCHOOL_ATTENDENCE_HOL\"}}},{\"count\":0,\"name\":\"SCHOOL_ATTENDENCE_HOP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"SCHOOL_ATTENDENCE_HOP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"SCHOOL_ATTENDENCE_HOP\"}}},{\"count\":0,\"name\":\"SSN\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"SSN.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"SSN\"}}},{\"count\":0,\"name\":\"STRATUM\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"STRATUM.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"STRATUM\"}}},{\"count\":0,\"name\":\"SUB_PRIVATE_EMPLOYMENT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"SUB_PRIVATE_EMPLOYMENT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"SUB_PRIVATE_EMPLOYMENT\"}}},{\"count\":0,\"name\":\"SUB_PUBLIC_EMPLOYMENT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"SUB_PUBLIC_EMPLOYMENT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"SUB_PUBLIC_EMPLOYMENT\"}}},{\"count\":0,\"name\":\"TRANSITION_NBR_MONTHS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"TRANSITION_SERVICES_AMOUNT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"TRANSP_AMOUNT\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"TRANSP_NBR_MONTHS\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"UNEARNED_INCOME_TAX_CREDIT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"UNEARNED_INCOME_TAX_CREDIT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"UNEARNED_INCOME_TAX_CREDIT\"}}},{\"count\":0,\"name\":\"UNEARNED_SOCIAL_SECURITY\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"UNEARNED_SOCIAL_SECURITY.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"UNEARNED_SOCIAL_SECURITY\"}}},{\"count\":0,\"name\":\"UNEARNED_SSI\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"UNEARNED_SSI.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"UNEARNED_SSI\"}}},{\"count\":0,\"name\":\"UNEARNED_WORKERS_COMP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"UNEARNED_WORKERS_COMP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"UNEARNED_WORKERS_COMP\"}}},{\"count\":0,\"name\":\"UNSUB_EMPLOYMENT\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"UNSUB_EMPLOYMENT.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"UNSUB_EMPLOYMENT\"}}},{\"count\":0,\"name\":\"VOCATIONAL_ED_TRAINING_EA\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"VOCATIONAL_ED_TRAINING_EA.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"VOCATIONAL_ED_TRAINING_EA\"}}},{\"count\":0,\"name\":\"VOCATIONAL_ED_TRAINING_HOL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"VOCATIONAL_ED_TRAINING_HOL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"VOCATIONAL_ED_TRAINING_HOL\"}}},{\"count\":0,\"name\":\"VOCATIONAL_ED_TRAINING_HOP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"VOCATIONAL_ED_TRAINING_HOP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"VOCATIONAL_ED_TRAINING_HOP\"}}},{\"count\":0,\"name\":\"WAIVER_EVAL_CONTROL_GRPS\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"WAIVER_EVAL_CONTROL_GRPS.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"WAIVER_EVAL_CONTROL_GRPS\"}}},{\"count\":0,\"name\":\"WORK_ELIGIBLE_INDICATOR\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"WORK_ELIGIBLE_INDICATOR.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"WORK_ELIGIBLE_INDICATOR\"}}},{\"count\":0,\"name\":\"WORK_EXPERIENCE_EA\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"WORK_EXPERIENCE_EA.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"WORK_EXPERIENCE_EA\"}}},{\"count\":0,\"name\":\"WORK_EXPERIENCE_HOL\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"WORK_EXPERIENCE_HOL.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"WORK_EXPERIENCE_HOL\"}}},{\"count\":0,\"name\":\"WORK_EXPERIENCE_HOP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"WORK_EXPERIENCE_HOP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"WORK_EXPERIENCE_HOP\"}}},{\"count\":0,\"name\":\"WORK_PART_STATUS\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"WORK_PART_STATUS.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"WORK_PART_STATUS\"}}},{\"count\":0,\"name\":\"WORK_REQ_SANCTION\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"ZIP_CODE\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"ZIP_CODE.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"ZIP_CODE\"}}},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"datafile.created_at\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"datafile.id\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"datafile.quarter\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"datafile.quarter.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"datafile.quarter\"}}},{\"count\":0,\"name\":\"datafile.stt.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"datafile.stt.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"datafile.stt.name\"}}},{\"count\":0,\"name\":\"datafile.stt.stt_code\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"datafile.stt.stt_code.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"datafile.stt.stt_code\"}}},{\"count\":0,\"name\":\"datafile.stt.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"datafile.stt.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"datafile.stt.type\"}}},{\"count\":0,\"name\":\"datafile.version\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"datafile.year\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"script\":\"SimpleDateFormat fmt = new SimpleDateFormat(\\\"yyyyMM\\\");\\nDate rpt_month_year = fmt.parse(doc['RPT_MONTH_YEAR'].value.toString());\\nCalendar cal = Calendar.getInstance();\\ncal.setTime(rpt_month_year);\\n// Have to add 1 to the month because the Java Date class stores months on the\\n// range [0, 11].\\nint month = cal.get(Calendar.MONTH) + 1;\\ncal.set(Calendar.MONTH, month);\\n\\n// Even though we are creating a Date object in Kibana, Elastic will only\\n// this scripted field correctly in aggregations if it returns a long. If you\\n// return anything but that, Elastic throws an exception. See the link to get\\n// the full description: https://github.com/elastic/kibana/issues/69121#:~:text=Jul%206%2C%202020-,Ahh,-...%20I%20see%20what%27s \\nreturn cal.getTimeInMillis();\\n\",\"lang\":\"painless\",\"name\":\"RPT_MONTH_YEAR_DATE\",\"type\":\"date\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]","timeFieldName":"datafile.created_at","title":"*tanf_t*"},"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2024-06-03T15:26:17.320Z","version":"WzYzNCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"U.S. Total Number of Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"U.S. Total Number of Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"U.S. Total Number of Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"M\",\"drop_partials\":true,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year and Month\"},\"schema\":\"segment\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":7,\"rotate\":90},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":true},\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"U.S. Total Number of Families\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"wiggle\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"U.S. Total Number of Families\"},\"type\":\"value\"}]}}"},"id":"04913050-187c-11ef-a24d-3b1903bfefcc","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:37:01.834Z","version":"WzQwOCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"U.S. Average Number of Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"U.S. Average Number of Families\",\"type\":\"metric\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR\",\"interval\":\"auto\",\"min_doc_count\":false,\"has_extended_bounds\":false,\"extended_bounds\":{\"min\":\"\",\"max\":\"\"}}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"params\":{}}},\"schema\":\"metric\"}],\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":false},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}}}"},"id":"32a198f0-187b-11ef-a24d-3b1903bfefcc","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T13:12:21.238Z","version":"WzcsMV0="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"STT Total Number of Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"STT Total Number of Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Total Number of Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15w\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"M\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year and Month\"},\"schema\":\"group\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"datafile.stt.name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":50,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"STT\"},\"schema\":\"segment\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"rotate\":90,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":true},\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Total Number of Families\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"wiggle\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Total Number of Families\"},\"type\":\"value\"}]}}"},"id":"5b45add0-1878-11ef-a24d-3b1903bfefcc","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:18:00.823Z","version":"WzI1NiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"STT Average Number of Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"STT Average Number of Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"params\":{\"filters\":[{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202010 AND RPT_MONTH_YEAR <= 202010\",\"language\":\"kuery\"},\"label\":\"\"},{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202011 AND RPT_MONTH_YEAR <= 202011\",\"language\":\"kuery\"},\"label\":\"\"},{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202012 AND RPT_MONTH_YEAR <= 202012\",\"language\":\"kuery\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"params\":{}},\"customLabel\":\"Average Number of Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15y\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"y\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year\"},\"schema\":\"group\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"datafile.stt.name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"STT\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"wiggle\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Average Number of Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Average Number of Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"a5793bf0-1879-11ef-a24d-3b1903bfefcc","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:26:50.792Z","version":"WzMyNywxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"2791a155-ed49-4aeb-b0c7-0c7d82d00fd6\"},\"panelIndex\":\"2791a155-ed49-4aeb-b0c7-0c7d82d00fd6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"24e50275-1a1d-4027-8edb-2e48bf1294e1\"},\"panelIndex\":\"24e50275-1a1d-4027-8edb-2e48bf1294e1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":15,\"w\":24,\"h\":15,\"i\":\"8077bd4c-7dac-4721-937c-caa759832e3a\"},\"panelIndex\":\"8077bd4c-7dac-4721-937c-caa759832e3a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":15,\"w\":24,\"h\":15,\"i\":\"d03b3362-db37-4070-8c76-b9ea526163d6\"},\"panelIndex\":\"d03b3362-db37-4070-8c76-b9ea526163d6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-15y","timeRestore":true,"timeTo":"now","title":"TANF Total Number of Families","version":1},"id":"2c614c00-187c-11ef-a24d-3b1903bfefcc","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"04913050-187c-11ef-a24d-3b1903bfefcc","name":"panel_0","type":"visualization"},{"id":"32a198f0-187b-11ef-a24d-3b1903bfefcc","name":"panel_1","type":"visualization"},{"id":"5b45add0-1878-11ef-a24d-3b1903bfefcc","name":"panel_2","type":"visualization"},{"id":"a5793bf0-1879-11ef-a24d-3b1903bfefcc","name":"panel_3","type":"visualization"}],"type":"dashboard","updated_at":"2024-06-03T13:12:21.238Z","version":"WzEwLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"U.S. Total Number of No Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"U.S. Total Number of No Parent Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_NO_PARENTS\",\"customLabel\":\"U.S. Total Number of No Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"M\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year and Month\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":7,\"rotate\":90},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"wiggle\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"U.S. Total Number of No Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"U.S. Total Number of No Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"1898cd10-192b-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:36:52.818Z","version":"WzQwNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"U.S. Average Number of No Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"U.S. Average Number of No Parent Families\",\"type\":\"metric\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR\",\"interval\":\"auto\",\"min_doc_count\":false,\"has_extended_bounds\":false,\"extended_bounds\":{\"min\":\"\",\"max\":\"\"}}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_NO_PARENTS\",\"customLabel\":\"\"}},\"customLabel\":\"Average Number of No Parent Families\"},\"schema\":\"metric\"}],\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":false},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}}}"},"id":"f523b1a0-192b-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T13:12:21.238Z","version":"WzEyLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"STT Total Number of No Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"STT Total Number of No Parent Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_NO_PARENTS\",\"customLabel\":\"Total Number of No Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15w\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"M\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year and Month\"},\"schema\":\"group\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"datafile.stt.name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"STT\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"wiggle\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Total Number of No Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Total Number of No Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"146ea0d0-192a-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:18:37.697Z","version":"WzI2MiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"STT Average Number of No Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"STT Average Number of No Parent Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"params\":{\"filters\":[{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202010 AND RPT_MONTH_YEAR <= 202010\",\"language\":\"kuery\"},\"label\":\"\"},{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202011 AND RPT_MONTH_YEAR <= 202011\",\"language\":\"kuery\"},\"label\":\"\"},{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202012 AND RPT_MONTH_YEAR <= 202012\",\"language\":\"kuery\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_NO_PARENTS\"}},\"customLabel\":\"Average Number of No Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15y\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"y\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year\"},\"schema\":\"group\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"datafile.stt.name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":498,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"STT\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Average Number of No Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Average Number of No Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"3ab46ba0-192d-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:28:25.015Z","version":"WzMzMSwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"9d5417cd-7646-49bb-9cb6-d3125cab4444\"},\"panelIndex\":\"9d5417cd-7646-49bb-9cb6-d3125cab4444\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"470d4dbc-1657-4e3f-a190-298d95c10058\"},\"panelIndex\":\"470d4dbc-1657-4e3f-a190-298d95c10058\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":15,\"w\":24,\"h\":15,\"i\":\"68d18b18-8db7-4de7-96d3-629b981ec733\"},\"panelIndex\":\"68d18b18-8db7-4de7-96d3-629b981ec733\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":15,\"w\":24,\"h\":15,\"i\":\"969fe8a0-d019-4980-adef-f14cd3068b6b\"},\"panelIndex\":\"969fe8a0-d019-4980-adef-f14cd3068b6b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-15M","timeRestore":true,"timeTo":"now","title":"TANF Total Number of No Parent Families","version":1},"id":"b7898340-192d-11ef-a9c8-cb02c1885723","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"1898cd10-192b-11ef-a9c8-cb02c1885723","name":"panel_0","type":"visualization"},{"id":"f523b1a0-192b-11ef-a9c8-cb02c1885723","name":"panel_1","type":"visualization"},{"id":"146ea0d0-192a-11ef-a9c8-cb02c1885723","name":"panel_2","type":"visualization"},{"id":"3ab46ba0-192d-11ef-a9c8-cb02c1885723","name":"panel_3","type":"visualization"}],"type":"dashboard","updated_at":"2024-06-03T13:12:21.238Z","version":"WzE1LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"U.S. Total Number of One Parent Familes","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"U.S. Total Number of One Parent Familes\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_1_PARENTS\",\"customLabel\":\"U.S. Total Number of One Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"M\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year and Month\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":7,\"rotate\":90},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"wiggle\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"U.S. Total Number of One Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"U.S. Total Number of One Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"2b5b0c10-192b-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:37:29.700Z","version":"WzQxMiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"U.S. Average Number of One Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"U.S. Average Number of One Parent Families\",\"type\":\"metric\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR\",\"interval\":\"auto\",\"min_doc_count\":false,\"has_extended_bounds\":false,\"extended_bounds\":{\"min\":\"\",\"max\":\"\"}}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_1_PARENTS\",\"customLabel\":\"\"}},\"customLabel\":\"Average Number of One Parent Families\"},\"schema\":\"metric\"}],\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":false},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}}}"},"id":"158296f0-192c-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T13:12:21.238Z","version":"WzE3LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"STT Total Number of One Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"STT Total Number of One Parent Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_1_PARENTS\",\"customLabel\":\"Total Number of One Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15w\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"M\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year and Month\"},\"schema\":\"group\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"datafile.stt.name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"STT\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"wiggle\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Total Number of One Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Total Number of One Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"a2e22600-192c-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:21:36.873Z","version":"WzI5MSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"STT Average Number of One Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"STT Average Number of One Parent Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"params\":{\"filters\":[{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202010 AND RPT_MONTH_YEAR <= 202010\",\"language\":\"kuery\"},\"label\":\"\"},{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202011 AND RPT_MONTH_YEAR <= 202011\",\"language\":\"kuery\"},\"label\":\"\"},{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202012 AND RPT_MONTH_YEAR <= 202012\",\"language\":\"kuery\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_1_PARENTS\"}},\"customLabel\":\"Average Number of No Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15y\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"y\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year\"},\"schema\":\"group\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"datafile.stt.name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":498,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"STT\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Average Number of No Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Average Number of No Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"0bd80070-192e-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:28:43.593Z","version":"WzMzNSwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"c56f15ef-94f9-4b9f-8996-c1aadc24b5dd\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"c56f15ef-94f9-4b9f-8996-c1aadc24b5dd\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"23504eae-984b-4b1a-afc7-30f903ea2ca1\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"23504eae-984b-4b1a-afc7-30f903ea2ca1\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"dea7c409-79d4-4bac-956b-ee9fcf4bac66\",\"w\":24,\"x\":0,\"y\":15},\"panelIndex\":\"dea7c409-79d4-4bac-956b-ee9fcf4bac66\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"dfe82918-64c3-4894-b077-546f10ac5fa9\",\"w\":24,\"x\":24,\"y\":15},\"panelIndex\":\"dfe82918-64c3-4894-b077-546f10ac5fa9\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_3\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-15M","timeRestore":true,"timeTo":"now","title":"TANF Total Number of One Parent Families","version":1},"id":"ca05bcf0-194b-11ef-a9c8-cb02c1885723","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"2b5b0c10-192b-11ef-a9c8-cb02c1885723","name":"panel_0","type":"visualization"},{"id":"158296f0-192c-11ef-a9c8-cb02c1885723","name":"panel_1","type":"visualization"},{"id":"a2e22600-192c-11ef-a9c8-cb02c1885723","name":"panel_2","type":"visualization"},{"id":"0bd80070-192e-11ef-a9c8-cb02c1885723","name":"panel_3","type":"visualization"}],"type":"dashboard","updated_at":"2024-06-03T13:12:21.238Z","version":"WzIwLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"U.S. Total Number of Two Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"U.S. Total Number of Two Parent Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_2_PARENTS\",\"customLabel\":\"U.S. Total Number of Two Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"M\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year and Month\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":7,\"rotate\":90},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"wiggle\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"U.S. Total Number of Two Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"U.S. Total Number of Two Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"3b4b3f50-192b-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:37:49.256Z","version":"WzQxNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"U.S. Average Number of Two Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"U.S. Average Number of Two Parent Families\",\"type\":\"metric\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR\",\"interval\":\"auto\",\"min_doc_count\":false,\"has_extended_bounds\":false,\"extended_bounds\":{\"min\":\"\",\"max\":\"\"}}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_2_PARENTS\",\"customLabel\":\"\"}},\"customLabel\":\"Average Number of Two Parent Families\"},\"schema\":\"metric\"}],\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":false},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}}}"},"id":"2a361ea0-192c-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T13:12:21.238Z","version":"WzIyLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"STT Total Number of Two Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"STT Total Number of Two Parent Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_2_PARENTS\",\"customLabel\":\"Total Number of Two Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15w\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"M\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year and Month\"},\"schema\":\"group\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"datafile.stt.name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"STT\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"wiggle\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Total Number of Two Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Total Number of Two Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"835bff60-192a-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:19:02.744Z","version":"WzI2NiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"STT Average Number of Two Parent Families","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"STT Average Number of Two Parent Families\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"params\":{\"filters\":[{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202010 AND RPT_MONTH_YEAR <= 202010\",\"language\":\"kuery\"},\"label\":\"\"},{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202011 AND RPT_MONTH_YEAR <= 202011\",\"language\":\"kuery\"},\"label\":\"\"},{\"input\":{\"query\":\"RPT_MONTH_YEAR >= 202012 AND RPT_MONTH_YEAR <= 202012\",\"language\":\"kuery\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NUM_2_PARENTS\"}},\"customLabel\":\"Average Number of One Parent Families\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"RPT_MONTH_YEAR_DATE\",\"timeRange\":{\"from\":\"now-15y\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"y\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Reporting Year\"},\"schema\":\"group\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"datafile.stt.name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":498,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"STT\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Average Number of One Parent Families\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Average Number of One Parent Families\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"4a11fbd0-192d-11ef-a9c8-cb02c1885723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"e449bc60-1e96-11ef-865e-439e1adf41ac","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2024-06-03T14:28:59.979Z","version":"WzMzOCwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"0d669da8-ea39-4cd6-95f1-cd91007859de\"},\"panelIndex\":\"0d669da8-ea39-4cd6-95f1-cd91007859de\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"07b45481-e07c-47dc-91ac-dc745b5583bb\"},\"panelIndex\":\"07b45481-e07c-47dc-91ac-dc745b5583bb\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":15,\"w\":24,\"h\":15,\"i\":\"7d979eca-f864-42e0-806f-be2439add112\"},\"panelIndex\":\"7d979eca-f864-42e0-806f-be2439add112\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":15,\"w\":24,\"h\":15,\"i\":\"372fe07d-ec64-468a-ad77-81f01595c985\"},\"panelIndex\":\"372fe07d-ec64-468a-ad77-81f01595c985\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-15M","timeRestore":true,"timeTo":"now","title":"TANF Total Number of Two Parent Families","version":1},"id":"106bbdc0-194c-11ef-a9c8-cb02c1885723","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"3b4b3f50-192b-11ef-a9c8-cb02c1885723","name":"panel_0","type":"visualization"},{"id":"2a361ea0-192c-11ef-a9c8-cb02c1885723","name":"panel_1","type":"visualization"},{"id":"835bff60-192a-11ef-a9c8-cb02c1885723","name":"panel_2","type":"visualization"},{"id":"4a11fbd0-192d-11ef-a9c8-cb02c1885723","name":"panel_3","type":"visualization"}],"type":"dashboard","updated_at":"2024-06-03T13:12:21.238Z","version":"WzI1LDFd"} +{"exportedCount":21,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py index a3b746a66..d0c7a9934 100644 --- a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py +++ b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py @@ -2,6 +2,7 @@ from django.core.management.base import BaseCommand from django.core.management import call_command +from django.core.paginator import Paginator from django.db.utils import DatabaseError from elasticsearch.exceptions import ElasticsearchException from tdpservice.data_files.models import DataFile @@ -33,7 +34,7 @@ def add_arguments(self, parser): parser.add_argument("-a", "--all", action='store_true', help="Clean and reparse all datafiles. If selected, " "fiscal_year/quarter aren't necessary.") - def __get_log_context(self, system_user): + def _get_log_context(self, system_user): """Return logger context.""" context = {'user_id': system_user.id, 'action_flag': ADDITION, @@ -41,7 +42,7 @@ def __get_log_context(self, system_user): } return context - def __backup(self, backup_file_name, log_context): + def _backup(self, backup_file_name, log_context): """Execute Postgres DB backup.""" try: logger.info("Beginning reparse DB Backup.") @@ -57,10 +58,11 @@ def __backup(self, backup_file_name, log_context): level='error') raise e - def __handle_elastic(self, new_indices, log_context): + def _handle_elastic(self, new_indices, log_context): """Create new Elastic indices and delete old ones.""" if new_indices: try: + logger.info("Creating new elastic indexes.") call_command('tdp_search_index', '--create', '-f', '--use-alias') log("Index creation complete.", logger_context=log_context, @@ -72,17 +74,20 @@ def __handle_elastic(self, new_indices, log_context): level='error') raise e except Exception as e: - log("Caught generic exception in __handle_elastic. Clean and reparse NOT executed. " + log("Caught generic exception in _handle_elastic. Clean and reparse NOT executed. " "Database is CONSISTENT, Elastic is INCONSISTENT!", logger_context=log_context, level='error') raise e - def __delete_summaries(self, file_ids, log_context): + def _delete_summaries(self, file_ids, log_context): """Raw delete all DataFileSummary objects.""" try: qset = DataFileSummary.objects.filter(datafile_id__in=file_ids) + count = qset.count() + logger.info(f"Deleting {count} datafile summary objects.") qset._raw_delete(qset.db) + logger.info("Successfully deleted datafile summary objects.") except DatabaseError as e: log('Encountered a DatabaseError while deleting DataFileSummary from Postgres. The database ' 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!', @@ -96,24 +101,34 @@ def __delete_summaries(self, file_ids, log_context): level='critical') raise e - def __delete_records(self, file_ids, new_indices, log_context): + def __handle_elastic_doc_delete(self, doc, qset, model, elastic_exceptions, new_indices): + """Delete documents from Elastic and handle exceptions.""" + if not new_indices: + # If we aren't creating new indices, then we don't want duplicate data in the existing indices. + # We alos use a Paginator here because it allows us to slice querysets based on a batch size. This + # prevents a very large queryset from being brought into main memory when `doc().update(...)` + # evaluates it by iterating over the queryset and deleting the models from ES. + paginator = Paginator(qset, settings.BULK_CREATE_BATCH_SIZE) + for page in paginator: + try: + doc().update(page.object_list, refresh=True, action='delete') + except ElasticsearchException: + elastic_exceptions[model] = elastic_exceptions.get(model, 0) + 1 + continue + + def _delete_records(self, file_ids, new_indices, log_context): """Delete records, errors, and documents from Postgres and Elastic.""" total_deleted = 0 + elastic_exceptions = dict() for doc in DOCUMENTS: try: model = doc.Django.model - qset = model.objects.filter(datafile_id__in=file_ids) - total_deleted += qset.count() - if not new_indices: - # If we aren't creating new indices, then we don't want duplicate data in the existing indices. - doc().update(qset, refresh=True, action='delete') + qset = model.objects.filter(datafile_id__in=file_ids).order_by('id') + count = qset.count() + total_deleted += count + logger.info(f"Deleting {count} records of type: {model}.") + self.__handle_elastic_doc_delete(doc, qset, model, elastic_exceptions, new_indices) qset._raw_delete(qset.db) - except ElasticsearchException as e: - log(f'Elastic document delete failed for type {model}. The database and Elastic are INCONSISTENT! ' - 'Restore the DB from the backup as soon as possible!', - logger_context=log_context, - level='critical') - raise e except DatabaseError as e: log(f'Encountered a DatabaseError while deleting records of type {model} from Postgres. The database ' 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!', @@ -126,13 +141,23 @@ def __delete_records(self, file_ids, new_indices, log_context): logger_context=log_context, level='critical') raise e + + if elastic_exceptions != {}: + msg = ("Warning: Elastic is inconsistent and the database is consistent. " + "Models which generated the Elastic exception(s) are below:\n") + for key, val in elastic_exceptions.items(): + msg += f"Model: {key} generated {val} Elastic Exception(s) while being deleted.\n" + log(msg, logger_context=log_context, level='warn') return total_deleted - def __delete_errors(self, file_ids, log_context): + def _delete_errors(self, file_ids, log_context): """Raw delete all ParserErrors for each file ID.""" try: qset = ParserError.objects.filter(file_id__in=file_ids) + count = qset.count() + logger.info(f"Deleting {count} parser errors.") qset._raw_delete(qset.db) + logger.info("Successfully deleted parser errors.") except DatabaseError as e: log('Encountered a DatabaseError while deleting ParserErrors from Postgres. The database ' 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!', @@ -146,14 +171,14 @@ def __delete_errors(self, file_ids, log_context): level='critical') raise e - def __delete_associated_models(self, meta_model, file_ids, new_indices, log_context): + def _delete_associated_models(self, meta_model, file_ids, new_indices, log_context): """Delete all models associated to the selected datafiles.""" - self.__delete_summaries(file_ids, log_context) - self.__delete_errors(file_ids, log_context) - num_deleted = self.__delete_records(file_ids, new_indices, log_context) + self._delete_summaries(file_ids, log_context) + self._delete_errors(file_ids, log_context) + num_deleted = self._delete_records(file_ids, new_indices, log_context) meta_model.num_records_deleted = num_deleted - def __handle_datafiles(self, files, meta_model, log_context): + def _handle_datafiles(self, files, meta_model, log_context): """Delete, re-save, and reparse selected datafiles.""" for file in files: try: @@ -167,13 +192,13 @@ def __handle_datafiles(self, files, meta_model, log_context): level='critical') raise e except Exception as e: - log('Caught generic exception in __handle_datafiles. Database and Elastic are INCONSISTENT! ' + log('Caught generic exception in _handle_datafiles. Database and Elastic are INCONSISTENT! ' 'Restore the DB from the backup as soon as possible!', logger_context=log_context, level='critical') raise e - def __count_total_num_records(self, log_context): + def _count_total_num_records(self, log_context): """Count total number of records in the database for meta object.""" try: return count_all_records() @@ -190,7 +215,7 @@ def __count_total_num_records(self, log_context): level='error') exit(1) - def __assert_sequential_execution(self, log_context): + def _assert_sequential_execution(self, log_context): """Assert that no other reparse commands are still executing.""" latest_meta_model = ReparseMeta.get_latest() now = timezone.now() @@ -200,20 +225,27 @@ def __assert_sequential_execution(self, log_context): "Cannot safely execute reparse, please fix manually.", logger_context=log_context, level='error') - exit(1) + return False if (is_not_none and not ReparseMeta.assert_all_files_done(latest_meta_model) and not now > latest_meta_model.timeout_at): log('A previous execution of the reparse command is RUNNING. Cannot execute in parallel, exiting.', logger_context=log_context, level='warn') - exit(1) + return False elif (is_not_none and latest_meta_model.timeout_at is not None and now > latest_meta_model.timeout_at and not ReparseMeta.assert_all_files_done(latest_meta_model)): log("Previous reparse has exceeded the timeout. Allowing execution of the command.", logger_context=log_context, level='warn') + return True + return True + + def _should_exit(self, condition): + """Exit on condition.""" + if condition: + exit(1) - def __calculate_timeout(self, num_files, num_records): + def _calculate_timeout(self, num_files, num_records): """Estimate a timeout parameter based on the number of files and the number of records.""" # Increase by an order of magnitude to have the bases covered. line_parse_time = settings.MEDIAN_LINE_PARSE_TIME * 10 @@ -223,6 +255,14 @@ def __calculate_timeout(self, num_files, num_records): logger.info(f"Setting timeout for the reparse event to be {delta} seconds from meta model creation date.") return delta + def _handle_input(self, testing, continue_msg): + """Handle user input.""" + if not testing: + c = str(input(f'\n{continue_msg}\nContinue [y/n]? ')).lower() + if c not in ['y', 'yes']: + print('Cancelled.') + exit(0) + def handle(self, *args, **options): """Delete and reparse datafiles matching a query.""" fiscal_year = options.get('fiscal_year', None) @@ -230,6 +270,10 @@ def handle(self, *args, **options): reparse_all = options.get('all', False) new_indices = reparse_all is True + # Option that can only be specified by calling `handle` directly and passing it. + testing = options.get('testing', False) + ## + args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all if not args_passed: @@ -270,15 +314,12 @@ def handle(self, *args, **options): fmt_str = f"ALL ({num_files})" if reparse_all else f"({num_files})" continue_msg += "\nThese options will delete and reparse {0} datafiles.".format(fmt_str) - c = str(input(f'\n{continue_msg}\nContinue [y/n]? ')).lower() - if c not in ['y', 'yes']: - print('Cancelled.') - return + self._handle_input(testing, continue_msg) system_user, created = User.objects.get_or_create(username='system') if created: logger.debug('Created reserved system user.') - log_context = self.__get_log_context(system_user) + log_context = self._get_log_context(system_user) all_fy = "All" all_q = "Q1-4" @@ -294,7 +335,8 @@ def handle(self, *args, **options): level='warn') return - self.__assert_sequential_execution(log_context) + is_sequential = self._assert_sequential_execution(log_context) + self._should_exit(not is_sequential) meta_model = ReparseMeta.objects.create(fiscal_quarter=fiscal_quarter, fiscal_year=fiscal_year, all=reparse_all, @@ -304,29 +346,29 @@ def handle(self, *args, **options): # Backup the Postgres DB backup_file_name += f"_rpv{meta_model.pk}.pg" - self.__backup(backup_file_name, log_context) + self._backup(backup_file_name, log_context) meta_model.db_backup_location = backup_file_name meta_model.save() # Create and delete Elastic indices if necessary - self.__handle_elastic(new_indices, log_context) + self._handle_elastic(new_indices, log_context) # Delete records from Postgres and Elastic if necessary file_ids = files.values_list('id', flat=True).distinct() - meta_model.total_num_records_initial = self.__count_total_num_records(log_context) + meta_model.total_num_records_initial = self._count_total_num_records(log_context) meta_model.save() - self.__delete_associated_models(meta_model, file_ids, new_indices, log_context) + self._delete_associated_models(meta_model, file_ids, new_indices, log_context) - meta_model.timeout_at = meta_model.created_at + self.__calculate_timeout(num_files, - meta_model.num_records_deleted) + meta_model.timeout_at = meta_model.created_at + self._calculate_timeout(num_files, + meta_model.num_records_deleted) meta_model.save() logger.info(f"Deleted a total of {meta_model.num_records_deleted} records accross {num_files} files.") # Delete and re-save datafiles to handle cascading dependencies logger.info(f'Deleting and re-parsing {num_files} files') - self.__handle_datafiles(files, meta_model, log_context) + self._handle_datafiles(files, meta_model, log_context) log("Database cleansing complete and all files have been re-scheduling for parsing and validation.", logger_context=log_context, diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0031_alter_tribal_tanf_t4_closure_reason.py b/tdrs-backend/tdpservice/search_indexes/migrations/0031_alter_tribal_tanf_t4_closure_reason.py new file mode 100644 index 000000000..4595c8f52 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0031_alter_tribal_tanf_t4_closure_reason.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2024-08-29 19:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0030_reparse_meta_model'), + ] + + operations = [ + migrations.AlterField( + model_name='tribal_tanf_t4', + name='CLOSURE_REASON', + field=models.CharField(max_length=2, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/reparse_meta.py b/tdrs-backend/tdpservice/search_indexes/models/reparse_meta.py index 15f659d64..ddbf4ce4a 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/reparse_meta.py +++ b/tdrs-backend/tdpservice/search_indexes/models/reparse_meta.py @@ -47,6 +47,21 @@ class Meta: new_indices = models.BooleanField(default=False) delete_old_indices = models.BooleanField(default=False) + @staticmethod + def file_counts_match(meta_model): + """ + Check whether the file counts match. + + This function assumes the meta_model has been passed in a distributed/thread safe way. If the database row + containing this model has not been locked the caller will experience race issues. + """ + print("\n\nINSIDE FILE COUNTS MATCH:") + print(f"{meta_model.num_files_to_reparse }, {meta_model.files_completed}, {meta_model.files_failed}\n\n") + return (meta_model.files_completed == meta_model.num_files_to_reparse or + meta_model.files_completed + meta_model.files_failed == + meta_model.num_files_to_reparse or + meta_model.files_failed == meta_model.num_files_to_reparse) + @staticmethod def assert_all_files_done(meta_model): """ @@ -55,9 +70,7 @@ def assert_all_files_done(meta_model): This function assumes the meta_model has been passed in a distributed/thread safe way. If the database row containing this model has not been locked the caller will experience race issues. """ - if (meta_model.finished or meta_model.files_completed == meta_model.num_files_to_reparse or - meta_model.files_completed + meta_model.files_failed == meta_model.num_files_to_reparse or - meta_model.files_failed == meta_model.num_files_to_reparse): + if meta_model.finished and ReparseMeta.file_counts_match(meta_model): return True return False @@ -88,7 +101,7 @@ def increment_files_completed(reparse_meta_models): try: meta_model = reparse_meta_models.select_for_update().latest("pk") meta_model.files_completed += 1 - if ReparseMeta.assert_all_files_done(meta_model): + if ReparseMeta.file_counts_match(meta_model): ReparseMeta.set_reparse_finished(meta_model) meta_model.save() except DatabaseError: @@ -109,7 +122,7 @@ def increment_files_failed(reparse_meta_models): try: meta_model = reparse_meta_models.select_for_update().latest("pk") meta_model.files_failed += 1 - if ReparseMeta.assert_all_files_done(meta_model): + if ReparseMeta.file_counts_match(meta_model): ReparseMeta.set_reparse_finished(meta_model) meta_model.save() except DatabaseError: diff --git a/tdrs-backend/tdpservice/search_indexes/models/tribal.py b/tdrs-backend/tdpservice/search_indexes/models/tribal.py index d42818336..47fb8887c 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tribal.py @@ -233,7 +233,7 @@ class Meta: STRATUM = models.CharField(max_length=2, null=True, blank=False) ZIP_CODE = models.CharField(max_length=5, null=True, blank=False) DISPOSITION = models.IntegerField(null=True, blank=False) - CLOSURE_REASON = models.IntegerField(null=True, blank=False) + CLOSURE_REASON = models.CharField(max_length=2, null=True, blank=False) REC_SUB_HOUSING = models.IntegerField(null=True, blank=False) REC_MED_ASSIST = models.IntegerField(null=True, blank=False) REC_FOOD_STAMPS = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index dd66010a9..fbaa28648 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -96,7 +96,7 @@ def test_can_create_and_index_tanf_t2_submission(test_datafile): submission.CASE_NUMBER = '1' submission.FAMILY_AFFILIATION = 1 submission.NONCUSTODIAL_PARENT = 1 - submission.DATE_OF_BIRTH = 1 + submission.DATE_OF_BIRTH = "1" submission.SSN = '1' submission.RACE_HISPANIC = 1 submission.RACE_AMER_INDIAN = 1 @@ -107,59 +107,59 @@ def test_can_create_and_index_tanf_t2_submission(test_datafile): submission.GENDER = 1 submission.FED_OASDI_PROGRAM = 1 submission.FED_DISABILITY_STATUS = 1 - submission.DISABLED_TITLE_XIVAPDT = 1 + submission.DISABLED_TITLE_XIVAPDT = "1" submission.AID_AGED_BLIND = 1 submission.RECEIVE_SSI = 1 submission.MARITAL_STATUS = 1 submission.RELATIONSHIP_HOH = "01" submission.PARENT_MINOR_CHILD = 1 submission.NEEDS_PREGNANT_WOMAN = 1 - submission.EDUCATION_LEVEL = 1 + submission.EDUCATION_LEVEL = "01" submission.CITIZENSHIP_STATUS = 1 submission.COOPERATION_CHILD_SUPPORT = 1 - submission.MONTHS_FED_TIME_LIMIT = 1 - submission.MONTHS_STATE_TIME_LIMIT = 1 + submission.MONTHS_FED_TIME_LIMIT = "01" + submission.MONTHS_STATE_TIME_LIMIT = "01" submission.CURRENT_MONTH_STATE_EXEMPT = 1 submission.EMPLOYMENT_STATUS = 1 - submission.WORK_ELIGIBLE_INDICATOR = 1 - submission.WORK_PART_STATUS = 1 - submission.UNSUB_EMPLOYMENT = 1 - submission.SUB_PRIVATE_EMPLOYMENT = 1 - submission.SUB_PUBLIC_EMPLOYMENT = 1 - submission.WORK_EXPERIENCE_HOP = 1 - submission.WORK_EXPERIENCE_EA = 1 - submission.WORK_EXPERIENCE_HOL = 1 - submission.OJT = 1 - submission.JOB_SEARCH_HOP = 1 - submission.JOB_SEARCH_EA = 1 - submission.JOB_SEARCH_HOL = 1 - submission.COMM_SERVICES_HOP = 1 - submission.COMM_SERVICES_EA = 1 - submission.COMM_SERVICES_HOL = 1 - submission.VOCATIONAL_ED_TRAINING_HOP = 1 - submission.VOCATIONAL_ED_TRAINING_EA = 1 - submission.VOCATIONAL_ED_TRAINING_HOL = 1 - submission.JOB_SKILLS_TRAINING_HOP = 1 - submission.JOB_SKILLS_TRAINING_EA = 1 - submission.JOB_SKILLS_TRAINING_HOL = 1 - submission.ED_NO_HIGH_SCHOOL_DIPL_HOP = 1 - submission.ED_NO_HIGH_SCHOOL_DIPL_EA = 1 - submission.ED_NO_HIGH_SCHOOL_DIPL_HOL = 1 - submission.SCHOOL_ATTENDENCE_HOP = 1 - submission.SCHOOL_ATTENDENCE_EA = 1 - submission.SCHOOL_ATTENDENCE_HOL = 1 - submission.PROVIDE_CC_HOP = 1 - submission.PROVIDE_CC_EA = 1 - submission.PROVIDE_CC_HOL = 1 - submission.OTHER_WORK_ACTIVITIES = 1 - submission.DEEMED_HOURS_FOR_OVERALL = 1 - submission.DEEMED_HOURS_FOR_TWO_PARENT = 1 - submission.EARNED_INCOME = 1 - submission.UNEARNED_INCOME_TAX_CREDIT = 1 - submission.UNEARNED_SOCIAL_SECURITY = 1 - submission.UNEARNED_SSI = 1 - submission.UNEARNED_WORKERS_COMP = 1 - submission.OTHER_UNEARNED_INCOME = 1 + submission.WORK_ELIGIBLE_INDICATOR = "01" + submission.WORK_PART_STATUS = "01" + submission.UNSUB_EMPLOYMENT = "01" + submission.SUB_PRIVATE_EMPLOYMENT = "01" + submission.SUB_PUBLIC_EMPLOYMENT = "01" + submission.WORK_EXPERIENCE_HOP = "01" + submission.WORK_EXPERIENCE_EA = "01" + submission.WORK_EXPERIENCE_HOL = "01" + submission.OJT = "01" + submission.JOB_SEARCH_HOP = "01" + submission.JOB_SEARCH_EA = "01" + submission.JOB_SEARCH_HOL = "01" + submission.COMM_SERVICES_HOP = "01" + submission.COMM_SERVICES_EA = "01" + submission.COMM_SERVICES_HOL = "01" + submission.VOCATIONAL_ED_TRAINING_HOP = "01" + submission.VOCATIONAL_ED_TRAINING_EA = "01" + submission.VOCATIONAL_ED_TRAINING_HOL = "01" + submission.JOB_SKILLS_TRAINING_HOP = "01" + submission.JOB_SKILLS_TRAINING_EA = "01" + submission.JOB_SKILLS_TRAINING_HOL = "01" + submission.ED_NO_HIGH_SCHOOL_DIPL_HOP = "01" + submission.ED_NO_HIGH_SCHOOL_DIPL_EA = "01" + submission.ED_NO_HIGH_SCHOOL_DIPL_HOL = "01" + submission.SCHOOL_ATTENDENCE_HOP = "01" + submission.SCHOOL_ATTENDENCE_EA = "01" + submission.SCHOOL_ATTENDENCE_HOL = "01" + submission.PROVIDE_CC_HOP = "01" + submission.PROVIDE_CC_EA = "01" + submission.PROVIDE_CC_HOL = "01" + submission.OTHER_WORK_ACTIVITIES = "01" + submission.DEEMED_HOURS_FOR_OVERALL = "01" + submission.DEEMED_HOURS_FOR_TWO_PARENT = "01" + submission.EARNED_INCOME = "01" + submission.UNEARNED_INCOME_TAX_CREDIT = "01" + submission.UNEARNED_SOCIAL_SECURITY = "01" + submission.UNEARNED_SSI = "01" + submission.UNEARNED_WORKERS_COMP = "01" + submission.OTHER_UNEARNED_INCOME = "01" submission.save() @@ -802,7 +802,7 @@ def test_can_create_and_index_tribal_tanf_t2_submission(test_datafile): submission.CASE_NUMBER = '1' submission.FAMILY_AFFILIATION = 1 submission.NONCUSTODIAL_PARENT = 1 - submission.DATE_OF_BIRTH = 1 + submission.DATE_OF_BIRTH = "1" submission.SSN = '1' submission.RACE_HISPANIC = 1 submission.RACE_AMER_INDIAN = 1 @@ -813,41 +813,41 @@ def test_can_create_and_index_tribal_tanf_t2_submission(test_datafile): submission.GENDER = 1 submission.FED_OASDI_PROGRAM = 1 submission.FED_DISABILITY_STATUS = 1 - submission.DISABLED_TITLE_XIVAPDT = 1 + submission.DISABLED_TITLE_XIVAPDT = "01" submission.AID_AGED_BLIND = 1 submission.RECEIVE_SSI = 1 submission.MARITAL_STATUS = 1 submission.RELATIONSHIP_HOH = "01" submission.NEEDS_PREGNANT_WOMAN = 1 - submission.EDUCATION_LEVEL = 1 + submission.EDUCATION_LEVEL = "01" submission.CITIZENSHIP_STATUS = 1 submission.COOPERATION_CHILD_SUPPORT = 1 - submission.MONTHS_FED_TIME_LIMIT = 1 - submission.MONTHS_STATE_TIME_LIMIT = 1 + submission.MONTHS_FED_TIME_LIMIT = "01" + submission.MONTHS_STATE_TIME_LIMIT = "01" submission.CURRENT_MONTH_STATE_EXEMPT = 1 submission.EMPLOYMENT_STATUS = 1 - submission.WORK_PART_STATUS = 1 - submission.UNSUB_EMPLOYMENT = 1 - submission.SUB_PRIVATE_EMPLOYMENT = 1 - submission.SUB_PUBLIC_EMPLOYMENT = 1 - submission.WORK_EXPERIENCE = 1 - submission.OJT = 1 - submission.JOB_SEARCH = 1 - submission.COMM_SERVICES = 1 - submission.VOCATIONAL_ED_TRAINING = 1 - submission.JOB_SKILLS_TRAINING = 1 - submission.ED_NO_HIGH_SCHOOL_DIPLOMA = 1 - submission.SCHOOL_ATTENDENCE = 1 - submission.PROVIDE_CC = 1 + submission.WORK_PART_STATUS = "01" + submission.UNSUB_EMPLOYMENT = "01" + submission.SUB_PRIVATE_EMPLOYMENT = "01" + submission.SUB_PUBLIC_EMPLOYMENT = "01" + submission.WORK_EXPERIENCE = "01" + submission.OJT = "01" + submission.JOB_SEARCH = "01" + submission.COMM_SERVICES = "01" + submission.VOCATIONAL_ED_TRAINING = "01" + submission.JOB_SKILLS_TRAINING = "01" + submission.ED_NO_HIGH_SCHOOL_DIPLOMA = "01" + submission.SCHOOL_ATTENDENCE = "01" + submission.PROVIDE_CC = "01" submission.ADD_WORK_ACTIVITIES = '01' - submission.OTHER_WORK_ACTIVITIES = 1 - submission.REQ_HRS_WAIVER_DEMO = 1 - submission.EARNED_INCOME = 1 - submission.UNEARNED_INCOME_TAX_CREDIT = 1 - submission.UNEARNED_SOCIAL_SECURITY = 1 - submission.UNEARNED_SSI = 1 - submission.UNEARNED_WORKERS_COMP = 1 - submission.OTHER_UNEARNED_INCOME = 1 + submission.OTHER_WORK_ACTIVITIES = "01" + submission.REQ_HRS_WAIVER_DEMO = "01" + submission.EARNED_INCOME = "01" + submission.UNEARNED_INCOME_TAX_CREDIT = "01" + submission.UNEARNED_SOCIAL_SECURITY = "01" + submission.UNEARNED_SSI = "01" + submission.UNEARNED_WORKERS_COMP = "01" + submission.OTHER_UNEARNED_INCOME = "01" submission.save() diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_reparse.py b/tdrs-backend/tdpservice/search_indexes/test/test_reparse.py new file mode 100644 index 000000000..2c8647cea --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/test/test_reparse.py @@ -0,0 +1,509 @@ +"""Test cases for reparse functions.""" + +import pytest +from tdpservice.parsers import util, parse +from tdpservice.parsers.test.factories import DataFileSummaryFactory +from tdpservice.search_indexes.management.commands import clean_and_reparse +from tdpservice.search_indexes.models.reparse_meta import ReparseMeta +from tdpservice.users.models import User + +from django.contrib.admin.models import LogEntry, ADDITION +from django.db.utils import DatabaseError +from django.utils import timezone +from elasticsearch.exceptions import ElasticsearchException + +from datetime import timedelta +import os +import time + +@pytest.fixture +def cat4_edge_case_file(stt_user, stt): + """Fixture for cat_4_edge_case.txt.""" + cat4_edge_case_file = util.create_test_datafile('cat_4_edge_case.txt', stt_user, stt) + cat4_edge_case_file.year = 2024 + cat4_edge_case_file.quarter = 'Q1' + cat4_edge_case_file.save() + return cat4_edge_case_file + +@pytest.fixture +def big_file(stt_user, stt): + """Fixture for ADS.E2J.FTP1.TS06.""" + return util.create_test_datafile('ADS.E2J.FTP1.TS06', stt_user, stt) + +@pytest.fixture +def small_ssp_section1_datafile(stt_user, stt): + """Fixture for small_ssp_section1.""" + small_ssp_section1_datafile = util.create_test_datafile('small_ssp_section1.txt', stt_user, + stt, 'SSP Active Case Data') + small_ssp_section1_datafile.year = 2024 + small_ssp_section1_datafile.quarter = 'Q1' + small_ssp_section1_datafile.save() + return small_ssp_section1_datafile + +@pytest.fixture +def tribal_section_1_file(stt_user, stt): + """Fixture for ADS.E2J.FTP4.TS06.""" + tribal_section_1_file = util.create_test_datafile('ADS.E2J.FTP1.TS142', stt_user, stt, "Tribal Active Case Data") + tribal_section_1_file.year = 2022 + tribal_section_1_file.quarter = 'Q1' + tribal_section_1_file.save() + return tribal_section_1_file + +@pytest.fixture +def dfs(): + """Fixture for DataFileSummary.""" + return DataFileSummaryFactory.build() + +@pytest.fixture +def log_context(): + """Fixture for logger context.""" + system_user, created = User.objects.get_or_create(username='system') + context = {'user_id': system_user.id, + 'action_flag': ADDITION, + 'object_repr': "Test Clean and Reparse" + } + return context + +def parse_files(summary, f1, f2, f3, f4): + """Parse all files.""" + summary.datafile = f1 + parse.parse_datafile(f1, summary) + + summary.datafile = f2 + parse.parse_datafile(f2, summary) + + summary.datafile = f3 + parse.parse_datafile(f3, summary) + + summary.datafile = f4 + parse.parse_datafile(f4, summary) + f1.save() + f2.save() + f3.save() + f4.save() + return [f1.pk, f2.pk, f3.pk, f4.pk] + +@pytest.mark.django_db +def test_count_total_num_records(log_context, dfs, cat4_edge_case_file, big_file, + small_ssp_section1_datafile, tribal_section_1_file): + """Count total number of records in DB.""" + parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + + cmd = clean_and_reparse.Command() + assert 3104 == cmd._count_total_num_records(log_context) + cat4_edge_case_file.delete() + assert 3096 == cmd._count_total_num_records(log_context) + +@pytest.mark.django_db +def test_reparse_backup_succeed(log_context, dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, + tribal_section_1_file): + """Verify a backup is created with the correct size.""" + parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + + cmd = clean_and_reparse.Command() + file_name = "/tmp/test_reparse.pg" + cmd._backup(file_name, log_context) + time.sleep(10) + + file_size = os.path.getsize(file_name) + assert file_size > 180000 + +@pytest.mark.django_db +def test_reparse_backup_fail(mocker, log_context, dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, + tribal_section_1_file): + """Verify a backup is created with the correct size.""" + parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + + mocker.patch( + 'tdpservice.search_indexes.management.commands.clean_and_reparse.Command._backup', + side_effect=Exception('Backup exception') + ) + cmd = clean_and_reparse.Command() + file_name = "/tmp/test_reparse.pg" + with pytest.raises(Exception): + cmd._backup(file_name, log_context) + assert os.path.exists(file_name) is False + exception_msg = LogEntry.objects.latest('pk').change_message + assert exception_msg == ("Database backup FAILED. Clean and reparse NOT executed. Database " + "and Elastic are CONSISTENT!") + +@pytest.mark.parametrize(("new_indexes"), [ + (True), + (False) +]) +@pytest.mark.django_db +def test_delete_associated_models(log_context, new_indexes, dfs, cat4_edge_case_file, big_file, + small_ssp_section1_datafile, tribal_section_1_file): + """Verify all records and models are deleted.""" + ids = parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + + cmd = clean_and_reparse.Command() + assert 3104 == cmd._count_total_num_records(log_context) + + class Fake: + pass + fake_meta = Fake() + cmd._delete_associated_models(fake_meta, ids, new_indexes, log_context) + + assert cmd._count_total_num_records(log_context) == 0 + +@pytest.mark.parametrize(("exc_msg, exception_type"), [ + (('Encountered a DatabaseError while deleting DataFileSummary from Postgres. The database ' + 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!'), DatabaseError), + (('Caught generic exception while deleting DataFileSummary. The database and Elastic are INCONSISTENT! ' + 'Restore the DB from the backup as soon as possible!'), Exception) +]) +@pytest.mark.django_db +def test_delete_summaries_exceptions(mocker, log_context, exc_msg, exception_type): + """Test summary exception handling.""" + mocker.patch( + 'tdpservice.search_indexes.management.commands.clean_and_reparse.Command._delete_summaries', + side_effect=exception_type('Summary delete exception') + ) + cmd = clean_and_reparse.Command() + with pytest.raises(exception_type): + cmd._delete_summaries([], log_context) + exception_msg = LogEntry.objects.latest('pk').change_message + assert exception_msg == exc_msg + +@pytest.mark.parametrize(("exc_msg, exception_type"), [ + (('Elastic index creation FAILED. Clean and reparse NOT executed. ' + 'Database is CONSISTENT, Elastic is INCONSISTENT!'), ElasticsearchException), + (('Caught generic exception in _handle_elastic. Clean and reparse NOT executed. ' + 'Database is CONSISTENT, Elastic is INCONSISTENT!'), Exception) +]) +@pytest.mark.django_db +def test_handle_elastic_exceptions(mocker, log_context, exc_msg, exception_type): + """Test summary exception handling.""" + mocker.patch( + 'tdpservice.search_indexes.management.commands.clean_and_reparse.Command._handle_elastic', + side_effect=exception_type('Summary delete exception') + ) + cmd = clean_and_reparse.Command() + with pytest.raises(exception_type): + cmd._handle_elastic([], True, log_context) + exception_msg = LogEntry.objects.latest('pk').change_message + assert exception_msg == exc_msg + +@pytest.mark.parametrize(("exc_msg, exception_type"), [ + (('Elastic document delete failed for type {model}. The database and Elastic are INCONSISTENT! ' + 'Restore the DB from the backup as soon as possible!'), ElasticsearchException), + (('Encountered a DatabaseError while deleting records of type {model} from Postgres. The database ' + 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!'), DatabaseError), + (('Caught generic exception while deleting records of type {model}. The database and Elastic are ' + 'INCONSISTENT! Restore the DB from the backup as soon as possible!'), Exception) +]) +@pytest.mark.django_db +def test_delete_records_exceptions(mocker, log_context, exc_msg, exception_type): + """Test record exception handling.""" + mocker.patch( + 'tdpservice.search_indexes.management.commands.clean_and_reparse.Command._delete_records', + side_effect=exception_type('Record delete exception') + ) + cmd = clean_and_reparse.Command() + with pytest.raises(exception_type): + cmd._delete_records([], True, log_context) + exception_msg = LogEntry.objects.latest('pk').change_message + assert exception_msg == exc_msg + +@pytest.mark.parametrize(("exc_msg, exception_type"), [ + (('Encountered a DatabaseError while deleting ParserErrors from Postgres. The database ' + 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!'), DatabaseError), + (('Caught generic exception while deleting ParserErrors. The database and Elastic are INCONSISTENT! ' + 'Restore the DB from the backup as soon as possible!'), Exception) +]) +@pytest.mark.django_db +def test_delete_errors_exceptions(mocker, log_context, exc_msg, exception_type): + """Test error exception handling.""" + mocker.patch( + 'tdpservice.search_indexes.management.commands.clean_and_reparse.Command._delete_errors', + side_effect=exception_type('Error delete exception') + ) + cmd = clean_and_reparse.Command() + with pytest.raises(exception_type): + cmd._delete_errors([], log_context) + exception_msg = LogEntry.objects.latest('pk').change_message + assert exception_msg == exc_msg + +@pytest.mark.parametrize(("exc_msg, exception_type"), [ + (('Encountered a DatabaseError while re-creating datafiles. The database ' + 'and Elastic are INCONSISTENT! Restore the DB from the backup as soon as possible!'), DatabaseError), + (('Caught generic exception in _handle_datafiles. Database and Elastic are INCONSISTENT! ' + 'Restore the DB from the backup as soon as possible!'), Exception) +]) +@pytest.mark.django_db +def test_handle_files_exceptions(mocker, log_context, exc_msg, exception_type): + """Test error exception handling.""" + mocker.patch( + 'tdpservice.search_indexes.management.commands.clean_and_reparse.Command._handle_datafiles', + side_effect=exception_type('Files exception') + ) + cmd = clean_and_reparse.Command() + with pytest.raises(exception_type): + cmd._handle_datafiles([], None, log_context) + exception_msg = LogEntry.objects.latest('pk').change_message + assert exception_msg == exc_msg + +@pytest.mark.django_db +def test_timeout_calculation(log_context, dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, + tribal_section_1_file): + """Verify calculated timeout.""" + ids = parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + + cmd = clean_and_reparse.Command() + num_records = cmd._count_total_num_records(log_context) + + assert cmd._calculate_timeout(len(ids), num_records).seconds == 57 + + assert cmd._calculate_timeout(len(ids), 50).seconds == 40 + +@pytest.mark.django_db +def test_reparse_dunce(): + """Test reparse no args.""" + cmd = clean_and_reparse.Command() + assert None is cmd.handle() + assert ReparseMeta.objects.count() == 0 + +@pytest.mark.django_db +def test_reparse_sequential(log_context): + """Test reparse _assert_sequential_execution.""" + cmd = clean_and_reparse.Command() + assert True is cmd._assert_sequential_execution(log_context) + + meta = ReparseMeta.objects.create(timeout_at=None) + assert False is cmd._assert_sequential_execution(log_context) + timeout_entry = LogEntry.objects.latest('pk') + assert timeout_entry.change_message == ( + f"The latest ReparseMeta model's (ID: {meta.pk}) timeout_at field is None. Cannot " + "safely execute reparse, please fix manually." + ) + + meta.timeout_at = timezone.now() + timedelta(seconds=100) + meta.save() + assert False is cmd._assert_sequential_execution(log_context) + not_seq_entry = LogEntry.objects.latest('pk') + assert not_seq_entry.change_message == ("A previous execution of the reparse command is RUNNING. " + "Cannot execute in parallel, exiting.") + + meta.timeout_at = timezone.now() + meta.save() + assert True is cmd._assert_sequential_execution(log_context) + timeout_entry = LogEntry.objects.latest('pk') + assert timeout_entry.change_message == ("Previous reparse has exceeded the timeout. Allowing " + "execution of the command.") + +@pytest.mark.django_db() +def test_reparse_quarter_and_year(mocker, dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, + tribal_section_1_file): + """Test reparse with year and quarter.""" + parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + cmd = clean_and_reparse.Command() + + mocker.patch( + 'tdpservice.scheduling.parser_task.parse', + return_value=None + ) + + opts = {'fiscal_quarter': 'Q1', 'fiscal_year': 2021, 'testing': True} + cmd.handle(**opts) + + latest = ReparseMeta.objects.select_for_update().latest("pk") + assert latest.num_files_to_reparse == 1 + assert latest.num_records_deleted == 3073 + +@pytest.mark.django_db() +def test_reparse_quarter(mocker, dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, + tribal_section_1_file): + """Test reparse with quarter.""" + parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + cmd = clean_and_reparse.Command() + + mocker.patch( + 'tdpservice.scheduling.parser_task.parse', + return_value=None + ) + + opts = {'fiscal_quarter': 'Q1', 'testing': True} + cmd.handle(**opts) + + latest = ReparseMeta.objects.select_for_update().latest("pk") + assert latest.num_files_to_reparse == 4 + assert latest.num_records_deleted == 3104 + +@pytest.mark.django_db() +def test_reparse_year(mocker, dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, + tribal_section_1_file): + """Test reparse year.""" + parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + cmd = clean_and_reparse.Command() + + mocker.patch( + 'tdpservice.scheduling.parser_task.parse', + return_value=None + ) + + opts = {'fiscal_year': 2024, 'testing': True} + cmd.handle(**opts) + + latest = ReparseMeta.objects.select_for_update().latest("pk") + assert latest.num_files_to_reparse == 2 + assert latest.num_records_deleted == 27 + +@pytest.mark.django_db() +def test_reparse_all(mocker, dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, + tribal_section_1_file): + """Test reparse all.""" + parse_files(dfs, cat4_edge_case_file, big_file, small_ssp_section1_datafile, tribal_section_1_file) + cmd = clean_and_reparse.Command() + + mocker.patch( + 'tdpservice.scheduling.parser_task.parse', + return_value=None + ) + + opts = {'all': True, 'testing': True} + cmd.handle(**opts) + + latest = ReparseMeta.objects.select_for_update().latest("pk") + assert latest.num_files_to_reparse == 4 + assert latest.num_records_deleted == 3104 + +@pytest.mark.django_db() +def test_reparse_no_files(mocker): + """Test reparse with no files in query.""" + cmd = clean_and_reparse.Command() + + mocker.patch( + 'tdpservice.scheduling.parser_task.parse', + return_value=None + ) + + opts = {'fiscal_year': 2025, 'testing': True} + res = cmd.handle(**opts) + + assert ReparseMeta.objects.count() == 0 + assert res is None + assert LogEntry.objects.latest('pk').change_message == ("No files available for the selected Fiscal Year: 2025 and " + "Quarter: Q1-4. Nothing to do.") + +@pytest.mark.django_db() +def test_mm_all_files_done(): + """Test meta model all files done.""" + meta_model = ReparseMeta.objects.create() + assert ReparseMeta.assert_all_files_done(meta_model) is False + + meta_model.finished = True + meta_model.files_completed = 1 + meta_model.num_files_to_reparse = 1 + assert ReparseMeta.assert_all_files_done(meta_model) is True + +@pytest.mark.django_db() +def test_mm_increment_files_completed(big_file): + """Test meta model increment files completed.""" + meta_model = ReparseMeta.objects.create(num_files_to_reparse=2, all=True) + big_file.reparse_meta_models.add(meta_model) + big_file.save() + + ReparseMeta.increment_files_completed(big_file.reparse_meta_models) + meta_model = ReparseMeta.get_latest() + assert meta_model.finished is False + assert meta_model.files_completed == 1 + assert meta_model.files_failed == 0 + + ReparseMeta.increment_files_completed(big_file.reparse_meta_models) + meta_model = ReparseMeta.get_latest() + assert meta_model.finished is True + assert meta_model.files_completed == 2 + assert meta_model.files_failed == 0 + + assert meta_model.success is True + + assert ReparseMeta.assert_all_files_done(meta_model) is True + +@pytest.mark.django_db() +def test_mm_increment_files_failed(big_file): + """Test meta model increment files failed.""" + meta_model = ReparseMeta.objects.create(num_files_to_reparse=2, all=True) + big_file.reparse_meta_models.add(meta_model) + big_file.save() + + ReparseMeta.increment_files_failed(big_file.reparse_meta_models) + meta_model = ReparseMeta.get_latest() + assert meta_model.finished is False + assert meta_model.files_completed == 0 + assert meta_model.files_failed == 1 + + ReparseMeta.increment_files_failed(big_file.reparse_meta_models) + meta_model = ReparseMeta.get_latest() + assert meta_model.finished is True + assert meta_model.files_completed == 0 + assert meta_model.files_failed == 2 + + assert meta_model.success is False + + assert ReparseMeta.assert_all_files_done(meta_model) is True + +@pytest.mark.django_db() +def test_mm_increment_files_failed_and_passed(big_file): + """Test meta model both increment failed and passed files.""" + meta_model = ReparseMeta.objects.create(num_files_to_reparse=2, all=True) + big_file.reparse_meta_models.add(meta_model) + big_file.save() + + ReparseMeta.increment_files_completed(big_file.reparse_meta_models) + meta_model = ReparseMeta.get_latest() + assert meta_model.finished is False + assert meta_model.files_completed == 1 + assert meta_model.files_failed == 0 + + ReparseMeta.increment_files_failed(big_file.reparse_meta_models) + meta_model = ReparseMeta.get_latest() + assert meta_model.finished is True + assert meta_model.files_completed == 1 + assert meta_model.files_failed == 1 + + assert meta_model.success is False + + assert ReparseMeta.assert_all_files_done(meta_model) is True + +@pytest.mark.django_db() +def test_mm_increment_records_created(big_file): + """Test meta model increment records created.""" + meta_model = ReparseMeta.objects.create(num_files_to_reparse=2, all=True) + big_file.reparse_meta_models.add(meta_model) + big_file.save() + + ReparseMeta.increment_records_created(big_file.reparse_meta_models, 500) + meta_model = ReparseMeta.get_latest() + assert meta_model.num_records_created == 500 + + ReparseMeta.increment_records_created(big_file.reparse_meta_models, 888) + meta_model = ReparseMeta.get_latest() + assert meta_model.num_records_created == 1388 + +@pytest.mark.django_db() +def test_mm_get_latest(): + """Test get latest meta model.""" + assert ReparseMeta.get_latest() is None + meta1 = ReparseMeta.objects.create() + assert ReparseMeta.get_latest() == meta1 + + ReparseMeta.objects.create() + assert ReparseMeta.get_latest() != meta1 + +@pytest.mark.django_db() +def test_mm_file_counts_match(): + """Test meta model file counts match.""" + meta_model = ReparseMeta.objects.create(num_files_to_reparse=2) + assert ReparseMeta.file_counts_match(meta_model) is False + + meta_model.files_completed = 2 + assert ReparseMeta.file_counts_match(meta_model) is True + + meta_model.files_completed = 0 + meta_model.files_failed = 2 + assert ReparseMeta.file_counts_match(meta_model) is True + + meta_model.files_completed = 1 + meta_model.files_failed = 1 + assert ReparseMeta.file_counts_match(meta_model) is True diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index cd7b5274b..ba936b545 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -499,6 +499,10 @@ class Common(Configuration): 'task': 'tdpservice.email.tasks.email_admin_num_access_requests', 'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST) }, + 'Email Admin Number of Stuck Files' : { + 'task': 'tdpservice.data_files.tasks.notify_stuck_files', + 'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST) + }, 'Email Data Analyst Q1 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', # Feb 9 at 1pm UTC (9am EST) diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml index 3ee327f3e..13094148b 100644 --- a/tdrs-frontend/docker-compose.yml +++ b/tdrs-frontend/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.4" services: zaproxy: - image: softwaresecurityproject/zap-stable:2.14.0 + image: tdp-docker.dev.raftlabs.tech/dependencies/softwaresecurityproject/zap-stable:2.14.0 container_name: zap-scan command: sleep 3600 ports: @@ -14,6 +14,7 @@ services: tdp-frontend: stdin_open: true # docker run -i tty: true # docker run -t + image: tdp-frontend build: context: . target: nginx diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index 0b81b8114..6257064f4 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -50,8 +50,8 @@ data "cloudfoundry_service" "rds" { resource "cloudfoundry_service_instance" "database" { name = "tdp-db-dev" space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.rds.service_plans["micro-psql"] - json_params = "{\"version\": \"15\"}" + service_plan = data.cloudfoundry_service.rds.service_plans["medium-gp-psql"] + json_params = "{\"version\": \"15\", \"storage_type\": \"gp3\", \"storage\": 50}" recursive_delete = true timeouts { create = "60m" @@ -106,7 +106,14 @@ data "cloudfoundry_service" "elasticsearch" { } resource "cloudfoundry_service_instance" "elasticsearch" { - name = "es-dev" - space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"] + name = "es-dev" + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"] + replace_on_params_change = true + json_params = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}" + timeouts { + create = "60m" + update = "60m" + delete = "2h" + } } diff --git a/terraform/production/main.tf b/terraform/production/main.tf index c9ecf505e..0a2ebb719 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -50,8 +50,8 @@ data "cloudfoundry_service" "rds" { resource "cloudfoundry_service_instance" "database" { name = "tdp-db-prod" space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.rds.service_plans["medium-psql"] - json_params = "{\"version\": \"15\"}" + service_plan = data.cloudfoundry_service.rds.service_plans["medium-gp-psql"] + json_params = "{\"version\": \"15\", \"storage_type\": \"gp3\", \"storage\": 500}" recursive_delete = true timeouts { create = "60m" @@ -87,7 +87,14 @@ data "cloudfoundry_service" "elasticsearch" { } resource "cloudfoundry_service_instance" "elasticsearch" { - name = "es-prod" - space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-medium"] + name = "es-prod" + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-medium"] + replace_on_params_change = true + json_params = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}" + timeouts { + create = "60m" + update = "60m" + delete = "2h" + } } diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 0c4cc2576..6a9af5bde 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -50,8 +50,8 @@ data "cloudfoundry_service" "rds" { resource "cloudfoundry_service_instance" "database" { name = "tdp-db-staging" space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.rds.service_plans["micro-psql"] - json_params = "{\"version\": \"15\"}" + service_plan = data.cloudfoundry_service.rds.service_plans["medium-gp-psql"] + json_params = "{\"version\": \"15\", \"storage_type\": \"gp3\", \"storage\": 50}" recursive_delete = true timeouts { create = "60m" @@ -87,7 +87,14 @@ data "cloudfoundry_service" "elasticsearch" { } resource "cloudfoundry_service_instance" "elasticsearch" { - name = "es-staging" - space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"] + name = "es-staging" + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"] + replace_on_params_change = true + json_params = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}" + timeouts { + create = "60m" + update = "60m" + delete = "2h" + } }