diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml index 634a3c29b..02d3c8f53 100644 --- a/.circleci/base_config.yml +++ b/.circleci/base_config.yml @@ -13,11 +13,11 @@ executors: machine-executor: machine: docker_layer_caching: false - image: ubuntu-2204:2024.01.1 + image: ubuntu-2204:2024.05.1 large-machine-executor: machine: docker_layer_caching: false - image: ubuntu-2204:2024.01.1 + image: ubuntu-2204:2024.05.1 resource_class: large parameters: diff --git a/.circleci/build-and-test/commands.yml b/.circleci/build-and-test/commands.yml index 52cfe7149..70ef3f98d 100644 --- a/.circleci/build-and-test/commands.yml +++ b/.circleci/build-and-test/commands.yml @@ -49,6 +49,13 @@ - run: name: Disable npm audit warnings in CI command: npm set audit false - + # This allows us to use the node orb to install packages within other commands install-nodejs-packages: node/install-packages + + docker-login: + steps: + - run: + name: Docker login + command: | + echo "$CIRCI_DOCKER_LOGIN" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-circi --password-stdin diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml index a40d1568f..469c92250 100644 --- a/.circleci/build-and-test/jobs.yml +++ b/.circleci/build-and-test/jobs.yml @@ -3,6 +3,7 @@ steps: - checkout - docker-compose-check + - docker-login - docker-compose-up-backend - run: name: Run Unit Tests And Create Code Coverage Report @@ -46,6 +47,7 @@ steps: - checkout - docker-compose-check + - docker-login - docker-compose-up-backend - docker-compose-up-frontend - install-nodejs-machine diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index af907351b..d1aa82b7d 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -1,4 +1,33 @@ # commands: + init-deploy: + steps: + - checkout + - sudo-check + - cf-check + + build-and-tag-images: + parameters: + backend-appname: + default: tdp-backend + type: string + frontend-appname: + default: tdp-frontend + type: string + steps: + - run: + name: Update Docker daemon + command: | + sudo echo '{"max-concurrent-uploads": 1}' | sudo tee /etc/docker/daemon.json + sudo service docker restart + - run: + name: Create builder + command: | + docker buildx create --name container-builder --driver docker-container --use --bootstrap + - run: + name: Build and tag images + command: | + ./scripts/build-and-tag-images.sh <> <> ./tdrs-backend ./tdrs-frontend $CIRCLE_BUILD_NUM $CIRCLE_SHA1 "$CIRCI_DOCKER_LOGIN" tdp-circi + deploy-cloud-dot-gov: parameters: environment: @@ -25,9 +54,6 @@ default: tdp-frontend type: string steps: - - checkout - - sudo-check - - cf-check - login-cloud-dot-gov: cf-password: <> cf-org: <> diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index 63d5bc070..ce163101f 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -1,3 +1,33 @@ + build-and-tag-develop: + executor: large-machine-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - build-and-tag-images: + backend-appname: tdp-backend-develop + frontend-appname: tdp-frontend-develop + + build-and-tag-staging: + executor: large-machine-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - build-and-tag-images: + backend-appname: tdp-backend-staging + frontend-appname: tdp-frontend-staging + + build-and-tag-production: + executor: large-machine-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - build-and-tag-images: + backend-appname: tdp-backend-production + frontend-appname: tdp-frontend-production + deploy-dev: parameters: target_env: @@ -5,6 +35,7 @@ executor: docker-executor working_directory: ~/tdp-deploy steps: + - init-deploy - deploy-cloud-dot-gov: backend-appname: tdp-backend-<< parameters.target_env >> frontend-appname: tdp-frontend-<< parameters.target_env >> @@ -13,6 +44,7 @@ executor: docker-executor working_directory: ~/tdp-deploy steps: + - init-deploy - deploy-cloud-dot-gov: backend-appname: tdp-backend-staging frontend-appname: tdp-frontend-staging @@ -24,6 +56,7 @@ executor: docker-executor working_directory: ~/tdp-deploy steps: + - init-deploy - deploy-cloud-dot-gov: backend-appname: tdp-backend-develop frontend-appname: tdp-frontend-develop @@ -133,6 +166,7 @@ executor: docker-executor working_directory: ~/tdp-deploy steps: + - init-deploy - deploy-cloud-dot-gov: environment: production backend-appname: tdp-backend-prod diff --git a/.circleci/deployment/workflows.yml b/.circleci/deployment/workflows.yml index 8a4269c04..a0de09f9e 100644 --- a/.circleci/deployment/workflows.yml +++ b/.circleci/deployment/workflows.yml @@ -93,27 +93,48 @@ - develop - main - master - - deploy-develop: + - build-and-tag-develop: requires: - deploy-infrastructure-staging filters: branches: only: - develop - - deploy-staging: + - deploy-develop: + requires: + - build-and-tag-develop + filters: + branches: + only: + - develop + - build-and-tag-staging: requires: - deploy-infrastructure-staging filters: branches: only: - main - - deploy-production: + - deploy-staging: + requires: + - build-and-tag-staging + filters: + branches: + only: + - main + - build-and-tag-production: requires: - deploy-infrastructure-production filters: branches: only: - master + - deploy-production: + requires: + - build-and-tag-production + filters: + branches: + only: + - master - test-deployment-e2e: requires: - deploy-develop diff --git a/.circleci/generate_config.sh b/.circleci/generate_config.sh old mode 100644 new mode 100755 diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml index 225758ef5..fdabb0a22 100644 --- a/.circleci/owasp/jobs.yml +++ b/.circleci/owasp/jobs.yml @@ -4,6 +4,7 @@ steps: - checkout - docker-compose-check + - docker-login - docker-compose-up-backend - docker-compose-up-frontend - run: @@ -26,6 +27,7 @@ steps: - checkout - docker-compose-check + - docker-login - docker-compose-up-backend - docker-compose-up-frontend - run: @@ -66,6 +68,7 @@ - sudo-check - cf-check - docker-compose-check + - docker-login - login-cloud-dot-gov: cf-password: <> cf-space: <> diff --git a/docs/Technical-Documentation/images/nexus-dev-admin-login.png b/docs/Technical-Documentation/images/nexus-dev-admin-login.png new file mode 100644 index 000000000..d3b00e903 Binary files /dev/null and b/docs/Technical-Documentation/images/nexus-dev-admin-login.png differ diff --git a/docs/Technical-Documentation/nexus-repo.md b/docs/Technical-Documentation/nexus-repo.md index 6f4a15bf5..5e504a384 100644 --- a/docs/Technical-Documentation/nexus-repo.md +++ b/docs/Technical-Documentation/nexus-repo.md @@ -40,7 +40,7 @@ After logging in as root for the first time, you will be taken to a page to set In order to use Nexus as a Docker repository, the DNS for the repo needs to be able to terminate https. We are currently using cloudflare to do this. -When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port. +When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port. In order to allow [Docker client login and connections](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats/docker-registry/docker-authentication) you must set up the Docker Bearer Token Realm in Settings -> Security -> Realms -> and move the Docker Bearer Token Realm over to Active. Also, any users will need nx-repository-view-docker-#{RepoName}-(browse && read) at a minimum and (add and edit) in order to push images. @@ -48,21 +48,86 @@ Also, any users will need nx-repository-view-docker-#{RepoName}-(browse && read) We have a separate endpoint to connect specifically to the docker repository. [https://tdp-docker.dev.raftlabs.tech](tdp-docker.dev.raftlabs.tech) -e.g. `docker login https://tdp-docker.dev.raftlabs.tech` +e.g. +``` +docker login https://tdp-docker.dev.raftlabs.tech +``` ### Pushing Images Before an image can be pushed to the nexus repository, it must be tagged for that repo: -`docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}` +``` +docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version} +``` then you can push: -`docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}` +``` +docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version} +``` ### Pulling Images -We have set up a proxy mirror to dockerhub that can pull and cache DockerHub images. -Then we have created a group docker repository that can be pulled from. If the container is in our hosted repo, the group will return that container. If not, it will see if we have a cached version of that container in our proxy repo and, if not, pull that from dockerhub, cache it and allow the docker pull to happen. +We do not allow anonymous access on our Nexus instance. With that said, if you have not [logged in with Docker](#docker-login) you will not be able to pull. If you are logged in: + +``` +docker pull tdp-docker.dev.raftlabs.tech/${ImageName}:${Version} +``` + +## Nexus Administration + +### UI Admin Login +To administer Nexus via the UI, you will need to access the service key in our dev cloud.gov environment. + +Log in with CloudFoundry +``` +cf login --sso +``` +Be sure to specify the space as `tanf-dev` -`docker pull https://tdp-docker-store.dev.raftlabs.tech/${ImageName}:${Version}` \ No newline at end of file +After you've authenticated you can grab the password from the key: +``` +cf service-key tanf-keys nexus-dev-admin +``` + +The key returns a username and a password: +``` +{ + "credentials": { + "password": REDACTED, + "username": REDACTED + } +} +``` +Copy the `password` to your clipboard and login into the Nexus UI with the `tdp-dev-admin` user. See below: + +![Nexus Dev Admin Login](./images/nexus-dev-admin-login.png) + +### VM Login +To access the VM running Nexus, you will need to gain access to the Raft internal network. To do this, you will need to install CloudFlare's WARP zero trust VPN. Follow the instructions [here](https://gorafttech-my.sharepoint.com/:w:/g/personal/tradin_teamraft_com/EZePOTv0dbdBguHITcoXQF0Bd5JAcqeLsJTlEOktTfIXHA?e=34WqB4) to get setup. From there, reach out to Eric Lipe or Connor Meehan for the IP, username, and password to access the VM. Once you have the credentials, you can login with SSH: +``` +ssh username@IP_Address +``` + +Once logged in, you can run `docker ps` or other docker commands to view and administer the Nexus container as necessary. You should also consider generating an ssh key to avoid having to enter the password each time you login. To do so, run the following commands on your local machine. +``` +ssh-keygen +``` + +``` +ssh-copy-id username@IP_Address +``` +Now you will no longer have to enter the password when logging in. + +## Local Docker Login +After logging into the `tanf-dev` space with the `cf` cli, execute the following commands to authenticate your local docker daemon +``` +export NEXUS_DOCKER_PASSWORD=`cf service-key tanf-keys nexus-dev | tail -n +2 | jq .credentials.password` +echo "$NEXUS_DOCKER_PASSWORD" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-dev --password-stdin +``` + +Sometimes the `docker login...` command above doesn't work. If that happens, just copy the content of `NEXUS_DOCKER_PASSWORD` to your clipboard and paste it when prompted for the password after executing the command below. +``` +docker login https://tdp-docker.dev.raftlabs.tech -u tdp-dev +``` diff --git a/scripts/build-and-tag-images.sh b/scripts/build-and-tag-images.sh new file mode 100755 index 000000000..679485d79 --- /dev/null +++ b/scripts/build-and-tag-images.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +if [ "$#" -ne 8 ]; then + echo "Error, this script expects 8 parameters." + echo "I.e: ./build-tag-images.sh BACKEND_APP_NAME FRONTEND_APP_NAME BACKEND_PATH FRONTEND_PATH BUILD_NUM COMMIT_HASH DOCKER_LOGIN DOCKER_USER" + exit 1 +fi + +BACKEND_APP_NAME=$1 +FRONTEND_APP_NAME=$2 +BACKEND_PATH=$3 +FRONTEND_PATH=$4 +BUILD_NUM=$5 +COMMIT_HASH=$6 +DOCKER_LOGIN=$7 +DOCKER_USER=$8 +BUILD_DATE=`date +%F` +TAG="${BUILD_DATE}_build-${BUILD_NUM}_${COMMIT_HASH}" + +export DOCKER_CLI_EXPERIMENTAL=enabled + +build_and_tag() { + echo "$DOCKER_LOGIN" | docker login https://tdp-docker.dev.raftlabs.tech -u $DOCKER_USER --password-stdin + docker buildx build --load --platform linux/amd64 -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:latest "$BACKEND_PATH" + docker buildx build --load --platform linux/arm64 -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:latest "$BACKEND_PATH" + docker push --all-tags tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME + docker buildx build --load --platform linux/amd64 -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:latest "$FRONTEND_PATH" + docker buildx build --load --platform linux/arm64 -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:latest "$FRONTEND_PATH" + docker push --all-tags tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME + docker logout +} + +echo "Building and Tagging images for $BACKEND_APP_NAME and $FRONTEND_APP_NAME" +build_and_tag diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index e2c396b64..1928a8be1 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.4" services: zaproxy: - image: softwaresecurityproject/zap-stable:2.14.0 + image: tdp-docker.dev.raftlabs.tech/dependencies/softwaresecurityproject/zap-stable:2.14.0 command: sleep 3600 depends_on: - web @@ -12,7 +12,7 @@ services: - ../scripts/zap-hook.py:/zap/scripts/zap-hook.py:ro postgres: - image: postgres:15.7 + image: tdp-docker.dev.raftlabs.tech/dependencies/postgres:15.7 environment: - PGDATA=/var/lib/postgresql/data/ - POSTGRES_DB=tdrs_test @@ -25,14 +25,14 @@ services: - postgres_data:/var/lib/postgresql/data/:rw clamav-rest: - image: rafttech/clamav-rest:0.103.2 + image: tdp-docker.dev.raftlabs.tech/dependencies/rafttech/clamav-rest:0.103.2 environment: - MAX_FILE_SIZE=200M ports: - "9000:9000" localstack: - image: localstack/localstack:0.13.3 + image: tdp-docker.dev.raftlabs.tech/dependencies/localstack/localstack:0.13.3 environment: - SERVICES=s3 - DATA_DIR=/tmp/localstack/data @@ -46,7 +46,7 @@ services: - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh kibana: - image: docker.elastic.co/kibana/kibana-oss:7.10.2 + image: tdp-docker.dev.raftlabs.tech/dependencies/docker.elastic.co/kibana/kibana-oss:7.10.2 ports: - 5601:5601 environment: @@ -61,7 +61,7 @@ services: - elastic elastic: - image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2 + image: tdp-docker.dev.raftlabs.tech/dependencies/docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2 environment: - discovery.type=single-node - logger.discovery.level=debug @@ -185,7 +185,7 @@ services: volumes: - .:/tdpapp - logs:/tmp - image: tdp + image: tdp-backend build: . command: > bash -c "./wait_for_services.sh && @@ -204,7 +204,7 @@ services: - elastic redis-server: - image: "redis:alpine" + image: tdp-docker.dev.raftlabs.tech/dependencies/redis:alpine command: redis-server /tdpapp/redis.conf ports: - "6379:6379" diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index c00541419..6fe5355e0 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -5,6 +5,7 @@ from io import StringIO from typing import Union +from django.conf import settings from django.contrib.admin.models import ADDITION, ContentType, LogEntry from django.core.files.base import File from django.db import models @@ -206,6 +207,10 @@ def submitted_by(self): """Return the author as a string for this data file.""" return self.user.get_full_name() + def admin_link(self): + """Return a link to the admin console for this file.""" + return f"{settings.FRONTEND_BASE_URL}/admin/data_files/datafile/?id={self.pk}" + @classmethod def create_new_version(self, data): """Create a new version of a data file with an incremented version.""" diff --git a/tdrs-backend/tdpservice/data_files/tasks.py b/tdrs-backend/tdpservice/data_files/tasks.py new file mode 100644 index 000000000..16e35de79 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/tasks.py @@ -0,0 +1,48 @@ +"""Celery shared tasks for use in scheduled jobs.""" + +from celery import shared_task +from datetime import timedelta +from django.utils import timezone +from django.contrib.auth.models import Group +from django.db.models import Q, Count +from tdpservice.users.models import AccountApprovalStatusChoices, User +from tdpservice.data_files.models import DataFile +from tdpservice.parsers.models import DataFileSummary +from tdpservice.email.helpers.data_file import send_stuck_file_email + + +def get_stuck_files(): + """Return a queryset containing files in a 'stuck' state.""" + stuck_files = DataFile.objects.annotate(reparse_count=Count('reparse_meta_models')).filter( + # non-reparse submissions over an hour old + Q( + reparse_count=0, + created_at__lte=timezone.now() - timedelta(hours=1), + ) | # OR + # reparse submissions past the timeout, where the reparse did not complete + Q( + reparse_count__gt=0, + reparse_meta_models__timeout_at__lte=timezone.now(), + reparse_meta_models__finished=False, + reparse_meta_models__success=False + ) + ).filter( + # where there is NO summary or the summary is in PENDING status + Q(summary=None) | Q(summary__status=DataFileSummary.Status.PENDING) + ) + + return stuck_files + + +@shared_task +def notify_stuck_files(): + """Find files stuck in 'Pending' and notify SysAdmins.""" + stuck_files = get_stuck_files() + + if stuck_files.count() > 0: + recipients = User.objects.filter( + account_approval_status=AccountApprovalStatusChoices.APPROVED, + groups=Group.objects.get(name='OFA System Admin') + ).values_list('username', flat=True).distinct() + + send_stuck_file_email(stuck_files, recipients) diff --git a/tdrs-backend/tdpservice/data_files/test/test_stuck_files.py b/tdrs-backend/tdpservice/data_files/test/test_stuck_files.py new file mode 100644 index 000000000..95f4f8f3a --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/test/test_stuck_files.py @@ -0,0 +1,252 @@ +"""Test the get_stuck_files function.""" + + +import pytest +from datetime import timedelta +from django.utils import timezone +from tdpservice.data_files.models import DataFile +from tdpservice.parsers.models import DataFileSummary +from tdpservice.data_files.tasks import get_stuck_files +from tdpservice.parsers.test.factories import ParsingFileFactory, DataFileSummaryFactory, ReparseMetaFactory + + +def _time_ago(hours=0, minutes=0, seconds=0): + return timezone.now() - timedelta(hours=hours, minutes=minutes, seconds=seconds) + + +def make_datafile(stt_user, stt, version): + """Create a test data file with default params.""" + datafile = ParsingFileFactory.create( + quarter=DataFile.Quarter.Q1, section=DataFile.Section.ACTIVE_CASE_DATA, + year=2023, version=version, user=stt_user, stt=stt + ) + return datafile + + +def make_summary(datafile, status): + """Create a test data file summary given a file and status.""" + return DataFileSummaryFactory.create( + datafile=datafile, + status=status, + ) + + +def make_reparse_meta(finished, success): + """Create a test reparse meta model.""" + return ReparseMetaFactory.create( + timeout_at=_time_ago(hours=1), + finished=finished, + success=success + ) + + +@pytest.mark.django_db +def test_find_pending_submissions__none_stuck(stt_user, stt): + """Finds no stuck files.""" + # an accepted standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.ACCEPTED) + + # an accepted reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.ACCEPTED) + rpm = make_reparse_meta(True, True) + df2.reparse_meta_models.add(rpm) + + # a pending standard submission, less than an hour old + df3 = make_datafile(stt_user, stt, 3) + df3.created_at = _time_ago(minutes=40) + df3.save() + make_summary(df3, DataFileSummary.Status.PENDING) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 0 + + +@pytest.mark.django_db +def test_find_pending_submissions__non_reparse_stuck(stt_user, stt): + """Finds standard upload/submission stuck in Pending.""" + # a pending standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.PENDING) + + # an accepted reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.ACCEPTED) + rpm = make_reparse_meta(True, True) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df1.pk + + +@pytest.mark.django_db +def test_find_pending_submissions__non_reparse_stuck__no_dfs(stt_user, stt): + """Finds standard upload/submission stuck in Pending.""" + # a standard submission with no summary + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + + # an accepted reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.ACCEPTED) + rpm = make_reparse_meta(True, True) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df1.pk + + +@pytest.mark.django_db +def test_find_pending_submissions__reparse_stuck(stt_user, stt): + """Finds a reparse submission stuck in pending, past the timeout.""" + # an accepted standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.ACCEPTED) + + # a pending reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.PENDING) + rpm = make_reparse_meta(False, False) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df2.pk + + +@pytest.mark.django_db +def test_find_pending_submissions__reparse_stuck__no_dfs(stt_user, stt): + """Finds a reparse submission stuck in pending, past the timeout.""" + # an accepted standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.ACCEPTED) + + # a reparse submission with no summary, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + rpm = make_reparse_meta(False, False) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df2.pk + + +@pytest.mark.django_db +def test_find_pending_submissions__reparse_and_non_reparse_stuck(stt_user, stt): + """Finds stuck submissions, both reparse and standard parse.""" + # a pending standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + make_summary(df1, DataFileSummary.Status.PENDING) + + # a pending reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + make_summary(df2, DataFileSummary.Status.PENDING) + rpm = make_reparse_meta(False, False) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 2 + for f in stuck_files: + assert f.pk in (df1.pk, df2.pk) + + +@pytest.mark.django_db +def test_find_pending_submissions__reparse_and_non_reparse_stuck_no_dfs(stt_user, stt): + """Finds stuck submissions, both reparse and standard parse.""" + # a pending standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + + # a pending reparse submission, past the timeout + df2 = make_datafile(stt_user, stt, 2) + df2.created_at = _time_ago(hours=1) + df2.save() + rpm = make_reparse_meta(False, False) + df2.reparse_meta_models.add(rpm) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 2 + for f in stuck_files: + assert f.pk in (df1.pk, df2.pk) + + +@pytest.mark.django_db +def test_find_pending_submissions__old_reparse_stuck__new_not_stuck(stt_user, stt): + """Finds no stuck files, as the new parse is successful.""" + # a pending standard submission, more than an hour old + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + dfs1 = make_summary(df1, DataFileSummary.Status.PENDING) + + # reparse fails the first time + rpm1 = make_reparse_meta(False, False) + df1.reparse_meta_models.add(rpm1) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + + # reparse again, succeeds this time + dfs1.delete() # reparse deletes the original dfs and creates the new one + make_summary(df1, DataFileSummary.Status.ACCEPTED) + + rpm2 = make_reparse_meta(True, True) + df1.reparse_meta_models.add(rpm2) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 0 + + +@pytest.mark.django_db +def test_find_pending_submissions__new_reparse_stuck__old_not_stuck(stt_user, stt): + """Finds files stuck from the new reparse, even though the old one was successful.""" + # file rejected on first upload + df1 = make_datafile(stt_user, stt, 1) + df1.created_at = _time_ago(hours=2) + df1.save() + dfs1 = make_summary(df1, DataFileSummary.Status.REJECTED) + + # reparse succeeds + rpm1 = make_reparse_meta(True, True) + df1.reparse_meta_models.add(rpm1) + + # reparse again, fails this time + dfs1.delete() # reparse deletes the original dfs and creates the new one + DataFileSummary.objects.create( + datafile=df1, + status=DataFileSummary.Status.PENDING, + ) + + rpm2 = make_reparse_meta(False, False) + df1.reparse_meta_models.add(rpm2) + + stuck_files = get_stuck_files() + assert stuck_files.count() == 1 + assert stuck_files.first().pk == df1.pk diff --git a/tdrs-backend/tdpservice/email/email_enums.py b/tdrs-backend/tdpservice/email/email_enums.py index 4527b6016..82e15e66d 100644 --- a/tdrs-backend/tdpservice/email/email_enums.py +++ b/tdrs-backend/tdpservice/email/email_enums.py @@ -15,3 +15,4 @@ class EmailType(Enum): ACCOUNT_DEACTIVATED = 'account-deactivated.html' ACCOUNT_DEACTIVATED_ADMIN = 'account-deactivated-admin.html' UPCOMING_SUBMISSION_DEADLINE = 'upcoming-submission-deadline.html' + STUCK_FILE_LIST = 'stuck-file-list.html' diff --git a/tdrs-backend/tdpservice/email/helpers/data_file.py b/tdrs-backend/tdpservice/email/helpers/data_file.py index 1ed966a87..3b9112b54 100644 --- a/tdrs-backend/tdpservice/email/helpers/data_file.py +++ b/tdrs-backend/tdpservice/email/helpers/data_file.py @@ -1,5 +1,6 @@ """Helper functions for sending data file submission emails.""" from django.conf import settings +from tdpservice.users.models import User from tdpservice.email.email_enums import EmailType from tdpservice.email.email import automated_email, log from tdpservice.parsers.util import get_prog_from_section @@ -69,3 +70,31 @@ def send_data_submitted_email( text_message=text_message, logger_context=logger_context ) + + +def send_stuck_file_email(stuck_files, recipients): + """Send an email to sys admins with details of files stuck in Pending.""" + logger_context = { + 'user_id': User.objects.get_or_create(username='system')[0].pk + } + + template_path = EmailType.STUCK_FILE_LIST.value + subject = 'List of submitted files with pending status after 1 hour' + text_message = 'The system has detected stuck files.' + + context = { + "subject": subject, + "url": settings.FRONTEND_BASE_URL, + "files": stuck_files, + } + + log(f'Emailing stuck files to SysAdmins: {list(recipients)}', logger_context=logger_context) + + automated_email( + email_path=template_path, + recipient_email=recipients, + subject=subject, + email_context=context, + text_message=text_message, + logger_context=logger_context + ) diff --git a/tdrs-backend/tdpservice/email/templates/stuck-file-list.html b/tdrs-backend/tdpservice/email/templates/stuck-file-list.html new file mode 100644 index 000000000..bfe5055a2 --- /dev/null +++ b/tdrs-backend/tdpservice/email/templates/stuck-file-list.html @@ -0,0 +1,40 @@ +{% extends 'base.html' %} +{% block content %} + + + + +

+

+ +

Hello,

+ +

The system has detected stuck data submissions.

+ + + + + + + + + + + + + {% for file in files %} + + + + + + + + {% endfor %} + +
SttSectionFiscal yearSubmitted onFile
{{ file.stt }}{{ file.section }}{{ file.fiscal_year }}{{ file.created_at }} {{ file.created_time_ago }} + View in Admin Console +
+ +{% endblock %} \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 5b558ef3f..c0f50e85b 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -1,11 +1,29 @@ """Factories for generating test data for parsers.""" import factory +from django.utils import timezone from tdpservice.parsers.models import DataFileSummary, ParserErrorCategoryChoices from faker import Faker from tdpservice.data_files.test.factories import DataFileFactory from tdpservice.users.test.factories import UserFactory from tdpservice.stts.test.factories import STTFactory + +class ReparseMetaFactory(factory.django.DjangoModelFactory): + """Generate test reparse meta model.""" + + class Meta: + """Hardcoded meta data for factory.""" + + model = "search_indexes.ReparseMeta" + + timeout_at = timezone.now() + finished = False + success = False + num_files_to_reparse = 1 + files_completed = 1 + files_failed = 0 + + class ParsingFileFactory(factory.django.DjangoModelFactory): """Generate test data for data files.""" diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_reparse.py b/tdrs-backend/tdpservice/search_indexes/test/test_reparse.py index 360988224..2c8647cea 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_reparse.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_reparse.py @@ -273,8 +273,10 @@ def test_reparse_sequential(log_context): meta = ReparseMeta.objects.create(timeout_at=None) assert False is cmd._assert_sequential_execution(log_context) timeout_entry = LogEntry.objects.latest('pk') - assert timeout_entry.change_message == ("The latest ReparseMeta model's (ID: 1) timeout_at field is None. Cannot " - "safely execute reparse, please fix manually.") + assert timeout_entry.change_message == ( + f"The latest ReparseMeta model's (ID: {meta.pk}) timeout_at field is None. Cannot " + "safely execute reparse, please fix manually." + ) meta.timeout_at = timezone.now() + timedelta(seconds=100) meta.save() diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index b67aca9ad..6ae5eefb4 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -499,6 +499,10 @@ class Common(Configuration): 'task': 'tdpservice.email.tasks.email_admin_num_access_requests', 'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST) }, + 'Email Admin Number of Stuck Files' : { + 'task': 'tdpservice.data_files.tasks.notify_stuck_files', + 'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST) + }, 'Email Data Analyst Q1 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', # Feb 9 at 1pm UTC (9am EST) diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml index 3ee327f3e..13094148b 100644 --- a/tdrs-frontend/docker-compose.yml +++ b/tdrs-frontend/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.4" services: zaproxy: - image: softwaresecurityproject/zap-stable:2.14.0 + image: tdp-docker.dev.raftlabs.tech/dependencies/softwaresecurityproject/zap-stable:2.14.0 container_name: zap-scan command: sleep 3600 ports: @@ -14,6 +14,7 @@ services: tdp-frontend: stdin_open: true # docker run -i tty: true # docker run -t + image: tdp-frontend build: context: . target: nginx