diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 3cf5f1030..c4ab36e2b 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -58,7 +58,6 @@ set_cf_envs() "LOGGING_LEVEL" "REDIS_URI" "JWT_KEY" - "STAGING_JWT_KEY" "SENDGRID_API_KEY" ) @@ -71,9 +70,13 @@ set_cf_envs() cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" $cf_cmd continue - elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then - sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') - cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" + elif [[ ("$CF_SPACE" = "tanf-staging") ]]; then + var_value=${!var_name} + staging_var="STAGING_$var_name" + if [[ "${!staging_var}" ]]; then + var_value=${!staging_var} + fi + cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${var_value}" else cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" fi diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile index 606b86b79..34ef5dd9b 100644 --- a/tdrs-backend/Dockerfile +++ b/tdrs-backend/Dockerfile @@ -21,9 +21,9 @@ RUN apt-get install -y gcc graphviz graphviz-dev libpq-dev python3-dev vim curl # Postgres client setup #RUN bash -c 'echo "deb [trusted=yes] https://tdp-nexus.dev.raftlabs.tech/repository/apt-proxy-postgres/ bullseye-pdpg main" >> /etc/apt/sources.list' RUN apt-get update -y && apt-get upgrade -y -RUN apt install -y postgresql-common && install -d /usr/share/postgresql-common/pgdg && \ -sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc trusted=yes] https://tdp-nexus.dev.raftlabs.tech/repository/apt-proxy-postgres/ bullseye-pgdg main" >> /etc/apt/sources.list' && \ +RUN apt --purge remove postgresql postgresql-* && apt install -y postgresql-common curl ca-certificates && install -d /usr/share/postgresql-common/pgdg && \ curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \ +sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \ apt -y update && apt -y upgrade && apt install postgresql-client-15 -y # Install pipenv diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index f0ab9eac2..a279d3d5a 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -3,7 +3,7 @@ from rest_framework import serializers from tdpservice.parsers.models import ParserError from tdpservice.data_files.errors import ImmutabilityError -from tdpservice.data_files.models import DataFile +from tdpservice.data_files.models import DataFile, ReparseFileMeta from tdpservice.data_files.validators import ( validate_file_extension, validate_file_infection, @@ -12,8 +12,26 @@ from tdpservice.stts.models import STT from tdpservice.users.models import User from tdpservice.parsers.serializers import DataFileSummarySerializer + + logger = logging.getLogger(__name__) + +class ReparseFileMetaSerializer(serializers.ModelSerializer): + """Serializer for ReparseFileMeta class.""" + + class Meta: + """Meta class.""" + + model = ReparseFileMeta + fields = [ + 'finished', + 'success', + 'started_at', + 'finished_at', + ] + + class DataFileSerializer(serializers.ModelSerializer): """Serializer for Data files.""" @@ -23,6 +41,7 @@ class DataFileSerializer(serializers.ModelSerializer): ssp = serializers.BooleanField(write_only=True) has_error = serializers.SerializerMethodField() summary = DataFileSummarySerializer(many=False, read_only=True) + latest_reparse_file_meta = serializers.SerializerMethodField() class Meta: """Metadata.""" @@ -46,7 +65,8 @@ class Meta: 's3_location', 's3_versioning_id', 'has_error', - 'summary' + 'summary', + 'latest_reparse_file_meta', ] read_only_fields = ("version",) @@ -56,6 +76,13 @@ def get_has_error(self, obj): parser_errors = ParserError.objects.filter(file=obj.id) return len(parser_errors) > 0 + def get_latest_reparse_file_meta(self, instance): + """Return related reparse_file_metas, ordered by finished_at decending.""" + reparse_file_metas = instance.reparse_file_metas.all().order_by('-finished_at') + if reparse_file_metas.count() > 0: + return ReparseFileMetaSerializer(reparse_file_metas.first(), many=False, read_only=True).data + return None + def create(self, validated_data): """Create a new entry with a new version number.""" ssp = validated_data.pop('ssp') diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index fdafea8ea..3216d366a 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -3,6 +3,7 @@ import logging import logging.handlers import os +from django.utils.dateparse import parse_datetime from distutils.util import strtobool from os.path import join from typing import Any, Optional diff --git a/tdrs-frontend/cypress/e2e/common-steps/common-steps.js b/tdrs-frontend/cypress/e2e/common-steps/common-steps.js index e6a735c33..fcc2e74f2 100644 --- a/tdrs-frontend/cypress/e2e/common-steps/common-steps.js +++ b/tdrs-frontend/cypress/e2e/common-steps/common-steps.js @@ -1,12 +1,15 @@ import { When } from '@badeball/cypress-cucumber-preprocessor' When('{string} visits the home page', (username) => { + cy.clearCookie('sessionid') + cy.clearCookie('csrftoken') cy.intercept('/v1/stts/alpha').as('getSttSearchList') cy.visit('/') cy.contains('Sign into TANF Data Portal', { timeout: 30000 }) }) When('The admin logs in', () => { + cy.visit('/') cy.adminLogin('cypress-admin@teamraft.com') }) @@ -14,25 +17,28 @@ When('{string} logs in', (username) => { cy.login(username) }) -When('The admin sets the approval status of {string} to {string}', (username, status) => { - cy.get('@cypressUser').then((cypressUser) => { - let body = { - username: username, - first_name: '', - last_name: '', - email: username, - stt: '6', - groups: '2', - account_approval_status: status, - access_requested_date_0: '0001-01-01', - access_requested_date_1: '00:00:00', - _save: 'Save', - } +When( + 'The admin sets the approval status of {string} to {string}', + (username, status) => { + cy.get('@cypressUser').then((cypressUser) => { + let body = { + username: username, + first_name: '', + last_name: '', + email: username, + stt: '6', + groups: '2', + account_approval_status: status, + access_requested_date_0: '0001-01-01', + access_requested_date_1: '00:00:00', + _save: 'Save', + } - cy.adminApiRequest( - 'POST', - `/users/user/${cypressUser.selector.id}/change/`, - body - ) - }) -}) + cy.adminApiRequest( + 'POST', + `/users/user/${cypressUser.selector.id}/change/`, + body + ) + }) + } +) diff --git a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx index 3ddfe7365..56b1b5ecc 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx @@ -3,8 +3,10 @@ import { useDispatch } from 'react-redux' import { SubmissionSummaryStatusIcon, formatDate, + hasReparsed, + getReprocessedDate, downloadFile, - downloadErrorReport, + getErrorReportStatus, } from './helpers' const MonthSubRow = ({ data }) => @@ -24,17 +26,13 @@ const MonthSubRow = ({ data }) => const CaseAggregatesRow = ({ file }) => { const dispatch = useDispatch() - const errorFileName = `${file.year}-${file.quarter}-${file.section}` return ( <>