diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 3cf5f1030..c4ab36e2b 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -58,7 +58,6 @@ set_cf_envs() "LOGGING_LEVEL" "REDIS_URI" "JWT_KEY" - "STAGING_JWT_KEY" "SENDGRID_API_KEY" ) @@ -71,9 +70,13 @@ set_cf_envs() cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" $cf_cmd continue - elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then - sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') - cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" + elif [[ ("$CF_SPACE" = "tanf-staging") ]]; then + var_value=${!var_name} + staging_var="STAGING_$var_name" + if [[ "${!staging_var}" ]]; then + var_value=${!staging_var} + fi + cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${var_value}" else cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" fi diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile index 606b86b79..34ef5dd9b 100644 --- a/tdrs-backend/Dockerfile +++ b/tdrs-backend/Dockerfile @@ -21,9 +21,9 @@ RUN apt-get install -y gcc graphviz graphviz-dev libpq-dev python3-dev vim curl # Postgres client setup #RUN bash -c 'echo "deb [trusted=yes] https://tdp-nexus.dev.raftlabs.tech/repository/apt-proxy-postgres/ bullseye-pdpg main" >> /etc/apt/sources.list' RUN apt-get update -y && apt-get upgrade -y -RUN apt install -y postgresql-common && install -d /usr/share/postgresql-common/pgdg && \ -sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc trusted=yes] https://tdp-nexus.dev.raftlabs.tech/repository/apt-proxy-postgres/ bullseye-pgdg main" >> /etc/apt/sources.list' && \ +RUN apt --purge remove postgresql postgresql-* && apt install -y postgresql-common curl ca-certificates && install -d /usr/share/postgresql-common/pgdg && \ curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \ +sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \ apt -y update && apt -y upgrade && apt install postgresql-client-15 -y # Install pipenv diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index f0ab9eac2..a279d3d5a 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -3,7 +3,7 @@ from rest_framework import serializers from tdpservice.parsers.models import ParserError from tdpservice.data_files.errors import ImmutabilityError -from tdpservice.data_files.models import DataFile +from tdpservice.data_files.models import DataFile, ReparseFileMeta from tdpservice.data_files.validators import ( validate_file_extension, validate_file_infection, @@ -12,8 +12,26 @@ from tdpservice.stts.models import STT from tdpservice.users.models import User from tdpservice.parsers.serializers import DataFileSummarySerializer + + logger = logging.getLogger(__name__) + +class ReparseFileMetaSerializer(serializers.ModelSerializer): + """Serializer for ReparseFileMeta class.""" + + class Meta: + """Meta class.""" + + model = ReparseFileMeta + fields = [ + 'finished', + 'success', + 'started_at', + 'finished_at', + ] + + class DataFileSerializer(serializers.ModelSerializer): """Serializer for Data files.""" @@ -23,6 +41,7 @@ class DataFileSerializer(serializers.ModelSerializer): ssp = serializers.BooleanField(write_only=True) has_error = serializers.SerializerMethodField() summary = DataFileSummarySerializer(many=False, read_only=True) + latest_reparse_file_meta = serializers.SerializerMethodField() class Meta: """Metadata.""" @@ -46,7 +65,8 @@ class Meta: 's3_location', 's3_versioning_id', 'has_error', - 'summary' + 'summary', + 'latest_reparse_file_meta', ] read_only_fields = ("version",) @@ -56,6 +76,13 @@ def get_has_error(self, obj): parser_errors = ParserError.objects.filter(file=obj.id) return len(parser_errors) > 0 + def get_latest_reparse_file_meta(self, instance): + """Return related reparse_file_metas, ordered by finished_at decending.""" + reparse_file_metas = instance.reparse_file_metas.all().order_by('-finished_at') + if reparse_file_metas.count() > 0: + return ReparseFileMetaSerializer(reparse_file_metas.first(), many=False, read_only=True).data + return None + def create(self, validated_data): """Create a new entry with a new version number.""" ssp = validated_data.pop('ssp') diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index fdafea8ea..3216d366a 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -3,6 +3,7 @@ import logging import logging.handlers import os +from django.utils.dateparse import parse_datetime from distutils.util import strtobool from os.path import join from typing import Any, Optional diff --git a/tdrs-frontend/cypress/e2e/common-steps/common-steps.js b/tdrs-frontend/cypress/e2e/common-steps/common-steps.js index e6a735c33..fcc2e74f2 100644 --- a/tdrs-frontend/cypress/e2e/common-steps/common-steps.js +++ b/tdrs-frontend/cypress/e2e/common-steps/common-steps.js @@ -1,12 +1,15 @@ import { When } from '@badeball/cypress-cucumber-preprocessor' When('{string} visits the home page', (username) => { + cy.clearCookie('sessionid') + cy.clearCookie('csrftoken') cy.intercept('/v1/stts/alpha').as('getSttSearchList') cy.visit('/') cy.contains('Sign into TANF Data Portal', { timeout: 30000 }) }) When('The admin logs in', () => { + cy.visit('/') cy.adminLogin('cypress-admin@teamraft.com') }) @@ -14,25 +17,28 @@ When('{string} logs in', (username) => { cy.login(username) }) -When('The admin sets the approval status of {string} to {string}', (username, status) => { - cy.get('@cypressUser').then((cypressUser) => { - let body = { - username: username, - first_name: '', - last_name: '', - email: username, - stt: '6', - groups: '2', - account_approval_status: status, - access_requested_date_0: '0001-01-01', - access_requested_date_1: '00:00:00', - _save: 'Save', - } +When( + 'The admin sets the approval status of {string} to {string}', + (username, status) => { + cy.get('@cypressUser').then((cypressUser) => { + let body = { + username: username, + first_name: '', + last_name: '', + email: username, + stt: '6', + groups: '2', + account_approval_status: status, + access_requested_date_0: '0001-01-01', + access_requested_date_1: '00:00:00', + _save: 'Save', + } - cy.adminApiRequest( - 'POST', - `/users/user/${cypressUser.selector.id}/change/`, - body - ) - }) -}) + cy.adminApiRequest( + 'POST', + `/users/user/${cypressUser.selector.id}/change/`, + body + ) + }) + } +) diff --git a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx index 3ddfe7365..56b1b5ecc 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx @@ -3,8 +3,10 @@ import { useDispatch } from 'react-redux' import { SubmissionSummaryStatusIcon, formatDate, + hasReparsed, + getReprocessedDate, downloadFile, - downloadErrorReport, + getErrorReportStatus, } from './helpers' const MonthSubRow = ({ data }) => @@ -24,17 +26,13 @@ const MonthSubRow = ({ data }) => const CaseAggregatesRow = ({ file }) => { const dispatch = useDispatch() - const errorFileName = `${file.year}-${file.quarter}-${file.section}` return ( <> - {formatDate(file.createdAt)} - - - - {file.submittedBy} + {formatDate(file.createdAt) + ' by ' + file.submittedBy} + {hasReparsed(file) && <>} @@ -64,22 +62,7 @@ const CaseAggregatesRow = ({ file }) => { - {file.summary && - file.summary.status && - file.summary.status !== 'Pending' ? ( - file.hasError > 0 ? ( - - ) : ( - 'No Errors' - ) - ) : ( - 'Pending' - )} + {getErrorReportStatus(file)} @@ -99,9 +82,6 @@ export const CaseAggregatesTable = ({ files }) => ( Submitted On - - Submitted By - File Name diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index 654339ed6..a1e28b7c0 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -1,5 +1,6 @@ import React from 'react' import PropTypes from 'prop-types' +import classNames from 'classnames' import { useDispatch, useSelector } from 'react-redux' import { fileUploadSections } from '../../reducers/reports' import Paginator from '../Paginator' diff --git a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx index 3f4ba24a4..2d05fba6c 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx @@ -3,8 +3,10 @@ import { useDispatch } from 'react-redux' import { SubmissionSummaryStatusIcon, formatDate, + hasReparsed, + getReprocessedDate, downloadFile, - downloadErrorReport, + getErrorReportStatus, } from './helpers' const MonthSubRow = ({ data }) => @@ -22,17 +24,13 @@ const MonthSubRow = ({ data }) => const TotalAggregatesRow = ({ file }) => { const dispatch = useDispatch() - const errorFileName = `${file.year}-${file.quarter}-${file.section}` return ( <> - {formatDate(file.createdAt)} - - - - {file.submittedBy} + {formatDate(file.createdAt) + ' by ' + file.submittedBy} + {hasReparsed(file) && <>} @@ -58,22 +56,7 @@ const TotalAggregatesRow = ({ file }) => { - {file.summary && - file.summary.status && - file.summary.status !== 'Pending' ? ( - file.hasError > 0 ? ( - - ) : ( - 'No Errors' - ) - ) : ( - 'Pending' - )} + {getErrorReportStatus(file)} @@ -93,9 +76,6 @@ export const TotalAggregatesTable = ({ files }) => ( Submitted On - - Submitted By - File Name diff --git a/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx b/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx index e0663636f..5bb71d0e4 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx @@ -26,6 +26,36 @@ export const downloadErrorReport = async (file, reportName) => { console.log(error) } } +export const hasReparsed = (f) => + f.latest_reparse_file_meta && + f.latest_reparse_file_meta.finished_at && + f.latest_reparse_file_meta.finished_at !== null + +export const getReprocessedDate = (f) => f.latest_reparse_file_meta.finished_at + +export const getErrorReportStatus = (file) => { + if ( + file.summary && + file.summary.status && + file.summary.status !== 'Pending' + ) { + const errorFileName = `${file.year}-${file.quarter}-${file.section}` + if (file.hasError) { + return ( + + ) + } else { + return 'No Errors' + } + } else { + return 'Pending' + } +} export const SubmissionSummaryStatusIcon = ({ status }) => { let icon = null diff --git a/tdrs-frontend/src/reducers/reports.js b/tdrs-frontend/src/reducers/reports.js index 8c085bd99..9de986716 100644 --- a/tdrs-frontend/src/reducers/reports.js +++ b/tdrs-frontend/src/reducers/reports.js @@ -68,6 +68,7 @@ export const serializeApiDataFile = (dataFile) => ({ submittedBy: dataFile.submitted_by, hasError: dataFile.has_error, summary: dataFile.summary, + latest_reparse_file_meta: dataFile.latest_reparse_file_meta, }) const initialState = {