Merge branch 'develop' into 3269-bug-reparse-action-confirmation-pop-up

raft-tech · Nov 18, 2024 · b49a0cd · b49a0cd
2 parents f4a1399 + 5500be8
commit b49a0cd
Show file tree

Hide file tree

Showing 9 changed files with 83 additions and 60 deletions.
diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh
@@ -58,7 +58,6 @@ set_cf_envs()
   "LOGGING_LEVEL"
   "REDIS_URI"
   "JWT_KEY"
-  "STAGING_JWT_KEY"
   "SENDGRID_API_KEY"
   )
 
@@ -71,9 +70,13 @@ set_cf_envs()
         cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}"
         $cf_cmd
         continue
-    elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then
-        sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g')
-        cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}"
+    elif [[ ("$CF_SPACE" = "tanf-staging") ]]; then
+        var_value=${!var_name}
+        staging_var="STAGING_$var_name"
+        if [[ "${!staging_var}" ]]; then
+          var_value=${!staging_var}
+        fi
+        cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${var_value}"
     else
       cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}"
     fi

diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile
@@ -21,9 +21,9 @@ RUN apt-get install -y gcc graphviz graphviz-dev libpq-dev python3-dev vim curl
 # Postgres client setup
 #RUN bash -c 'echo "deb [trusted=yes] https://tdp-nexus.dev.raftlabs.tech/repository/apt-proxy-postgres/ bullseye-pdpg main" >> /etc/apt/sources.list'
 RUN apt-get update -y && apt-get upgrade -y
-RUN apt install -y postgresql-common && install -d /usr/share/postgresql-common/pgdg && \
-sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc trusted=yes] https://tdp-nexus.dev.raftlabs.tech/repository/apt-proxy-postgres/ bullseye-pgdg main" >> /etc/apt/sources.list' && \
+RUN apt --purge remove postgresql postgresql-* && apt install -y postgresql-common curl ca-certificates && install -d /usr/share/postgresql-common/pgdg && \
 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \
+sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
 apt -y update && apt -y upgrade && apt install postgresql-client-15 -y
 
 # Install pipenv

diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py
@@ -3,7 +3,7 @@
 from rest_framework import serializers
 from tdpservice.parsers.models import ParserError
 from tdpservice.data_files.errors import ImmutabilityError
-from tdpservice.data_files.models import DataFile
+from tdpservice.data_files.models import DataFile, ReparseFileMeta
 from tdpservice.data_files.validators import (
     validate_file_extension,
     validate_file_infection,
@@ -12,8 +12,26 @@
 from tdpservice.stts.models import STT
 from tdpservice.users.models import User
 from tdpservice.parsers.serializers import DataFileSummarySerializer
+
+
 logger = logging.getLogger(__name__)
 
+
+class ReparseFileMetaSerializer(serializers.ModelSerializer):
+    """Serializer for ReparseFileMeta class."""
+
+    class Meta:
+        """Meta class."""
+
+        model = ReparseFileMeta
+        fields = [
+            'finished',
+            'success',
+            'started_at',
+            'finished_at',
+        ]
+
+
 class DataFileSerializer(serializers.ModelSerializer):
     """Serializer for Data files."""
 
@@ -23,6 +41,7 @@ class DataFileSerializer(serializers.ModelSerializer):
     ssp = serializers.BooleanField(write_only=True)
     has_error = serializers.SerializerMethodField()
     summary = DataFileSummarySerializer(many=False, read_only=True)
+    latest_reparse_file_meta = serializers.SerializerMethodField()
 
     class Meta:
         """Metadata."""
@@ -46,7 +65,8 @@ class Meta:
             's3_location',
             's3_versioning_id',
             'has_error',
-            'summary'
+            'summary',
+            'latest_reparse_file_meta',
         ]
 
         read_only_fields = ("version",)
@@ -56,6 +76,13 @@ def get_has_error(self, obj):
         parser_errors = ParserError.objects.filter(file=obj.id)
         return len(parser_errors) > 0
 
+    def get_latest_reparse_file_meta(self, instance):
+        """Return related reparse_file_metas, ordered by finished_at decending."""
+        reparse_file_metas = instance.reparse_file_metas.all().order_by('-finished_at')
+        if reparse_file_metas.count() > 0:
+            return ReparseFileMetaSerializer(reparse_file_metas.first(), many=False, read_only=True).data
+        return None
+
     def create(self, validated_data):
         """Create a new entry with a new version number."""
         ssp = validated_data.pop('ssp')

diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py
@@ -3,6 +3,7 @@
 import logging
 import logging.handlers
 import os
+from django.utils.dateparse import parse_datetime
 from distutils.util import strtobool
 from os.path import join
 from typing import Any, Optional

diff --git a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx
@@ -3,8 +3,10 @@ import { useDispatch } from 'react-redux'
 import {
   SubmissionSummaryStatusIcon,
   formatDate,
+  hasReparsed,
+  getReprocessedDate,
   downloadFile,
-  downloadErrorReport,
+  getErrorReportStatus,
 } from './helpers'
 
 const MonthSubRow = ({ data }) =>
@@ -24,17 +26,13 @@ const MonthSubRow = ({ data }) =>
 
 const CaseAggregatesRow = ({ file }) => {
   const dispatch = useDispatch()
-  const errorFileName = `${file.year}-${file.quarter}-${file.section}`
 
   return (
     <>
       <tr>
         <th scope="rowgroup" rowSpan={3}>
-          {formatDate(file.createdAt)}
-        </th>
-
-        <th scope="rowgroup" rowSpan={3}>
-          {file.submittedBy}
+          {formatDate(file.createdAt) + ' by ' + file.submittedBy}
+          {hasReparsed(file) && <></>}
         </th>
 
         <th scope="rowgroup" rowSpan={3}>
@@ -64,22 +62,7 @@ const CaseAggregatesRow = ({ file }) => {
         </th>
 
         <th scope="rowgroup" rowSpan={3}>
-          {file.summary &&
-          file.summary.status &&
-          file.summary.status !== 'Pending' ? (
-            file.hasError > 0 ? (
-              <button
-                className="section-download"
-                onClick={() => downloadErrorReport(file, errorFileName)}
-              >
-                {errorFileName}.xlsx
-              </button>
-            ) : (
-              'No Errors'
-            )
-          ) : (
-            'Pending'
-          )}
+          {getErrorReportStatus(file)}
         </th>
       </tr>
       <tr>
@@ -99,9 +82,6 @@ export const CaseAggregatesTable = ({ files }) => (
         <th scope="col" rowSpan={2}>
           Submitted On
         </th>
-        <th scope="col" rowSpan={2}>
-          Submitted By
-        </th>
         <th scope="col" rowSpan={2}>
           File Name
         </th>

diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx
@@ -1,5 +1,6 @@
 import React from 'react'
 import PropTypes from 'prop-types'
+import classNames from 'classnames'
 import { useDispatch, useSelector } from 'react-redux'
 import { fileUploadSections } from '../../reducers/reports'
 import Paginator from '../Paginator'

diff --git a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx
@@ -3,8 +3,10 @@ import { useDispatch } from 'react-redux'
 import {
   SubmissionSummaryStatusIcon,
   formatDate,
+  hasReparsed,
+  getReprocessedDate,
   downloadFile,
-  downloadErrorReport,
+  getErrorReportStatus,
 } from './helpers'
 
 const MonthSubRow = ({ data }) =>
@@ -22,17 +24,13 @@ const MonthSubRow = ({ data }) =>
 
 const TotalAggregatesRow = ({ file }) => {
   const dispatch = useDispatch()
-  const errorFileName = `${file.year}-${file.quarter}-${file.section}`
 
   return (
     <>
       <tr>
         <th scope="rowgroup" rowSpan={3}>
-          {formatDate(file.createdAt)}
-        </th>
-
-        <th scope="rowgroup" rowSpan={3}>
-          {file.submittedBy}
+          {formatDate(file.createdAt) + ' by ' + file.submittedBy}
+          {hasReparsed(file) && <></>}
         </th>
 
         <th scope="rowgroup" rowSpan={3}>
@@ -58,22 +56,7 @@ const TotalAggregatesRow = ({ file }) => {
         </th>
 
         <th scope="rowgroup" rowSpan={3}>
-          {file.summary &&
-          file.summary.status &&
-          file.summary.status !== 'Pending' ? (
-            file.hasError > 0 ? (
-              <button
-                className="section-download"
-                onClick={() => downloadErrorReport(file, errorFileName)}
-              >
-                {errorFileName}.xlsx
-              </button>
-            ) : (
-              'No Errors'
-            )
-          ) : (
-            'Pending'
-          )}
+          {getErrorReportStatus(file)}
         </th>
       </tr>
       <tr>
@@ -93,9 +76,6 @@ export const TotalAggregatesTable = ({ files }) => (
         <th scope="col" rowSpan={2}>
           Submitted On
         </th>
-        <th scope="col" rowSpan={2}>
-          Submitted By
-        </th>
         <th scope="col" rowSpan={2}>
           File Name
         </th>

diff --git a/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx b/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx
@@ -26,6 +26,36 @@ export const downloadErrorReport = async (file, reportName) => {
     console.log(error)
   }
 }
+export const hasReparsed = (f) =>
+  f.latest_reparse_file_meta &&
+  f.latest_reparse_file_meta.finished_at &&
+  f.latest_reparse_file_meta.finished_at !== null
+
+export const getReprocessedDate = (f) => f.latest_reparse_file_meta.finished_at
+
+export const getErrorReportStatus = (file) => {
+  if (
+    file.summary &&
+    file.summary.status &&
+    file.summary.status !== 'Pending'
+  ) {
+    const errorFileName = `${file.year}-${file.quarter}-${file.section}`
+    if (file.hasError) {
+      return (
+        <button
+          className="section-download"
+          onClick={() => downloadErrorReport(file, errorFileName)}
+        >
+          {errorFileName}.xlsx
+        </button>
+      )
+    } else {
+      return 'No Errors'
+    }
+  } else {
+    return 'Pending'
+  }
+}
 
 export const SubmissionSummaryStatusIcon = ({ status }) => {
   let icon = null

diff --git a/tdrs-frontend/src/reducers/reports.js b/tdrs-frontend/src/reducers/reports.js
@@ -68,6 +68,7 @@ export const serializeApiDataFile = (dataFile) => ({
   submittedBy: dataFile.submitted_by,
   hasError: dataFile.has_error,
   summary: dataFile.summary,
+  latest_reparse_file_meta: dataFile.latest_reparse_file_meta,
 })
 
 const initialState = {