From 58ddd8df6c3d1f01d43870405abbb8d1a677ee9c Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 5 Jan 2024 10:46:21 -0500 Subject: [PATCH 01/18] total_errors_by_month aggregator --- tdrs-backend/tdpservice/parsers/util.py | 70 ++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 2a497e48e..107c8ea51 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -42,7 +42,8 @@ def generate_parser_error(datafile, line_number, schema, error_category, error_m row_number=line_number, column_number=getattr(field, 'item', None), item_number=getattr(field, 'item', None), - field_name=getattr(field, 'name', None) if hasattr(field, 'name') else field, + field_name=getattr(field, 'name', None) if hasattr( + field, 'name') else field, rpt_month_year=getattr(record, 'RPT_MONTH_YEAR', None), case_number=getattr(record, 'CASE_NUMBER', None), error_message=error_message, @@ -50,7 +51,8 @@ def generate_parser_error(datafile, line_number, schema, error_category, error_m content_type=ContentType.objects.get_for_model( model=schema.model if schema else None ) if record and not isinstance(record, dict) else None, - object_id=getattr(record, 'id', None) if record and not isinstance(record, dict) else None, + object_id=getattr(record, 'id', None) if record and not isinstance( + record, dict) else None, fields_json=fields_json ) @@ -98,7 +100,8 @@ def parse_and_validate(self, line, generate_error): records = [] for schema in self.schemas: - record, is_valid, errors = schema.parse_and_validate(line, generate_error) + record, is_valid, errors = schema.parse_and_validate( + line, generate_error) records.append((record, is_valid, errors)) return records @@ -110,12 +113,14 @@ def update_encrypted_fields(self, is_encrypted): if type(field) == TransformField and "is_encrypted" in field.kwargs: field.kwargs['is_encrypted'] = is_encrypted + def contains_encrypted_indicator(line, encryption_field): """Determine if line contains encryption indicator.""" if encryption_field is not None: return encryption_field.parse_value(line) == "E" return False + def get_schema_options(program, section, query=None, model=None, model_name=None): """Centralized function to return the appropriate schema for a given program, section, and query. @@ -225,22 +230,27 @@ def get_schema_options(program, section, query=None, model=None, model_name=None text**: input string from the header/file ''' + def get_program_models(str_prog, str_section): """Return the models dict for a given program and section.""" return get_schema_options(program=str_prog, section=str_section, query='models') + def get_program_model(str_prog, str_section, str_model): """Return singular model for a given program, section, and name.""" return get_schema_options(program=str_prog, section=str_section, query='models', model_name=str_model) + def get_section_reference(str_prog, str_section): """Return the named section reference for a given program and section.""" return get_schema_options(program=str_prog, section=str_section, query='section') + def get_text_from_df(df): """Return the short-hand text for program, section for a given datafile.""" return get_schema_options("", section=df.section, query='text') + def get_prog_from_section(str_section): """Return the program type for a given section.""" # e.g., 'SSP Closed Case Data' @@ -254,11 +264,13 @@ def get_prog_from_section(str_section): # TODO: if given a datafile (section), we can reverse back to the program b/c the # section string has "tribal/ssp" in it, then process of elimination we have tanf + def get_schema(line, section, program_type): """Return the appropriate schema for the line.""" line_type = line[0:2] return get_schema_options(program_type, section, query='models', model_name=line_type) + def fiscal_to_calendar(year, fiscal_quarter): """Decrement the input quarter text by one.""" array = [1, 2, 3, 4] # wrapping around an array @@ -266,8 +278,11 @@ def fiscal_to_calendar(year, fiscal_quarter): if int_qtr == 1: year = year - 1 - ind_qtr = array.index(int_qtr) # get the index so we can easily wrap-around end of array - return year, "Q{}".format(array[ind_qtr - 1]) # return the previous quarter + # get the index so we can easily wrap-around end of array + ind_qtr = array.index(int_qtr) + # return the previous quarter + return year, "Q{}".format(array[ind_qtr - 1]) + def transform_to_months(quarter): """Return a list of months in a quarter.""" @@ -283,6 +298,7 @@ def transform_to_months(quarter): case _: raise ValueError("Invalid quarter value.") + def month_to_int(month): """Return the integer value of a month.""" return datetime.strptime(month, '%b').strftime('%m') @@ -291,7 +307,8 @@ def month_to_int(month): def case_aggregates_by_month(df, dfs_status): """Return case aggregates by month.""" section = str(df.section) # section -> text - program_type = get_prog_from_section(section) # section -> program_type -> text + program_type = get_prog_from_section( + section) # section -> program_type -> text # from datafile year/quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' calendar_year, calendar_qtr = fiscal_to_calendar(df.year, df.quarter) @@ -335,6 +352,45 @@ def case_aggregates_by_month(df, dfs_status): "accepted_without_errors": accepted, "accepted_with_errors": cases_with_errors}) - aggregate_data['rejected'] = ParserError.objects.filter(file=df).filter(case_number=None).count() + aggregate_data['rejected'] = ParserError.objects.filter( + file=df).filter(case_number=None).count() return aggregate_data + + +def total_errors_by_month(df, dfs_status): + """Return total errors for each month in the reporting period.""" + section = str(df.section) # section -> text + program_type = get_prog_from_section( + section) # section -> program_type -> text + + # from datafile year/quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' + calendar_year, calendar_qtr = fiscal_to_calendar(df.year, df.quarter) + month_list = transform_to_months(calendar_qtr) + + short_section = get_text_from_df(df)['section'] + schema_models_dict = get_program_models(program_type, short_section) + schema_models = [model for model in schema_models_dict.values()] + + total_errors_data = {"months": []} + + for month in month_list: + if dfs_status == "Rejected": + total_errors_data["months"].append( + {"month": month, "total_errors": "N/A"}) + continue + + month_int = month_to_int(month) + rpt_month_year = int(f"{calendar_year}{month_int}") + + for schema_model in schema_models: + if isinstance(schema_model, SchemaManager): + schema_model = schema_model.schemas[0] + + # records = schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year) + error_count = ParserError.objects.filter( + file=df, rpt_month_year=rpt_month_year).count() + total_errors_data["months"].append( + {"month": month, "total_errors": error_count}) + + return total_errors_data From b86e66fd301cff331db26c79548e9a6b7339a8e3 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 5 Jan 2024 10:46:35 -0500 Subject: [PATCH 02/18] impl total_errors aggregator --- tdrs-backend/tdpservice/scheduling/parser_task.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index be47703c5..7126b8b83 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -5,7 +5,7 @@ from tdpservice.data_files.models import DataFile from tdpservice.parsers.parse import parse_datafile from tdpservice.parsers.models import DataFileSummary -from tdpservice.parsers.util import case_aggregates_by_month +from tdpservice.parsers.util import case_aggregates_by_month, total_errors_by_month logger = logging.getLogger(__name__) @@ -27,6 +27,8 @@ def parse(data_file_id): if "Case Data" in data_file.section: dfs.case_aggregates = case_aggregates_by_month(data_file, dfs.status) + else: + dfs.case_aggregates = total_errors_by_month(data_file, dfs.status) dfs.save() From 15be2195bd9050019a321f97c39ec237b64d22c7 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 5 Jan 2024 10:47:02 -0500 Subject: [PATCH 03/18] submission history component structure --- .../SubmissionHistory/CaseAggregatesTable.jsx | 134 ++++++++++ .../SubmissionHistory/SubmissionHistory.jsx | 252 +----------------- .../components/SubmissionHistory/helpers.jsx | 61 +++++ 3 files changed, 201 insertions(+), 246 deletions(-) create mode 100644 tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx create mode 100644 tdrs-frontend/src/components/SubmissionHistory/helpers.jsx diff --git a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx new file mode 100644 index 000000000..19f5e55fa --- /dev/null +++ b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx @@ -0,0 +1,134 @@ +import React from 'react' +import { useDispatch } from 'react-redux' +import { + SubmissionSummaryStatusIcon, + formatDate, + downloadFile, + downloadErrorReport, +} from './helpers' + +const MonthSubRow = ({ data }) => + data ? ( + <> + {data.month} + {data.accepted_without_errors} + {data.accepted_with_errors} + + ) : ( + <> + - + N/A + N/A + + ) + +const CaseAggregatesRow = ({ file }) => { + const dispatch = useDispatch() + const errorFileName = `${file.year}-${file.quarter}-${file.section}` + + return ( + <> + + + {formatDate(file.createdAt)} + + + + {file.submittedBy} + + + + + + + + + + {file.summary?.case_aggregates?.rejected || 'N/A'} + + + + + + + {file.summary && file.summary.status + ? file.summary.status + : 'Pending'} + + + + {file.summary && + file.summary.status && + file.summary.status !== 'Pending' ? ( + file.hasError > 0 ? ( + + ) : ( + 'No Errors' + ) + ) : ( + 'Pending' + )} + + + + + + + + + + ) +} + +export const CaseAggregatesTable = ({ files }) => ( + <> + + + + Submitted On + + + Submitted By + + + File Name + + + Month + + + Cases Without Errors + + + Cases With Errors + + + Records Unable To Process + + + Status + + + Error Reports (In development) + + + + + {files.map((file) => ( + + ))} + + +) diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index 55df49364..8ac180d76 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -1,230 +1,13 @@ import React from 'react' -import axios from 'axios' import PropTypes from 'prop-types' import { useDispatch, useSelector } from 'react-redux' import { fileUploadSections } from '../../reducers/reports' -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome' -import { - faCheckCircle, - faExclamationCircle, - faXmarkCircle, - faClock, -} from '@fortawesome/free-solid-svg-icons' import Paginator from '../Paginator' import { getAvailableFileList, download } from '../../actions/reports' import { useEffect } from 'react' import { useState } from 'react' -import { getParseErrors } from '../../actions/createXLSReport' - -const formatDate = (dateStr) => new Date(dateStr).toLocaleString() - -const SubmissionSummaryStatusIcon = ({ status }) => { - let icon = null - let color = null - - switch (status) { - case 'Pending': - icon = faClock - color = '#005EA2' - break - case 'Accepted': - icon = faCheckCircle - color = '#40bb45' - break - case 'Partially Accepted with Errors': - icon = faExclamationCircle - color = '#ec4e11' - break - case 'Accepted with Errors': - icon = faExclamationCircle - color = '#ec4e11' - break - case 'Rejected': - icon = faXmarkCircle - color = '#bb0000' - break - default: - break - } - return ( - - ) -} - -const CaseAggregatesHeader = ({ section }) => - section === 1 || section === 2 ? ( - <> - - Month - - - Cases Without Errors - - - Cases With Errors - - - Records Unable To Process - - - ) : ( - <> - - Month - - - Total - - - Cases With Errors - - - ) - -const CaseAggregatesRow = ({ data, section }) => - section === 1 || section === 2 ? ( - data ? ( - <> - {data.month} - {data.accepted_without_errors} - {data.accepted_with_errors} - - ) : ( - <> - - - N/A - N/A - - ) - ) : data ? ( - <> - {data.month} - {data.total} - - ) : ( - <> - - - N/A - - ) - -const SubmissionHistoryRow = ({ file }) => { - const dispatch = useDispatch() - - const downloadFile = () => dispatch(download(file)) - const errorFileName = `${file.year}-${file.quarter}-${file.section}` - - const returned_errors = async () => { - try { - const promise = axios.get( - `${process.env.REACT_APP_BACKEND_URL}/parsing/parsing_errors/?file=${file.id}`, - { - responseType: 'json', - } - ) - const dataPromise = await promise.then((response) => response.data) - getParseErrors(dataPromise, errorFileName) - } catch (error) { - console.log(error) - } - } - - const section_index = (element) => file.section.includes(element) - - const section = fileUploadSections.findIndex(section_index) + 1 - - return ( - <> - - - {formatDate(file.createdAt)} - - - {file.submittedBy} - - - - - - - - {file.summary && - file.summary.case_aggregates && - file.summary.case_aggregates.months - ? file.summary.case_aggregates.rejected - : 'N/A'} - - - - - - - {file.summary && file.summary.status - ? file.summary.status - : 'Pending'} - - - - {file.summary && - file.summary.status && - file.summary.status !== 'Pending' ? ( - file.hasError > 0 ? ( - - ) : ( - 'No Errors' - ) - ) : ( - 'Pending' - )} - - - - - - - - - - - ) -} - -SubmissionHistoryRow.propTypes = { - file: PropTypes.object, -} +import { CaseAggregatesTable } from './CaseAggregatesTable' +import { TotalAggregatesTable } from './TotalAggregatesTable' const SectionSubmissionHistory = ({ section, label, files }) => { const pageSize = 5 @@ -235,6 +18,9 @@ const SectionSubmissionHistory = ({ section, label, files }) => { const pageStart = (resultsPage - 1) * pageSize const pageEnd = Math.min(files.length, pageStart + pageSize) + const TableComponent = + section === 1 || section === 2 ? CaseAggregatesTable : TotalAggregatesTable + return (
{ {files && files.length > 0 ? ( - <> - - - - - - - - - - - - {files.slice(pageStart, pageEnd).map((file) => ( - - ))} - - + ) : ( No data available. )} diff --git a/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx b/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx new file mode 100644 index 000000000..ff2aca7bb --- /dev/null +++ b/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx @@ -0,0 +1,61 @@ +import React from 'react' +import axios from 'axios' +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome' +import { + faCheckCircle, + faExclamationCircle, + faXmarkCircle, + faClock, +} from '@fortawesome/free-solid-svg-icons' +import { getAvailableFileList, download } from '../../actions/reports' +import { getParseErrors } from '../../actions/createXLSReport' + +export const formatDate = (dateStr) => new Date(dateStr).toLocaleString() +export const downloadFile = (dispatch, file) => dispatch(download(file)) +export const downloadErrorReport = async (file, reportName) => { + try { + const promise = axios.get( + `${process.env.REACT_APP_BACKEND_URL}/parsing/parsing_errors/?file=${file.id}`, + { + responseType: 'json', + } + ) + const dataPromise = await promise.then((response) => response.data) + getParseErrors(dataPromise, reportName) + } catch (error) { + console.log(error) + } +} + +export const SubmissionSummaryStatusIcon = ({ status }) => { + let icon = null + let color = null + + switch (status) { + case 'Pending': + icon = faClock + color = '#005EA2' + break + case 'Accepted': + icon = faCheckCircle + color = '#40bb45' + break + case 'Partially Accepted with Errors': + icon = faExclamationCircle + color = '#ec4e11' + break + case 'Accepted with Errors': + icon = faExclamationCircle + color = '#ec4e11' + break + case 'Rejected': + icon = faXmarkCircle + color = '#bb0000' + break + default: + break + } + return ( + + ) +} From 525c18c8fb8ddadc8c473f09cb7e1a7d4472ea1c Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 5 Jan 2024 10:47:09 -0500 Subject: [PATCH 04/18] total errors frontend impl --- .../TotalAggregatesTable.jsx | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx diff --git a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx new file mode 100644 index 000000000..514735972 --- /dev/null +++ b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx @@ -0,0 +1,126 @@ +import React from 'react' +import { useDispatch } from 'react-redux' +import { + SubmissionSummaryStatusIcon, + formatDate, + downloadFile, + downloadErrorReport, +} from './helpers' + +// const section_index = (element) => file.section.includes(element) + +// const section = fileUploadSections.findIndex(section_index) + 1 + +const MonthSubRow = ({ data }) => + data ? ( + <> + + + + ) : ( + <> + + + + ) + +const TotalAggregatesRow = ({ file }) => { + const dispatch = useDispatch() + const errorFileName = `${file.year}-${file.quarter}-${file.section}` + + return ( + <> + + + + + + + + + + + + + + + + + + + + + ) +} + +export const TotalAggregatesTable = ({ files }) => ( + <> + + + + + + + + + + + + + {files.map((file) => ( + + ))} + + +) From da3f2b04d534e22a41f1bcc4dedc1117a2ad0296 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 17 Jan 2024 13:35:07 -0500 Subject: [PATCH 05/18] add section 3/4 aggregates to tests --- .../tdpservice/parsers/test/test_parse.py | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 93357ae6d..5ed9d396d 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -835,10 +835,20 @@ def tanf_section3_file(stt_user, stt): @pytest.mark.django_db() -def test_parse_tanf_section3_file(tanf_section3_file): +def test_parse_tanf_section3_file(tanf_section3_file, dfs): """Test parsing TANF Section 3 submission.""" + dfs.datafile = tanf_section3_file + dfs.save() + parse.parse_datafile(tanf_section3_file) + dfs.status = dfs.get_status() + dfs.case_aggregates = util.total_errors_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {"months": [{"month": "Oct", "total_errors": 0}, {"month": "Nov", "total_errors": 0}, {"month": "Dec", "total_errors": 0}]} + + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + assert TANF_T6.objects.all().count() == 3 parser_errors = ParserError.objects.filter(file=tanf_section3_file) @@ -895,10 +905,20 @@ def tanf_section4_file(stt_user, stt): @pytest.mark.django_db() -def test_parse_tanf_section4_file(tanf_section4_file): +def test_parse_tanf_section4_file(tanf_section4_file, dfs): """Test parsing TANF Section 4 submission.""" + dfs.datafile = tanf_section4_file + dfs.save() + parse.parse_datafile(tanf_section4_file) + dfs.status = dfs.get_status() + dfs.case_aggregates = util.total_errors_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {"months": [{"month": "Oct", "total_errors": 0}, {"month": "Nov", "total_errors": 0}, {"month": "Dec", "total_errors": 0}]} + + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + assert TANF_T7.objects.all().count() == 18 parser_errors = ParserError.objects.filter(file=tanf_section4_file) @@ -924,12 +944,20 @@ def ssp_section4_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.NDM4.MS24', stt_user, stt, "SSP Stratum Data") @pytest.mark.django_db() -def test_parse_ssp_section4_file(ssp_section4_file): +def test_parse_ssp_section4_file(ssp_section4_file, dfs): """Test parsing SSP Section 4 submission.""" + + dfs.datafile = ssp_section4_file + dfs.save() parse.parse_datafile(ssp_section4_file) m7_objs = SSP_M7.objects.all().order_by('FAMILIES_MONTH') + dfs.status = dfs.get_status() + dfs.case_aggregates = util.total_errors_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {"months": [{"month": "Oct", "total_errors": 0}, {"month": "Nov", "total_errors": 0}, {"month": "Dec", "total_errors": 0}]} + assert m7_objs.count() == 12 first = m7_objs.first() @@ -942,10 +970,28 @@ def ssp_section2_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.NDM2.MS24', stt_user, stt, 'SSP Closed Case Data') @pytest.mark.django_db() -def test_parse_ssp_section2_file(ssp_section2_file): +def test_parse_ssp_section2_file(ssp_section2_file, dfs): """Test parsing SSP Section 2 submission.""" + dfs.datafile = ssp_section2_file + dfs.save() + parse.parse_datafile(ssp_section2_file) + dfs.status = dfs.get_status() + dfs.case_aggregates = util.case_aggregates_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {'rejected': 0, + 'months': [ + {'accepted_without_errors': 0, + 'accepted_with_errors': 58, 'month': 'Oct'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 52, 'month': 'Nov'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 40, 'month': 'Dec'} + ]} + + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + m4_objs = SSP_M4.objects.all().order_by('id') m5_objs = SSP_M5.objects.all().order_by('AMOUNT_EARNED_INCOME') @@ -967,10 +1013,20 @@ def ssp_section3_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.NDM3.MS24', stt_user, stt, "SSP Aggregate Data") @pytest.mark.django_db() -def test_parse_ssp_section3_file(ssp_section3_file): +def test_parse_ssp_section3_file(ssp_section3_file, dfs): """Test parsing TANF Section 3 submission.""" + dfs.datafile = ssp_section3_file + dfs.save() + parse.parse_datafile(ssp_section3_file) + dfs.status = dfs.get_status() + dfs.case_aggregates = util.total_errors_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {"months": [{"month": "Oct", "total_errors": 0}, {"month": "Nov", "total_errors": 0}, {"month": "Dec", "total_errors": 0}]} + + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + m6_objs = SSP_M6.objects.all().order_by('RPT_MONTH_YEAR') assert m6_objs.count() == 3 From 6b6410c850011cca14ca4638f3bc206880573e5a Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 19 Jan 2024 10:01:49 -0500 Subject: [PATCH 06/18] lint --- .../tdpservice/parsers/test/test_parse.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 50a8f47e0..459bb79ed 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -897,7 +897,11 @@ def test_parse_tanf_section3_file(tanf_section3_file, dfs): dfs.status = dfs.get_status() dfs.case_aggregates = util.total_errors_by_month( dfs.datafile, dfs.status) - assert dfs.case_aggregates == {"months": [{"month": "Oct", "total_errors": 0}, {"month": "Nov", "total_errors": 0}, {"month": "Dec", "total_errors": 0}]} + assert dfs.case_aggregates == {"months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0} + ]} assert dfs.get_status() == DataFileSummary.Status.ACCEPTED @@ -967,7 +971,11 @@ def test_parse_tanf_section4_file(tanf_section4_file, dfs): dfs.status = dfs.get_status() dfs.case_aggregates = util.total_errors_by_month( dfs.datafile, dfs.status) - assert dfs.case_aggregates == {"months": [{"month": "Oct", "total_errors": 0}, {"month": "Nov", "total_errors": 0}, {"month": "Dec", "total_errors": 0}]} + assert dfs.case_aggregates == {"months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0} + ]} assert dfs.get_status() == DataFileSummary.Status.ACCEPTED @@ -998,7 +1006,6 @@ def ssp_section4_file(stt_user, stt): @pytest.mark.django_db() def test_parse_ssp_section4_file(ssp_section4_file, dfs): """Test parsing SSP Section 4 submission.""" - dfs.datafile = ssp_section4_file dfs.save() parse.parse_datafile(ssp_section4_file) @@ -1008,7 +1015,11 @@ def test_parse_ssp_section4_file(ssp_section4_file, dfs): dfs.status = dfs.get_status() dfs.case_aggregates = util.total_errors_by_month( dfs.datafile, dfs.status) - assert dfs.case_aggregates == {"months": [{"month": "Oct", "total_errors": 0}, {"month": "Nov", "total_errors": 0}, {"month": "Dec", "total_errors": 0}]} + assert dfs.case_aggregates == {"months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0} + ]} assert m7_objs.count() == 12 @@ -1092,7 +1103,11 @@ def test_parse_ssp_section3_file(ssp_section3_file, dfs): dfs.status = dfs.get_status() dfs.case_aggregates = util.total_errors_by_month( dfs.datafile, dfs.status) - assert dfs.case_aggregates == {"months": [{"month": "Oct", "total_errors": 0}, {"month": "Nov", "total_errors": 0}, {"month": "Dec", "total_errors": 0}]} + assert dfs.case_aggregates == {"months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0} + ]} assert dfs.get_status() == DataFileSummary.Status.ACCEPTED From 6d4f28200c274b49fa2ff532b999026dd53def1a Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 19 Jan 2024 11:05:22 -0500 Subject: [PATCH 07/18] fix null coalescing --- .../components/SubmissionHistory/CaseAggregatesTable.jsx | 6 +++--- .../components/SubmissionHistory/TotalAggregatesTable.jsx | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx index 19f5e55fa..9800206c6 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx @@ -46,7 +46,7 @@ const CaseAggregatesRow = ({ file }) => { - + - + - + ) diff --git a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx index 514735972..1019313a3 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx @@ -48,7 +48,7 @@ const TotalAggregatesRow = ({ file }) => { - + - + - + ) From dd039b18d8dfc2857f6409a355d0c7d464e50d12 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 19 Jan 2024 11:05:30 -0500 Subject: [PATCH 08/18] fix+add aggregates tests --- .../SubmissionHistory.test.js | 167 ++++++++++++------ 1 file changed, 115 insertions(+), 52 deletions(-) diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js index abf370523..06baeb730 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js @@ -341,76 +341,139 @@ describe('SubmissionHistory', () => { it.each([ ['Pending', 'Active Case Data'], ['Pending', 'Closed Case Data'], - ['Pending', 'Aggregate Data'], ['Accepted', 'Active Case Data'], ['Accepted', 'Closed Case Data'], - ['Accepted', 'Aggregate Data'], ['Accepted with Errors', 'Active Case Data'], ['Accepted with Errors', 'Closed Case Data'], - ['Accepted with Errors', 'Aggregate Data'], ['Partially Accepted with Errors', 'Active Case Data'], ['Partially Accepted with Errors', 'Closed Case Data'], - ['Partially Accepted with Errors', 'Aggregate Data'], ['Rejected', 'Active Case Data'], ['Rejected', 'Closed Case Data'], - ['Rejected', 'Aggregate Data'], [null, 'Active Case Data'], [null, 'Closed Case Data'], - [null, 'Aggregate Data'], - ])('Shows the submission acceptance status section 3', (status, section) => { - const state = { - reports: { - files: [ - { - id: '123', - fileName: 'test1.txt', - fileType: 'TANF', - quarter: 'Q1', - section: section, - uuid: '123-4-4-321', - year: '2023', - s3_version_id: '321-0-0-123', - createdAt: '12/12/2012 12:12', - submittedBy: 'test@teamraft.com', - summary: { - datafile: '123', - status: status, - case_aggregates: { - Oct: { - total: 0, - accepted: 0, - rejected: 0, - }, - Nov: { - total: 0, - accepted: 0, - rejected: 0, - }, - Dec: { - total: 0, - accepted: 0, + ])( + 'Shows the submission acceptance status section 1+2', + (status, section) => { + const state = { + reports: { + files: [ + { + id: '123', + fileName: 'test1.txt', + fileType: 'TANF', + quarter: 'Q1', + section: section, + uuid: '123-4-4-321', + year: '2023', + s3_version_id: '321-0-0-123', + createdAt: '12/12/2012 12:12', + submittedBy: 'test@teamraft.com', + summary: { + datafile: '123', + status: status, + case_aggregates: { + months: [ + { + accepted_without_errors: 0, + accepted_with_errors: 58, + month: 'Oct', + }, + { + accepted_without_errors: 0, + accepted_with_errors: 52, + month: 'Nov', + }, + { + accepted_without_errors: 0, + accepted_with_errors: 40, + month: 'Dec', + }, + ], rejected: 0, }, }, }, - }, - ], - }, + ], + }, + } + + const store = appConfigureStore(state) + const dispatch = jest.fn(store.dispatch) + store.dispatch = dispatch + + setup(store) + + expect(screen.queryByText('Status')).toBeInTheDocument() + expect(screen.queryByText('test1.txt')).toBeInTheDocument() + + if (status && status !== 'Pending') { + expect(screen.queryByText(status)).toBeInTheDocument() + } else { + expect(screen.queryAllByText('Pending')).toHaveLength(2) + } } + ) - const store = appConfigureStore(state) - const dispatch = jest.fn(store.dispatch) - store.dispatch = dispatch + it.each([ + ['Pending', 'Aggregate Data'], + ['Pending', 'Stratum Data'], + ['Accepted', 'Aggregate Data'], + ['Accepted', 'Stratum Data'], + ['Accepted with Errors', 'Aggregate Data'], + ['Accepted with Errors', 'Stratum Data'], + ['Partially Accepted with Errors', 'Aggregate Data'], + ['Partially Accepted with Errors', 'Stratum Data'], + ['Rejected', 'Aggregate Data'], + ['Rejected', 'Stratum Data'], + [null, 'Aggregate Data'], + [null, 'Stratum Data'], + ])( + 'Shows the submission acceptance status section 3+4', + (status, section) => { + const state = { + reports: { + files: [ + { + id: '123', + fileName: 'test1.txt', + fileType: 'TANF', + quarter: 'Q1', + section: section, + uuid: '123-4-4-321', + year: '2023', + s3_version_id: '321-0-0-123', + createdAt: '12/12/2012 12:12', + submittedBy: 'test@teamraft.com', + summary: { + datafile: '123', + status: status, + case_aggregates: { + months: [ + { month: 'Oct', total_errors: 0 }, + { month: 'Nov', total_errors: 0 }, + { month: 'Dec', total_errors: 0 }, + ], + }, + }, + }, + ], + }, + } - setup(store) + const store = appConfigureStore(state) + const dispatch = jest.fn(store.dispatch) + store.dispatch = dispatch - expect(screen.queryByText('Status')).toBeInTheDocument() - expect(screen.queryByText('test1.txt')).toBeInTheDocument() + setup(store) - if (status && status !== 'Pending') { - expect(screen.queryByText(status)).toBeInTheDocument() - } else { - expect(screen.queryAllByText('Pending')).toHaveLength(2) + expect(screen.queryByText('Status')).toBeInTheDocument() + expect(screen.queryByText('test1.txt')).toBeInTheDocument() + + if (status && status !== 'Pending') { + expect(screen.queryByText(status)).toBeInTheDocument() + } else { + expect(screen.queryAllByText('Pending')).toHaveLength(2) + } } - }) + ) }) From cd5f7331ae2f1b62be89a553c3eea7a8f23bfe94 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 23 Jan 2024 13:18:01 -0500 Subject: [PATCH 09/18] add tribal tests, positive case test --- .../test/data/tanf_section4_with_errors.txt | 3 + .../tdpservice/parsers/test/test_parse.py | 94 ++++++++++++++++++- 2 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt diff --git a/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt b/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt new file mode 100644 index 000000000..53590eccf --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt @@ -0,0 +1,3 @@ +HEADER20204S06 TAN1 N +T720204700006853700680540068454103000312400000000003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 459bb79ed..089091168 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1200,10 +1200,28 @@ def tribal_section_2_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.FTP2.TS142.txt', stt_user, stt, "Tribal Closed Case Data") @pytest.mark.django_db() -def test_parse_tribal_section_2_file(tribal_section_2_file): +def test_parse_tribal_section_2_file(tribal_section_2_file, dfs): """Test parsing Tribal TANF Section 2 submission.""" + dfs.datafile = tribal_section_2_file + dfs.save() + parse.parse_datafile(tribal_section_2_file) + dfs.status = dfs.get_status() + dfs.case_aggregates = util.case_aggregates_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {'rejected': 0, + 'months': [ + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Nov'}, + {'accepted_without_errors': 0, + 'accepted_with_errors': 0, 'month': 'Dec'} + ]} + + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert Tribal_TANF_T4.objects.all().count() == 6 assert Tribal_TANF_T5.objects.all().count() == 13 @@ -1222,10 +1240,24 @@ def tribal_section_3_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.FTP3.TS142', stt_user, stt, "Tribal Aggregate Data") @pytest.mark.django_db() -def test_parse_tribal_section_3_file(tribal_section_3_file): +def test_parse_tribal_section_3_file(tribal_section_3_file, dfs): """Test parsing Tribal TANF Section 3 submission.""" + dfs.datafile = tribal_section_3_file + dfs.save() + parse.parse_datafile(tribal_section_3_file) + dfs.status = dfs.get_status() + dfs.case_aggregates = util.total_errors_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {"months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0} + ]} + + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + assert Tribal_TANF_T6.objects.all().count() == 3 t6_objs = Tribal_TANF_T6.objects.all().order_by("NUM_APPLICATIONS") @@ -1242,10 +1274,22 @@ def tribal_section_4_file(stt_user, stt): return util.create_test_datafile('tribal_section_4_fake.txt', stt_user, stt, "Tribal Stratum Data") @pytest.mark.django_db() -def test_parse_tribal_section_4_file(tribal_section_4_file): +def test_parse_tribal_section_4_file(tribal_section_4_file, dfs): """Test parsing Tribal TANF Section 4 submission.""" + dfs.datafile = tribal_section_4_file + dfs.save() + parse.parse_datafile(tribal_section_4_file) + dfs.status = dfs.get_status() + dfs.case_aggregates = util.total_errors_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {"months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0} + ]} + assert Tribal_TANF_T7.objects.all().count() == 18 t7_objs = Tribal_TANF_T7.objects.all().order_by('FAMILIES_MONTH') @@ -1261,3 +1305,47 @@ def test_parse_tribal_section_4_file(tribal_section_4_file): assert first.FAMILIES_MONTH == 274 assert sixth.FAMILIES_MONTH == 499 + + +@pytest.fixture +def tanf_section_4_file_with_errors(stt_user, stt): + """Fixture for tanf_section4_with_errors.""" + return util.create_test_datafile('tanf_section4_with_errors.txt', stt_user, stt, "Stratum Data") + +@pytest.mark.django_db() +def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, dfs): + """Test parsing TANF Section 4 submission.""" + dfs.datafile = tanf_section_4_file_with_errors + dfs.save() + + parse.parse_datafile(tanf_section_4_file_with_errors) + + dfs.status = dfs.get_status() + dfs.case_aggregates = util.total_errors_by_month( + dfs.datafile, dfs.status) + assert dfs.case_aggregates == {"months": [ + {"month": "Oct", "total_errors": 2}, + {"month": "Nov", "total_errors": 2}, + {"month": "Dec", "total_errors": 2} + ]} + + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + + assert TANF_T7.objects.all().count() == 18 + + parser_errors = ParserError.objects.filter(file=tanf_section_4_file_with_errors) + assert parser_errors.count() == 6 + + t7_objs = TANF_T7.objects.all().order_by('FAMILIES_MONTH') + + first = t7_objs.first() + sixth = t7_objs[5] + + assert first.RPT_MONTH_YEAR == 202011 + assert sixth.RPT_MONTH_YEAR == 202010 + + assert first.TDRS_SECTION_IND == '1' + assert sixth.TDRS_SECTION_IND == '1' + + assert first.FAMILIES_MONTH == 0 + assert sixth.FAMILIES_MONTH == 446 From b1889aa15f405eb2c5a10118c500f26d20670cc7 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 23 Jan 2024 13:18:18 -0500 Subject: [PATCH 10/18] remove unused code from total_errors aggregate func --- tdrs-backend/tdpservice/parsers/util.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index d7a66c4a6..5e0b5bce8 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -381,18 +381,9 @@ def case_aggregates_by_month(df, dfs_status): def total_errors_by_month(df, dfs_status): """Return total errors for each month in the reporting period.""" - section = str(df.section) # section -> text - program_type = get_prog_from_section( - section) # section -> program_type -> text - - # from datafile year/quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' calendar_year, calendar_qtr = fiscal_to_calendar(df.year, df.quarter) month_list = transform_to_months(calendar_qtr) - short_section = get_text_from_df(df)['section'] - schema_models_dict = get_program_models(program_type, short_section) - schema_models = [model for model in schema_models_dict.values()] - total_errors_data = {"months": []} for month in month_list: @@ -404,14 +395,9 @@ def total_errors_by_month(df, dfs_status): month_int = month_to_int(month) rpt_month_year = int(f"{calendar_year}{month_int}") - for schema_model in schema_models: - if isinstance(schema_model, SchemaManager): - schema_model = schema_model.schemas[0] - - # records = schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year) - error_count = ParserError.objects.filter( - file=df, rpt_month_year=rpt_month_year).count() - total_errors_data["months"].append( - {"month": month, "total_errors": error_count}) + error_count = ParserError.objects.filter( + file=df, rpt_month_year=rpt_month_year).count() + total_errors_data["months"].append( + {"month": month, "total_errors": error_count}) return total_errors_data From 850bbd086d924097f03a79f8307e81f808dce096 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 23 Jan 2024 13:23:20 -0500 Subject: [PATCH 11/18] remove unnecessary saves --- .../tdpservice/parsers/test/test_parse.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 089091168..bccb2bfe7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -35,7 +35,6 @@ def dfs(): def test_parse_small_correct_file(test_datafile, dfs): """Test parsing of small_correct_file.""" dfs.datafile = test_datafile - dfs.save() parse.parse_datafile(test_datafile) dfs.status = dfs.get_status() @@ -75,7 +74,6 @@ def test_parse_section_mismatch(test_datafile, dfs): test_datafile.save() dfs.datafile = test_datafile - dfs.save() errors = parse.parse_datafile(test_datafile) dfs.status = dfs.get_status() @@ -116,7 +114,6 @@ def test_parse_wrong_program_type(test_datafile, dfs): test_datafile.save() dfs.datafile = test_datafile - dfs.save() errors = parse.parse_datafile(test_datafile) assert dfs.get_status() == DataFileSummary.Status.REJECTED @@ -149,7 +146,6 @@ def test_parse_big_file(test_big_file, dfs): expected_t3_record_count = 1376 dfs.datafile = test_big_file - dfs.save() parse.parse_datafile(test_big_file) dfs.status = dfs.get_status() @@ -226,7 +222,6 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): """Test parsing of bad_missing_header.""" errors = parse.parse_datafile(bad_file_missing_header) dfs.datafile = bad_file_missing_header - dfs.save() assert dfs.get_status() == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_missing_header).order_by('created_at') @@ -256,7 +251,6 @@ def test_parse_bad_file_multiple_headers(bad_file_multiple_headers, dfs): """Test parsing of bad_two_headers.""" errors = parse.parse_datafile(bad_file_multiple_headers) dfs.datafile = bad_file_multiple_headers - dfs.save() assert dfs.get_status() == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_multiple_headers) @@ -305,7 +299,6 @@ def bad_trailer_file(stt_user, stt): def test_parse_bad_trailer_file(bad_trailer_file, dfs): """Test parsing bad_trailer_1.""" dfs.datafile = bad_trailer_file - dfs.save() errors = parse.parse_datafile(bad_trailer_file) @@ -387,7 +380,6 @@ def empty_file(stt_user, stt): def test_parse_empty_file(empty_file, dfs): """Test parsing of empty_file.""" dfs.datafile = empty_file - dfs.save() errors = parse.parse_datafile(empty_file) dfs.status = dfs.get_status() @@ -441,7 +433,6 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): small_ssp_section1_datafile.save() dfs.datafile = small_ssp_section1_datafile - dfs.save() parse.parse_datafile(small_ssp_section1_datafile) @@ -504,7 +495,6 @@ def small_tanf_section1_datafile(stt_user, stt): def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): """Test parsing of small_tanf_section1_datafile and validate T2 model data.""" dfs.datafile = small_tanf_section1_datafile - dfs.save() parse.parse_datafile(small_tanf_section1_datafile) @@ -673,7 +663,6 @@ def bad_tanf_s1__row_missing_required_field(stt_user, stt): def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field, dfs): """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" dfs.datafile = bad_tanf_s1__row_missing_required_field - dfs.save() parse.parse_datafile(bad_tanf_s1__row_missing_required_field) @@ -772,11 +761,9 @@ def test_dfs_set_case_aggregates(test_datafile, dfs): # this still needs to execute to create db objects to be queried parse.parse_datafile(test_datafile) dfs.file = test_datafile - dfs.save() dfs.status = dfs.get_status() dfs.case_aggregates = util.case_aggregates_by_month( test_datafile, dfs.status) - dfs.save() for month in dfs.case_aggregates['months']: if month['month'] == 'Oct': @@ -890,7 +877,6 @@ def tanf_section3_file(stt_user, stt): def test_parse_tanf_section3_file(tanf_section3_file, dfs): """Test parsing TANF Section 3 submission.""" dfs.datafile = tanf_section3_file - dfs.save() parse.parse_datafile(tanf_section3_file) @@ -964,7 +950,6 @@ def tanf_section4_file(stt_user, stt): def test_parse_tanf_section4_file(tanf_section4_file, dfs): """Test parsing TANF Section 4 submission.""" dfs.datafile = tanf_section4_file - dfs.save() parse.parse_datafile(tanf_section4_file) @@ -1007,7 +992,6 @@ def ssp_section4_file(stt_user, stt): def test_parse_ssp_section4_file(ssp_section4_file, dfs): """Test parsing SSP Section 4 submission.""" dfs.datafile = ssp_section4_file - dfs.save() parse.parse_datafile(ssp_section4_file) m7_objs = SSP_M7.objects.all().order_by('FAMILIES_MONTH') @@ -1036,7 +1020,6 @@ def ssp_section2_file(stt_user, stt): def test_parse_ssp_section2_file(ssp_section2_file, dfs): """Test parsing SSP Section 2 submission.""" dfs.datafile = ssp_section2_file - dfs.save() parse.parse_datafile(ssp_section2_file) @@ -1096,7 +1079,6 @@ def ssp_section3_file(stt_user, stt): def test_parse_ssp_section3_file(ssp_section3_file, dfs): """Test parsing TANF Section 3 submission.""" dfs.datafile = ssp_section3_file - dfs.save() parse.parse_datafile(ssp_section3_file) @@ -1146,7 +1128,6 @@ def test_parse_tribal_section_1_file(tribal_section_1_file, dfs): tribal_section_1_file.save() dfs.datafile = tribal_section_1_file - dfs.save() parse.parse_datafile(tribal_section_1_file) @@ -1203,7 +1184,6 @@ def tribal_section_2_file(stt_user, stt): def test_parse_tribal_section_2_file(tribal_section_2_file, dfs): """Test parsing Tribal TANF Section 2 submission.""" dfs.datafile = tribal_section_2_file - dfs.save() parse.parse_datafile(tribal_section_2_file) @@ -1243,7 +1223,6 @@ def tribal_section_3_file(stt_user, stt): def test_parse_tribal_section_3_file(tribal_section_3_file, dfs): """Test parsing Tribal TANF Section 3 submission.""" dfs.datafile = tribal_section_3_file - dfs.save() parse.parse_datafile(tribal_section_3_file) @@ -1277,7 +1256,6 @@ def tribal_section_4_file(stt_user, stt): def test_parse_tribal_section_4_file(tribal_section_4_file, dfs): """Test parsing Tribal TANF Section 4 submission.""" dfs.datafile = tribal_section_4_file - dfs.save() parse.parse_datafile(tribal_section_4_file) @@ -1316,7 +1294,6 @@ def tanf_section_4_file_with_errors(stt_user, stt): def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, dfs): """Test parsing TANF Section 4 submission.""" dfs.datafile = tanf_section_4_file_with_errors - dfs.save() parse.parse_datafile(tanf_section_4_file_with_errors) From 1c2ce4f7db411f1c652f95b43ec6916ebd97b1d3 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 23 Jan 2024 13:42:01 -0500 Subject: [PATCH 12/18] move total_errors to aggregates.py --- tdrs-backend/tdpservice/parsers/aggregates.py | 24 ++++++++++++++++ .../tdpservice/parsers/test/test_parse.py | 28 +++++++++---------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/aggregates.py b/tdrs-backend/tdpservice/parsers/aggregates.py index 8f27a7445..a3ab85ded 100644 --- a/tdrs-backend/tdpservice/parsers/aggregates.py +++ b/tdrs-backend/tdpservice/parsers/aggregates.py @@ -55,3 +55,27 @@ def case_aggregates_by_month(df, dfs_status): aggregate_data['rejected'] = ParserError.objects.filter(file=df).filter(case_number=None).count() return aggregate_data + + +def total_errors_by_month(df, dfs_status): + """Return total errors for each month in the reporting period.""" + calendar_year, calendar_qtr = fiscal_to_calendar(df.year, df.quarter) + month_list = transform_to_months(calendar_qtr) + + total_errors_data = {"months": []} + + for month in month_list: + if dfs_status == "Rejected": + total_errors_data["months"].append( + {"month": month, "total_errors": "N/A"}) + continue + + month_int = month_to_int(month) + rpt_month_year = int(f"{calendar_year}{month_int}") + + error_count = ParserError.objects.filter( + file=df, rpt_month_year=rpt_month_year).count() + total_errors_data["months"].append( + {"month": month, "total_errors": error_count}) + + return total_errors_data \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 31fb47544..06a61ab68 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -931,7 +931,7 @@ def test_parse_tanf_section3_file(tanf_section3_file, dfs): parse.parse_datafile(tanf_section3_file) dfs.status = dfs.get_status() - dfs.case_aggregates = util.total_errors_by_month( + dfs.case_aggregates = aggregates.total_errors_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {"months": [ {"month": "Oct", "total_errors": 0}, @@ -1010,7 +1010,7 @@ def test_parse_tanf_section4_file(tanf_section4_file, dfs): parse.parse_datafile(tanf_section4_file) dfs.status = dfs.get_status() - dfs.case_aggregates = util.total_errors_by_month( + dfs.case_aggregates = aggregates.total_errors_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {"months": [ {"month": "Oct", "total_errors": 0}, @@ -1056,7 +1056,7 @@ def test_parse_ssp_section4_file(ssp_section4_file, dfs): m7_objs = SSP_M7.objects.all().order_by('FAMILIES_MONTH') dfs.status = dfs.get_status() - dfs.case_aggregates = util.total_errors_by_month( + dfs.case_aggregates = aggregates.total_errors_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {"months": [ {"month": "Oct", "total_errors": 0}, @@ -1086,16 +1086,16 @@ def test_parse_ssp_section2_file(ssp_section2_file, dfs): parse.parse_datafile(ssp_section2_file) dfs.status = dfs.get_status() - dfs.case_aggregates = util.case_aggregates_by_month( + dfs.case_aggregates = aggregates.case_aggregates_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 0, 'months': [ {'accepted_without_errors': 0, - 'accepted_with_errors': 58, 'month': 'Oct'}, + 'accepted_with_errors': 78, 'month': 'Oct'}, {'accepted_without_errors': 0, - 'accepted_with_errors': 52, 'month': 'Nov'}, + 'accepted_with_errors': 78, 'month': 'Nov'}, {'accepted_without_errors': 0, - 'accepted_with_errors': 40, 'month': 'Dec'} + 'accepted_with_errors': 75, 'month': 'Dec'} ]} assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS @@ -1147,7 +1147,7 @@ def test_parse_ssp_section3_file(ssp_section3_file, dfs): parse.parse_datafile(ssp_section3_file) dfs.status = dfs.get_status() - dfs.case_aggregates = util.total_errors_by_month( + dfs.case_aggregates = aggregates.total_errors_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {"months": [ {"month": "Oct", "total_errors": 0}, @@ -1284,14 +1284,14 @@ def test_parse_tribal_section_2_file(tribal_section_2_file, dfs): parse.parse_datafile(tribal_section_2_file) dfs.status = dfs.get_status() - dfs.case_aggregates = util.case_aggregates_by_month( + dfs.case_aggregates = aggregates.case_aggregates_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 0, 'months': [ {'accepted_without_errors': 0, - 'accepted_with_errors': 0, 'month': 'Oct'}, + 'accepted_with_errors': 3, 'month': 'Oct'}, {'accepted_without_errors': 0, - 'accepted_with_errors': 0, 'month': 'Nov'}, + 'accepted_with_errors': 3, 'month': 'Nov'}, {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} ]} @@ -1326,7 +1326,7 @@ def test_parse_tribal_section_3_file(tribal_section_3_file, dfs): parse.parse_datafile(tribal_section_3_file) dfs.status = dfs.get_status() - dfs.case_aggregates = util.total_errors_by_month( + dfs.case_aggregates = aggregates.total_errors_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {"months": [ {"month": "Oct", "total_errors": 0}, @@ -1361,7 +1361,7 @@ def test_parse_tribal_section_4_file(tribal_section_4_file, dfs): parse.parse_datafile(tribal_section_4_file) dfs.status = dfs.get_status() - dfs.case_aggregates = util.total_errors_by_month( + dfs.case_aggregates = aggregates.total_errors_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {"months": [ {"month": "Oct", "total_errors": 0}, @@ -1399,7 +1399,7 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d parse.parse_datafile(tanf_section_4_file_with_errors) dfs.status = dfs.get_status() - dfs.case_aggregates = util.total_errors_by_month( + dfs.case_aggregates = aggregates.total_errors_by_month( dfs.datafile, dfs.status) assert dfs.case_aggregates == {"months": [ {"month": "Oct", "total_errors": 2}, From 7c3180c9a5102e7fe39cb8efa4c369cc68aca65e Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 23 Jan 2024 13:42:13 -0500 Subject: [PATCH 13/18] extra blank line --- tdrs-backend/tdpservice/parsers/aggregates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/aggregates.py b/tdrs-backend/tdpservice/parsers/aggregates.py index a3ab85ded..4757e3bd2 100644 --- a/tdrs-backend/tdpservice/parsers/aggregates.py +++ b/tdrs-backend/tdpservice/parsers/aggregates.py @@ -78,4 +78,4 @@ def total_errors_by_month(df, dfs_status): total_errors_data["months"].append( {"month": month, "total_errors": error_count}) - return total_errors_data \ No newline at end of file + return total_errors_data From 54e98e524c5f072e2f7aeefa0f73bc56e6bb7de2 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 23 Jan 2024 13:49:40 -0500 Subject: [PATCH 14/18] undo formatter changes --- tdrs-backend/tdpservice/parsers/util.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 0a5f80c76..cebcd56a3 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -41,8 +41,7 @@ def generate_parser_error(datafile, line_number, schema, error_category, error_m row_number=line_number, column_number=getattr(field, 'item', None), item_number=getattr(field, 'item', None), - field_name=getattr(field, 'name', None) if hasattr( - field, 'name') else field, + field_name=getattr(field, 'name', None) if hasattr(field, 'name') else field, rpt_month_year=getattr(record, 'RPT_MONTH_YEAR', None), case_number=getattr(record, 'CASE_NUMBER', None), error_message=error_message, @@ -50,8 +49,7 @@ def generate_parser_error(datafile, line_number, schema, error_category, error_m content_type=ContentType.objects.get_for_model( model=schema.document.Django.model if schema else None ) if record and not isinstance(record, dict) else None, - object_id=getattr(record, 'id', None) if record and not isinstance( - record, dict) else None, + object_id=getattr(record, 'id', None) if record and not isinstance(record, dict) else None, fields_json=fields_json ) @@ -232,27 +230,22 @@ def get_schema_options(program, section, query=None, model=None, model_name=None text**: input string from the header/file ''' - def get_program_models(str_prog, str_section): """Return the models dict for a given program and section.""" return get_schema_options(program=str_prog, section=str_section, query='models') - def get_program_model(str_prog, str_section, str_model): """Return singular model for a given program, section, and name.""" return get_schema_options(program=str_prog, section=str_section, query='models', model_name=str_model) - def get_section_reference(str_prog, str_section): """Return the named section reference for a given program and section.""" return get_schema_options(program=str_prog, section=str_section, query='section') - def get_text_from_df(df): """Return the short-hand text for program, section for a given datafile.""" return get_schema_options("", section=df.section, query='text') - def get_prog_from_section(str_section): """Return the program type for a given section.""" # e.g., 'SSP Closed Case Data' @@ -266,13 +259,11 @@ def get_prog_from_section(str_section): # TODO: if given a datafile (section), we can reverse back to the program b/c the # section string has "tribal/ssp" in it, then process of elimination we have tanf - def get_schema(line, section, program_type): """Return the appropriate schema for the line.""" line_type = line[0:2] return get_schema_options(program_type, section, query='models', model_name=line_type) - def fiscal_to_calendar(year, fiscal_quarter): """Decrement the input quarter text by one.""" array = [1, 2, 3, 4] # wrapping around an array @@ -280,10 +271,8 @@ def fiscal_to_calendar(year, fiscal_quarter): if int_qtr == 1: year = year - 1 - # get the index so we can easily wrap-around end of array - ind_qtr = array.index(int_qtr) - # return the previous quarter - return year, "Q{}".format(array[ind_qtr - 1]) + ind_qtr = array.index(int_qtr) # get the index so we can easily wrap-around end of array + return year, "Q{}".format(array[ind_qtr - 1]) # return the previous quarter def transform_to_months(quarter): @@ -300,7 +289,6 @@ def transform_to_months(quarter): case _: raise ValueError("Invalid quarter value.") - def month_to_int(month): """Return the integer value of a month.""" return datetime.strptime(month, '%b').strftime('%m') From 1124d5a2d1ebd5c113cb9d35371b9979718c1cb1 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 23 Jan 2024 13:50:22 -0500 Subject: [PATCH 15/18] extra space --- tdrs-backend/tdpservice/parsers/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index cebcd56a3..53ef27952 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -274,7 +274,6 @@ def fiscal_to_calendar(year, fiscal_quarter): ind_qtr = array.index(int_qtr) # get the index so we can easily wrap-around end of array return year, "Q{}".format(array[ind_qtr - 1]) # return the previous quarter - def transform_to_months(quarter): """Return a list of months in a quarter.""" match quarter: From 12c3d6b9f916a427216d9314ba01896469d4e0c1 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 13 Feb 2024 13:08:36 -0500 Subject: [PATCH 16/18] make the query more efficient --- tdrs-backend/tdpservice/parsers/aggregates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/aggregates.py b/tdrs-backend/tdpservice/parsers/aggregates.py index 4757e3bd2..6f35f964b 100644 --- a/tdrs-backend/tdpservice/parsers/aggregates.py +++ b/tdrs-backend/tdpservice/parsers/aggregates.py @@ -64,6 +64,8 @@ def total_errors_by_month(df, dfs_status): total_errors_data = {"months": []} + errors = ParserError.objects.all().filter(file=df) + for month in month_list: if dfs_status == "Rejected": total_errors_data["months"].append( @@ -73,8 +75,7 @@ def total_errors_by_month(df, dfs_status): month_int = month_to_int(month) rpt_month_year = int(f"{calendar_year}{month_int}") - error_count = ParserError.objects.filter( - file=df, rpt_month_year=rpt_month_year).count() + error_count = errors.filter(rpt_month_year=rpt_month_year).count() total_errors_data["months"].append( {"month": month, "total_errors": error_count}) From f836320dc148e9d4b21b3dc3f6380cbe09be351f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 26 Feb 2024 13:06:16 -0500 Subject: [PATCH 17/18] rm unused code --- .../src/components/SubmissionHistory/SubmissionHistory.jsx | 2 +- .../src/components/SubmissionHistory/TotalAggregatesTable.jsx | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index 8ac180d76..d61ce85ec 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -3,7 +3,7 @@ import PropTypes from 'prop-types' import { useDispatch, useSelector } from 'react-redux' import { fileUploadSections } from '../../reducers/reports' import Paginator from '../Paginator' -import { getAvailableFileList, download } from '../../actions/reports' +import { getAvailableFileList } from '../../actions/reports' import { useEffect } from 'react' import { useState } from 'react' import { CaseAggregatesTable } from './CaseAggregatesTable' diff --git a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx index 1019313a3..8a8e6fd5c 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx @@ -7,10 +7,6 @@ import { downloadErrorReport, } from './helpers' -// const section_index = (element) => file.section.includes(element) - -// const section = fileUploadSections.findIndex(section_index) + 1 - const MonthSubRow = ({ data }) => data ? ( <> From b0e063a4b4c58e868e48f25164b514d905e49506 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 26 Feb 2024 13:09:33 -0500 Subject: [PATCH 18/18] fix error report url --- tdrs-frontend/src/components/SubmissionHistory/helpers.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx b/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx index ff2aca7bb..e0663636f 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/helpers.jsx @@ -15,7 +15,7 @@ export const downloadFile = (dispatch, file) => dispatch(download(file)) export const downloadErrorReport = async (file, reportName) => { try { const promise = axios.get( - `${process.env.REACT_APP_BACKEND_URL}/parsing/parsing_errors/?file=${file.id}`, + `${process.env.REACT_APP_BACKEND_URL}/data_files/${file.id}/download_error_report/`, { responseType: 'json', }
{`Section ${section} - ${label}`}
- Submitted On - - Submitted By - - File Name - - Status - - Error Reports (In development) -
{data.month}{data.total_errors}-N/A
+ {formatDate(file.createdAt)} + + {file.submittedBy} + + + + + + + {file.summary && file.summary.status + ? file.summary.status + : 'Pending'} + + {file.summary && + file.summary.status && + file.summary.status !== 'Pending' ? ( + file.hasError > 0 ? ( + + ) : ( + 'No Errors' + ) + ) : ( + 'Pending' + )} +
+ Submitted On + + Submitted By + + File Name + + Month + + Total Errors + + Status + + Error Reports (In development) +
{file.summary?.case_aggregates?.rejected || 'N/A'} @@ -83,10 +83,10 @@ const CaseAggregatesRow = ({ file }) => {
@@ -81,10 +81,10 @@ const TotalAggregatesRow = ({ file }) => {