From 954edc74a494040db5f82a33a97fb25cbdaec47b Mon Sep 17 00:00:00 2001 From: jtimpe <111305129+jtimpe@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:33:32 -0400 Subject: [PATCH] 1612 case aggregates frontend (#2643) * saving state real quick * finishing merge with latest * Missed old test script * Added new test, more cleanup * Updating unit tests in DFS, preparing for 1610 * Merging in Jan's 1610 code for parserError useful-ness * Revert "Merging in Jan's 1610 code for parserError useful-ness" This reverts commit c5796da69d0e9a6d356057550378d536e2be5f8b. * update to test to use dfs fixture * saving state before new 1610 merge * Resolving merge conflicts with 1610. * Linting changes and comparing to 1610 * Some unit test linting but inherited 1610 issues * Re-ordering job to run tests vs lint first. * Updates to linting and unit tests. * Fixing linting. * Update tdrs-backend/setup.cfg * updates per PR. * Excluding trailers for rejection * VSCode merge resolution is garbage. * Fixing precheck for not implemented types * Updating to error-handle not implemented schema types * - Updated view to show latest datafiles - Added admin filter to show newest or all datafile records - Updated indices to allow easier elastic queries * - Updated search indices to have parent FK * - Fix lint errors * - Updated submission tests - Moved create_datafile to util * - fix lint errors * - removing frontend filtering * - addding datafile to admin model * Revert "- addding datafile to admin model" This reverts commit 35a6f24c36c3a4c00ddcfc40f20833530b0199f4. * - Fixed issue where datafile FK wasnt populating - Regenerated migration * - Readding datafile back to admin view now that the error is resolved * - adding datafile back * Revert "- Readding datafile back to admin view now that the error is resolved" This reverts commit 2807425059fd1b5b355edfb16d30d170cf869d7b. * - Removed unnecessary fields - Updated dependencies - Updated filter * - Updated document to include required fields * - Fixed failing test * add adminUrl to deployment cypress overrides * Adding "beta" banners to relevant error report sections (#2522) * Update views.py * Update views.py * Update SubmissionHistory.jsx * Update SubmissionHistory.test.js * Apply suggestions from code review Co-authored-by: Miles Reiter * lint fixes --------- Co-authored-by: Miles Reiter Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: andrew-jameson * Create sprint-73-summary.md (#2565) * hotfix for large file sizes (#2542) * hotfix for large file sizes * apply timeouts/req limits to dev * filter identity pages from scan * IGNORE sql injection --------- Co-authored-by: Jan Timpe Co-authored-by: mo sohani Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> * updating validation error language * accidentally included coding challenge * rm comments * 2550 deactivation email link (#2557) * - updated nginx buildpack * - specifying different nginx version * - Updating changelog * - added script to update certain apps in cf - added workflow for each environment in circi * - fixed base config * - fixing jobs * - Updated based on feedback in OH * - Updating defaults * - Removing defaults * - Fixing indent * - Adding params to config * test * test * - updating work dir * - Adding checkout * - adding cf check * - logging into cf * - update cf check to install required binary * - removing unnecessary switch * - Forcing plugin installation * - test installing plugin from script also * - Adding url to email * - test code for sandbox * - using my email * Revert "Merge branch 'update-cf-os' into 2551-deactivation-email-link" This reverts commit e963b9df48dd1f72ca0c5b192c979bac11851d11, reversing changes made to cc9cf81e9d76c42f51ffd5e102f6027d3eb5e645. * Revert "- using my email" This reverts commit cc9cf81e9d76c42f51ffd5e102f6027d3eb5e645. * Revert "- test code for sandbox" This reverts commit 06037747197d17ed8e63b086fcfcf048ecb50dc4. --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * Update README.md (#2577) Add ATO Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * Create 2023, Spring - Testing CSV & Excel-based error reports.md * Update README.md * Updating deliverable links (#2584) * User viewset not returning/duplicating users (#2573) * - Fixed issue not allowing pagination to work locally with nginx - Added ordering to user field to fix duplicates issue * - fix lint error * - Removing ID check since we cannot guarantee that the uuid that is generated per test run will be lexigraphically consistent --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * Update cf os (#2523) * - updated nginx buildpack * - specifying different nginx version * - Updating changelog * - added script to update certain apps in cf - added workflow for each environment in circi * - fixed base config * - fixing jobs * - Updated based on feedback in OH * - Updating defaults * - Removing defaults * - Fixing indent * - Adding params to config * test * test * - updating work dir * - Adding checkout * - adding cf check * - logging into cf * - update cf check to install required binary * - removing unnecessary switch * - Forcing plugin installation * - test installing plugin from script also * - Adding new dependencies * - adding package * - fixing broken install * - fixing libs * - using correct command * - gettign correct version of libc * - trying to upgrade libs * - testing * - Updated README and script * Revert "- Updated README and script" This reverts commit 92697b3e53d1fd87b8d3e7995abb9093aa26e307. * - Removed unnecessary circi stuff - Removed script - Updated docs to callout updating secondary apps * - Correct spelling error --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * Item Number Mismatch (#2578) * - Updated schemas and models to reflect correct item numbers of fields * - Revert migration * - Updated header/trailer item numbers * - Fixed item numbers off by one errors --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * pipeline filtering (#2538) * pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md --------- Co-authored-by: George Hudson Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * Hotfix Devops/2457 path filtering for documentation (#2597) * pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md * build and test all on PRs even for documentation --------- Co-authored-by: George Hudson * Create sprint-74-summary.md (#2596) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * added URL filters (#2580) * added URL filters * allow github to trigger owasp and label deploys (#2601) Co-authored-by: George Hudson --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> Co-authored-by: George Hudson Co-authored-by: George Hudson * Create sprint-75-summary.md (#2608) * Create sprint-76-summary.md (#2609) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * - Resolved failing tests * - Corrected merge thrash * - Using randbits to generate pk to get around confilcting sequence pks * Revert "- Using randbits to generate pk to get around confilcting sequence pks" This reverts commit ac9b0659a62f64c4114c41faf0baa659a92be07c. * - Updating region in fixture instead of factory - letting django handle transaction for test * - Moved datafile reference to avoid confusion * pushing up incomplete codebase * Other unit tests now have passed w/ good error handling * Working tests, need to get setup for case aggregates populating via DB * - Updated queries - Added helper function - Need to merge in 2579 for queries to work * minor improvement to month2int * - Fixing most merge errors * - Fixing functions * - Updated queries based on generic relation * - Updated queries to count by case number instead of record number * - Added route - Updated task to create dfs * - updated tests to include dfs * Cleaning up most comments that are no longer necessary and fixed lint issues. * making minor updates, still broken tests. * updating pipfile.lock and rebuild image resolved test issues * Reorganizing tests, still failing in test_parse.py * deleted summary file, split into other test scripts. * Fixed missing self reference. * Linting fixes. * Found reference failure in deployed env. * Removing extra returns for missing record type. * lint fix * Addressed invocation of datafile for failing test * lint update for whitespace * Intermediary commit, broken test * add reverse relation * frontend summary status * new assignemnts in util * - updated rejected query to correctly count objs * - Fixing most tests * - Fixed user error. Swapped numbers by accident. * - make region None to avoid PK collision * - Fix lint errors * - Updating to avoid warning * add acceptance icon * add reverse relationship to summary model * vscode merge conflict resolution (#2623) * auto-create the external network * didn't stage commit properly * checking diffs, matching 1613.2 * doesn't work in pipeline. must be cached local * re-commenting in unit test * lint failures fixed --------- Co-authored-by: andrew-jameson * add a pending status * add acceptance status tests * update warning icon * rm old assert * update rejected icon * add case aggregates to submission history * url change per me, want pipeline to run e2e * update case aggregate columns * update col names * Upgraded to querysets, fix PR comments, PE str * missing : not caught locally * Feat/1613 merge 2 (#2650) * Create sprint-78-summary.md (#2645) * Missing/unsaved parser_error for record_type * removing redundant tests * Hopefully resolved on unit tests and lint --------- Co-authored-by: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Co-authored-by: andrew-jameson * icontains * tests * Changing dict structure per 1612. * fixed tests and lint issues, parse is too complex * schema_manager replaces schema check * Saving state prior to merge-conflict. * Adopting latest manager, removing old error style. * Commented out t6 line during Office hours * minor reference update * Acclimating to schemaManager * lint-fix isinstance * syntax mistake with isinstance * Apply suggestions from code review * reverting search_index merge artifacts. * adjusting for removing unused "get-schema()" * whitespace lint * Feedback from Jan * Ensuring tests run/work. * Ensure we have leading zero in rptmonthyear. * Minor lint fix for exception logging * fix migration * add backend partially accepted status * add frontend partially accepted status * add partially accepted test * rm accidental commit * rm unused variable * conditional submission history * fix test * resolving merge conflict problems * fixing tests from merge conflicts. * - fixed migration reference * dumb lint fix * - Fixed render error * Adding section * - Resolved type mismatches causing render failures * - Adding branch coverage * - Fix lint errors * - Fix lint errors * - Consolodated tetss * reducing line length for lint * - updated maxWidth to be percentage * Moving DFS migration into it's own file to avoid conflicts. * rename migrations * left align error report text * github got stuck :( * rm comment * rm margin, add text for no error report * fix tests * fix ssp case aggregates display --------- Co-authored-by: Andrew Jameson Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> Co-authored-by: andrew-jameson Co-authored-by: elipe17 Co-authored-by: Miles Reiter Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Co-authored-by: mo sohani Co-authored-by: Eric Lipe <125676261+elipe17@users.noreply.github.com> Co-authored-by: Lauren Frohlich <61251539+lfrohlich@users.noreply.github.com> Co-authored-by: Miles Reiter Co-authored-by: George Hudson Co-authored-by: George Hudson Co-authored-by: raftmsohani <97037188+raftmsohani@users.noreply.github.com> --- .../tdpservice/parsers/test/test_parse.py | 1 - tdrs-backend/tdpservice/parsers/util.py | 1 - tdrs-frontend/src/assets/Reports.scss | 1 + .../SubmissionHistory/SubmissionHistory.jsx | 199 +++++++++++++++--- .../SubmissionHistory.test.js | 37 +++- 5 files changed, 194 insertions(+), 45 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index f6962b1c4..d75abafeb 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -43,7 +43,6 @@ def test_parse_small_correct_file(test_datafile, dfs): ]} assert dfs.get_status() == DataFileSummary.Status.ACCEPTED - assert TANF_T1.objects.count() == 1 # spot check diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index e3e13538c..6ed48e44e 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -276,7 +276,6 @@ def transform_to_months(quarter): case _: raise ValueError("Invalid quarter value.") - def month_to_int(month): """Return the integer value of a month.""" return datetime.strptime(month, '%b').strftime('%m') diff --git a/tdrs-frontend/src/assets/Reports.scss b/tdrs-frontend/src/assets/Reports.scss index f515530a6..58b89ce6c 100644 --- a/tdrs-frontend/src/assets/Reports.scss +++ b/tdrs-frontend/src/assets/Reports.scss @@ -35,6 +35,7 @@ border: none; text-decoration: underline; color: #264A64; + text-align: left; } .section-download:hover { diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index d7045e15d..55df49364 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -51,6 +51,63 @@ const SubmissionSummaryStatusIcon = ({ status }) => { ) } +const CaseAggregatesHeader = ({ section }) => + section === 1 || section === 2 ? ( + <> + + Month + + + Cases Without Errors + + + Cases With Errors + + + Records Unable To Process + + + ) : ( + <> + + Month + + + Total + + + Cases With Errors + + + ) + +const CaseAggregatesRow = ({ data, section }) => + section === 1 || section === 2 ? ( + data ? ( + <> + {data.month} + {data.accepted_without_errors} + {data.accepted_with_errors} + + ) : ( + <> + - + N/A + N/A + + ) + ) : data ? ( + <> + {data.month} + {data.total} + + ) : ( + <> + - + N/A + + ) + const SubmissionHistoryRow = ({ file }) => { const dispatch = useDispatch() @@ -72,35 +129,96 @@ const SubmissionHistoryRow = ({ file }) => { } } + const section_index = (element) => file.section.includes(element) + + const section = fileUploadSections.findIndex(section_index) + 1 + return ( - - {formatDate(file.createdAt)} - {file.submittedBy} - - - - - {file.summary ? ( - <> - - {file.summary && file.summary.status - ? file.summary.status - : 'Pending'} - - ) : ( - 'N/A' - )} - - - {file.hasError > 0 ? ( - - ) : null} - - + + + + + {file.summary && + file.summary.case_aggregates && + file.summary.case_aggregates.months + ? file.summary.case_aggregates.rejected + : 'N/A'} + + + + + + + {file.summary && file.summary.status + ? file.summary.status + : 'Pending'} + + + + {file.summary && + file.summary.status && + file.summary.status !== 'Pending' ? ( + file.hasError > 0 ? ( + + ) : ( + 'No Errors' + ) + ) : ( + 'Pending' + )} + + + + + + + + + + ) } @@ -118,18 +236,33 @@ const SectionSubmissionHistory = ({ section, label, files }) => { const pageEnd = Math.min(files.length, pageStart + pageSize) return ( -
+
{files && files.length > 0 ? ( <> - - - - - + + + + + + diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js index 1926cce32..abf370523 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js @@ -339,13 +339,25 @@ describe('SubmissionHistory', () => { }) it.each([ - 'Pending', - 'Accepted', - 'Accepted with Errors', - 'Partially Accepted with Errors', - 'Rejected', - null, - ])('Shows the submission acceptance status', (status) => { + ['Pending', 'Active Case Data'], + ['Pending', 'Closed Case Data'], + ['Pending', 'Aggregate Data'], + ['Accepted', 'Active Case Data'], + ['Accepted', 'Closed Case Data'], + ['Accepted', 'Aggregate Data'], + ['Accepted with Errors', 'Active Case Data'], + ['Accepted with Errors', 'Closed Case Data'], + ['Accepted with Errors', 'Aggregate Data'], + ['Partially Accepted with Errors', 'Active Case Data'], + ['Partially Accepted with Errors', 'Closed Case Data'], + ['Partially Accepted with Errors', 'Aggregate Data'], + ['Rejected', 'Active Case Data'], + ['Rejected', 'Closed Case Data'], + ['Rejected', 'Aggregate Data'], + [null, 'Active Case Data'], + [null, 'Closed Case Data'], + [null, 'Aggregate Data'], + ])('Shows the submission acceptance status section 3', (status, section) => { const state = { reports: { files: [ @@ -354,7 +366,7 @@ describe('SubmissionHistory', () => { fileName: 'test1.txt', fileType: 'TANF', quarter: 'Q1', - section: 'Active Case Data', + section: section, uuid: '123-4-4-321', year: '2023', s3_version_id: '321-0-0-123', @@ -392,8 +404,13 @@ describe('SubmissionHistory', () => { setup(store) - expect(screen.queryByText('Acceptance Status')).toBeInTheDocument() + expect(screen.queryByText('Status')).toBeInTheDocument() expect(screen.queryByText('test1.txt')).toBeInTheDocument() - expect(screen.queryByText(status || 'Pending')).toBeInTheDocument() + + if (status && status !== 'Pending') { + expect(screen.queryByText(status)).toBeInTheDocument() + } else { + expect(screen.queryAllByText('Pending')).toHaveLength(2) + } }) })
{`Section ${section} - ${label}`}
Submitted OnSubmitted ByFile NameAcceptance StatusError Reports (In development) + Submitted On + + Submitted By + + File Name + + Status + + Error Reports (In development) +