Frontend File Validation (#2690)

* - small grammar fix

* - Fix lint suggestions

* - updated message

* - Fixed lint errors

* - Added correct extension to datafiles without one

* - Adding cherry picks for file extension error handling

* - Updated regex

* - updating to keep file in dropbox in event of error to help user correct their mistake.

* - Fix icon rendering incorrectly

* update test file extensions

* - making timeout longer

* - Resolved issue causing test failure
- resetting timeout

* - passing param

---------

Co-authored-by: Alex P <[email protected]>
Co-authored-by: Miles Reiter <[email protected]>
Co-authored-by: Jan Timpe <[email protected]>

tdrs-backend/tdpservice/data_files/validators.py

@@ -22,7 +22,7 @@
 def _get_unsupported_msg(_type, value, supported_options):
     """Construct a message to convey an unsupported operation."""
     return (
-        f'Unsupported {_type}: {value}, supported {pluralize(_type)} '
+        f'Unsupported {_type}: supported {pluralize(_type)} '
         f'are: {supported_options}'
     )
 

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -19,7 +19,7 @@
 @pytest.fixture
 def test_datafile(stt_user, stt):
     """Fixture for small_correct_file."""
-    return util.create_test_datafile('small_correct_file', stt_user, stt)
+    return util.create_test_datafile('small_correct_file.txt', stt_user, stt)
 
 @pytest.fixture
 def dfs():
@@ -596,7 +596,7 @@ def test_parse_super_big_s1_file_with_rollback(super_big_s1_rollback_file):
 @pytest.fixture
 def bad_tanf_s1__row_missing_required_field(stt_user, stt):
     """Fixture for small_tanf_section1."""
-    return util.create_test_datafile('small_bad_tanf_s1', stt_user, stt)
+    return util.create_test_datafile('small_bad_tanf_s1.txt', stt_user, stt)
 
 
 @pytest.mark.django_db
@@ -643,7 +643,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field
 @pytest.fixture
 def bad_ssp_s1__row_missing_required_field(stt_user, stt):
     """Fixture for ssp_section1_datafile."""
-    return util.create_test_datafile('small_bad_ssp_s1', stt_user, stt, 'SSP Active Case Data')
+    return util.create_test_datafile('small_bad_ssp_s1.txt', stt_user, stt, 'SSP Active Case Data')
 
 
 @pytest.mark.django_db()

tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py

@@ -12,8 +12,8 @@
 
 @pytest.fixture
 def test_datafile(stt_user, stt):
-    """Fixture for small_correct_file."""
-    return create_test_datafile('small_correct_file', stt_user, stt)
+    """Fixture for small_correct_file.txt."""
+    return create_test_datafile('small_correct_file.txt', stt_user, stt)
 
 
 @pytest.mark.django_db

tdrs-frontend/cypress/e2e/integration/file_upload.js

@@ -5,7 +5,7 @@ Then('{string} can see Data Files page', (username) => {
     cy.visit('/data-files')
     cy.contains('Data Files').should('exist')
   })
-  
+
 Then('{string} can see search form', (username) => {
     cy.contains('Fiscal Year').should('exist')
     cy.contains('Quarter').should('exist')
@@ -19,7 +19,7 @@ Then('{string} can browse upload file form', (username) => {
 
 When('{string} uploads a file', (username) => {
     cy.get('button').contains('Search').should('exist').click()
-    cy.get('#closed-case-data').selectFile('../tdrs-backend/tdpservice/parsers/test/data/small_correct_file',{ action: 'drag-drop' })
+    cy.get('#closed-case-data').selectFile('../tdrs-backend/tdpservice/parsers/test/data/small_correct_file.txt',{ action: 'drag-drop' })
     cy.get('button').contains('Submit Data Files').should('exist').click()
 })
 

tdrs-frontend/src/actions/reports.js

@@ -11,6 +11,7 @@ export const SET_FILE = 'SET_FILE'
 export const CLEAR_FILE = 'CLEAR_FILE'
 export const CLEAR_FILE_LIST = 'CLEAR_FILE_LIST'
 export const SET_FILE_ERROR = 'SET_FILE_ERROR'
+export const FILE_EXT_ERROR = 'FILE_EXT_ERROR'
 export const SET_FILE_SUBMITTED = 'SET_FILE_SUBMITTED'
 export const CLEAR_ERROR = 'CLEAR_ERROR'
 
@@ -254,7 +255,11 @@ export const submit =
         setLocalAlertState({
           active: true,
           type: 'error',
-          message: error.message,
+          message: ''.concat(
+            error.message,
+            ': ',
+            error.response?.data?.file[0]
+          ),
         })
       )
   }

tdrs-frontend/src/actions/reports.test.js

@@ -201,7 +201,7 @@ describe('actions/reports', () => {
     expect(axios.post).toHaveBeenCalledTimes(1)
     expect(setLocalAlertState).toHaveBeenCalledWith({
       active: true,
-      message: undefined,
+      message: 'undefined: undefined',
       type: 'error',
     })
   })

tdrs-frontend/src/actions/requestAccess.js

@@ -1,6 +1,5 @@
 import { SET_AUTH } from './auth'
 import axios from 'axios'
-import axiosInstance from '../axios-instance'
 import { logErrorToServer } from '../utils/eventLogger'
 
 export const PATCH_REQUEST_ACCESS = 'PATCH_REQUEST_ACCESS'

tdrs-frontend/src/components/FileUpload/FileUpload.jsx

@@ -7,6 +7,7 @@ import {
   clearError,
   clearFile,
   SET_FILE_ERROR,
+  FILE_EXT_ERROR,
   upload,
   download,
 } from '../../actions/reports'
@@ -17,6 +18,9 @@ import { handlePreview, getTargetClassName } from './utils'
 const INVALID_FILE_ERROR =
   'We can’t process that file format. Please provide a plain text file.'
 
+const INVALID_EXT_ERROR =
+  'Invalid extension. Accepted file types are: .txt, .ms##, .ts##, or .ts###.'
+
 function FileUpload({ section, setLocalAlertState }) {
   // e.g. 'Aggregate Case Data' => 'aggregate-case-data'
   // The set of uploaded files in our Redux state
@@ -31,6 +35,10 @@ function FileUpload({ section, setLocalAlertState }) {
     (file) => file.section.includes(sectionName) && file.uuid
   )
 
+  const hasPreview = files?.some(
+    (file) => file.section.includes(sectionName) && file.name
+  )
+
   const selectedFile = files?.find((file) => file.section.includes(sectionName))
 
   const formattedSectionName = selectedFile?.section
@@ -54,8 +62,10 @@ function FileUpload({ section, setLocalAlertState }) {
         setTimeout(trySettingPreview, 100)
       }
     }
-    if (hasFile) trySettingPreview()
-  }, [hasFile, fileName, targetClassName])
+    if (hasPreview || hasFile) {
+      trySettingPreview()
+    }
+  }, [hasPreview, hasFile, fileName, targetClassName])
 
   const downloadFile = ({ target }) => {
     dispatch(clearError({ section: sectionName }))
@@ -89,6 +99,19 @@ function FileUpload({ section, setLocalAlertState }) {
     filereader.onloadend = (evt) => {
       /* istanbul ignore next */
       if (!evt.target.error) {
+        // Validate file extension before proceeding
+        const re = /(\.txt|\.ms\d{2}|\.ts\d{2,3})$/i
+        if (!re.exec(file.name)) {
+          dispatch({
+            type: FILE_EXT_ERROR,
+            payload: {
+              error: { message: INVALID_EXT_ERROR },
+              section,
+            },
+          })
+          return
+        }
+
         // Read in the file blob "headers: and create a hex string signature
         const uint = new Uint8Array(evt.target.result)
         const bytes = []

tdrs-frontend/src/components/Paginator/Paginator.test.js

@@ -1,5 +1,5 @@
 import React from 'react'
-import { render, fireEvent, waitFor, screen } from '@testing-library/react'
+import { render, fireEvent, screen } from '@testing-library/react'
 import Paginator from './Paginator'
 
 describe('Paginator', () => {

tdrs-frontend/src/components/SiteMap/SiteMap.test.js

@@ -1,7 +1,6 @@
 import React from 'react'
 import { render } from '@testing-library/react'
 import SiteMap from './SiteMap'
-import { mount } from 'enzyme'
 import thunk from 'redux-thunk'
 import { Provider } from 'react-redux'
 import configureStore from 'redux-mock-store'

tdrs-frontend/src/reducers/reports.js

@@ -2,6 +2,7 @@ import {
   SET_FILE,
   CLEAR_FILE,
   SET_FILE_ERROR,
+  FILE_EXT_ERROR,
   CLEAR_ERROR,
   SET_SELECTED_YEAR,
   SET_SELECTED_STT,
@@ -177,6 +178,11 @@ const reports = (state = initialState, action) => {
       const updatedFiles = getUpdatedFiles({ state, section, error })
       return { ...state, submittedFiles: updatedFiles }
     }
+    case FILE_EXT_ERROR: {
+      const { error, section } = payload
+      const updatedFiles = getUpdatedFiles({ state, section, error })
+      return { ...state, submittedFiles: updatedFiles }
+    }
     case CLEAR_ERROR: {
       const { section } = payload
       const file = getFile(state.submittedFiles, section)