From dbb1e23b2042ac9413bd04c81530db6e65df08a0 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 26 Apr 2024 12:17:52 -0400 Subject: [PATCH 01/52] clean option string util --- .../tdpservice/parsers/test/test_util.py | 16 +++++++++++++++- tdrs-backend/tdpservice/parsers/util.py | 6 ++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index cd50f9c4a..54fbc73b5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -3,7 +3,7 @@ import pytest from ..fields import Field from ..row_schema import RowSchema, SchemaManager -from ..util import make_generate_parser_error, create_test_datafile +from ..util import make_generate_parser_error, create_test_datafile, clean_options_string def passing_validator(): @@ -525,3 +525,17 @@ def postparse_validator(): assert is_valid is False assert errors[0].fields_json == {'friendly_name': {'FIRST': 'first', 'SECOND': 'second'}} assert errors[0].error_message == "an Error" + + +@pytest.mark.parametrize('options, expected', [ + ([1, 2, 3, 4], '[1, 2, 3, 4]'), + (['1', '2', '3', '4'], '[1, 2, 3, 4]'), + (['a', 'b', 'c', 'd'], '[a, b, c, d]'), + (('a', 'b', 'c', 'd'), '[a, b, c, d]'), + (["'a'", "'b'", "'c'", "'d'"], "['a', 'b', 'c', 'd']"), + (['words', 'are very', 'weird'], '[words, are very, weird]'), +]) +def test_clean_options_string(options, expected): + """Test `clean_options_string` util func.""" + result = clean_options_string(options) + assert result == expected diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 3a5528391..bcb163c6f 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -92,6 +92,12 @@ def contains_encrypted_indicator(line, encryption_field): return False +def clean_options_string(options, remove=['\'', '"', ' ']): + """Return a prettied-up version of an options array.""" + options_str = ', '.join(str(o) for o in options) + return f'[{options_str}]' + + ''' text -> section YES text -> models{} YES From c45c6cbddeb0bf6254cbf7a9397c1c82c00c6310 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 26 Apr 2024 12:18:09 -0400 Subject: [PATCH 02/52] add friendly name and item num to cat 2 --- .../tdpservice/data_files/test/test_api.py | 10 +- .../tdpservice/parsers/test/test_parse.py | 34 ++--- .../parsers/test/test_validators.py | 90 +++++++----- tdrs-backend/tdpservice/parsers/validators.py | 132 +++++++++--------- 4 files changed, 148 insertions(+), 118 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index bfcc3f2c5..62fb7dad4 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -103,7 +103,7 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1: 0 is not larger than 0." + + " then number of months T1 Item -1 number of months: 0 is not larger than 0." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -114,7 +114,7 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \ + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ "but must be 23." @staticmethod @@ -134,8 +134,10 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1: 0 is not larger than 0.") + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == ( + "if CASH_AMOUNT :873 validator1 passed then " + "NBR_MONTHS T1 Item -1 NBR_MONTHS: 0 is not larger than 0." + ) @staticmethod def assert_data_file_exists(data_file_data, version, user): diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 1de564969..d3ae3f829 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -232,7 +232,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 24 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 24 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -261,7 +261,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 14 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 14 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -343,7 +343,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): trailer_error = parser_errors.get(row_number=3) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'TRAILER record length is 11 characters but must be 23.' + assert trailer_error.error_message == 'TRAILER: record length is 11 characters but must be 23.' assert trailer_error.content_type is None assert trailer_error.object_id is None @@ -354,7 +354,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors_list.append(row_error) assert row_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.error_message in [ - 'TRAILER record length is 11 characters but must be 23.', + 'TRAILER: record length is 11 characters but must be 23.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.'] assert row_error.content_type is None assert row_error.object_id is None @@ -368,7 +368,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors = list(parser_errors.filter(row_number=2).order_by("id")) length_error = row_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1 record length is 7 characters but must be 156.' + assert length_error.error_message == 'T1: record length is 7 characters but must be 156.' assert length_error.content_type is None assert length_error.object_id is None assert errors == { @@ -400,7 +400,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): trailer_error_1 = trailer_errors[0] assert trailer_error_1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error_1.error_message == 'TRAILER record length is 7 characters but must be 23.' + assert trailer_error_1.error_message == 'TRAILER: record length is 7 characters but must be 23.' assert trailer_error_1.content_type is None assert trailer_error_1.object_id is None @@ -412,7 +412,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert row_2_error.error_message == 'T1 record length is 117 characters but must be 156.' + assert row_2_error.error_message == 'T1: record length is 117 characters but must be 156.' assert row_2_error.content_type is None assert row_2_error.object_id is None @@ -424,10 +424,10 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_error_list.append(row_3_error) assert row_3_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert row_3_error.error_message in { - 'T1 record length is 7 characters but must be 156.', + 'T1: record length is 7 characters but must be 156.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.', 'T1trash does not start with TRAILER.', - 'TRAILER record length is 7 characters but must be 23.', + 'TRAILER: record length is 7 characters but must be 23.', 'T1: Case number T1trash cannot contain blanks.', 'Your file does not end with a TRAILER record.'} assert row_3_error.content_type is None @@ -446,7 +446,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_errors = [trailer_errors[2], trailer_errors[3]] length_error = row_3_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1 record length is 7 characters but must be 156.' + assert length_error.error_message == 'T1: record length is 7 characters but must be 156.' assert length_error.content_type is None assert length_error.object_id is None @@ -509,7 +509,7 @@ def test_parse_empty_file(empty_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 0 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 0 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -590,7 +590,7 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile, dfs): assert err.row_number == 2 assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == 'M1: 3 is not larger or equal to 1 and smaller or equal to 2.' + assert err.error_message == 'M1 Item 11 receives subsidized housing: 3 is not larger or equal to 1 and smaller or equal to 2.' assert err.content_type is not None assert err.object_id is not None assert parser_errors.count() == 32486 @@ -884,7 +884,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel trailer_error = parser_errors.get( row_number=6, - error_message='TRAILER record length is 15 characters but must be 23.' + error_message='TRAILER: record length is 15 characters but must be 23.' ) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.content_type is None @@ -1571,9 +1571,9 @@ def test_parse_t2_invalid_dob(t2_invalid_dob_file, dfs): year_error = parser_errors[1] digits_error = parser_errors[0] - assert month_error.error_message == "T2: $9 is not a valid month." - assert year_error.error_message == "T2: Year Q897 must be larger than 1900." - assert digits_error.error_message == "T2: Q897$9 3 does not have exactly 8 digits." + assert month_error.error_message == "T2 Item 32 date of birth: $9 is not a valid month." + assert year_error.error_message == "T2 Item 32 date of birth: Year Q897 must be larger than 1900." + assert digits_error.error_message == "T2 Item 32 date of birth: Q897$9 3 does not have exactly 8 digits." @pytest.mark.django_db @@ -1703,7 +1703,7 @@ def test_parse_tanf_section_1_file_with_bad_update_indicator(tanf_section_1_file error = parser_errors.first() assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert error.error_message == "HEADER update indicator: U does not match D." + assert error.error_message == "HEADER Item 10 update indicator: U does not match D." @pytest.fixture def tribal_section_4_bad_quarter(stt_user, stt): diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 8f77292f5..76a9a6cf7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -48,8 +48,8 @@ def test_or_validators(): assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) assert validator("3", RowSchema(), "friendly_name", "item_no") == (True, None) assert validator("5", RowSchema(), "friendly_name", "item_no") == (False, - "T1 friendly_name: 5 does not match 2. or " - "T1 friendly_name: 5 does not " + "T1 Item item_no friendly_name: 5 does not match 2. or " + "T1 Item item_no friendly_name: 5 does not " "match 3.") validator = validators.or_validators(validators.matches(("2")), validators.matches(("3")), @@ -64,16 +64,16 @@ def test_or_validators(): value = "5" assert validator(value, RowSchema(), "friendly_name", "item_no") == (False, - 'T1 friendly_name: 5 does not match 2. or ' - 'T1 friendly_name: 5 does not match 3. or ' - 'T1 friendly_name: 5 does not match 4.') + 'T1 Item item_no friendly_name: 5 does not match 2. or ' + 'T1 Item item_no friendly_name: 5 does not match 3. or ' + 'T1 Item item_no friendly_name: 5 does not match 4.') validator = validators.or_validators(validators.matches((2)), validators.matches((3)), validators.isLargerThan(4)) assert validator(5, RowSchema(), "friendly_name", "item_no") == (True, None) assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, - "T1 friendly_name: 1 does not match 2. " - "or T1 friendly_name: 1 does not " - "match 3. or T1: 1 is not larger than 4.") + "T1 Item item_no friendly_name: 1 does not match 2. " + "or T1 Item item_no friendly_name: 1 does not " + "match 3. or T1 Item item_no friendly_name: 1 is not larger than 4.") def test_if_validators(): """Test `if_then_validator` gives a valid result.""" @@ -89,14 +89,14 @@ def test_if_validators(): result_field_name="Field2", result_function=validators.matches('1'), ) result = validator(value, RowSchema()) - assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Field2: 2 does not match 1.', + assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Item -1 Field2: 2 does not match 1.', ['Field1', 'Field2']) def test_and_validators(): """Test `and_validators` gives a valid result.""" validator = validators.and_validators(validators.isLargerThan(2), validators.isLargerThan(0)) - assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, 'T1: 1 is not larger than 2.') + assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, 'T1 Item item_no friendly_name: 1 is not larger than 2.') assert validator(3, RowSchema(), "friendly_name", "item_no") == (True, None) @@ -133,7 +133,7 @@ def test_quarterIsValid(value, valid): val = validators.quarterIsValid() result = val(value, RowSchema(), "friendly_name", "item_no") - errorText = None if valid else f"T1: {value[-1:]} is not a valid quarter." + errorText = None if valid else f"T1 Item item_no friendly_name: {value[-1:]} is not a valid quarter." assert result == (valid, errorText) def test_validateSSN(): @@ -146,7 +146,7 @@ def test_validateSSN(): value = "111111111" options = [str(i) * 9 for i in range(0, 10)] result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: {value} is in {options}.") + assert result == (False, f"T1 Item item_no friendly_name: {value} is in {options}.") def test_validateRace(): """Test `validateRace`.""" @@ -157,7 +157,7 @@ def test_validateRace(): value = 3 result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: {value} is not greater than or equal to 0 or smaller than or equal to 2.") + assert result == (False, f"T1 Item item_no friendly_name: {value} is not greater than or equal to 0 or smaller than or equal to 2.") def test_validateRptMonthYear(): """Test `validateRptMonthYear`.""" @@ -168,17 +168,17 @@ def test_validateRptMonthYear(): value = "T1 " result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " + assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " "Month.") value = "T1189912" result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " + assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " "Month.") value = "T1202013" result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " + assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " "Month.") def test_matches_returns_valid(): @@ -200,7 +200,7 @@ def test_matches_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 friendly_name: TEST does not match test.' + assert error == 'T1 Item item_no friendly_name: TEST does not match test.' def test_oneOf_returns_valid(): @@ -224,7 +224,7 @@ def test_oneOf_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 64 is not in [17, 24, 36].' + assert error == 'T1 Item item_no friendly_name: 64 is not in [17, 24, 36].' def test_between_returns_valid(): @@ -257,7 +257,31 @@ def test_between_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 47 is not between 48 and 400.' + assert error == 'T1 Item item_no friendly_name: 47 is not between 48 and 400.' + + +@pytest.mark.parametrize('value, expected_is_valid, expected_error', [ + (7, True, None), + (77731, True, None), + ('7', True, None), + ('234897', True, None), + ('a', False, 'T1 Item item_no friendly_name: a is not a number.'), + ('houston, we have a problem', False, 'T1 Item item_no friendly_name: houston, we have a problem is not a number.'), + (' test', False, 'T1 Item item_no friendly_name: test is not a number.'), + (' 7 ', True, None), + (' 8388323', True, None), + ('87932875 ', True, None), + (' 00 ', True, None), + (' 88 ', True, None), + (' 088 ', True, None), + (' 8 8 ', False, 'T1 Item item_no friendly_name: 8 8 is not a number.'), +]) +def test_isNumber(value, expected_is_valid, expected_error): + """Test `isNumber` validator.""" + validator = validators.isNumber() + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + assert is_valid == expected_is_valid + assert error == expected_error def test_date_month_is_valid_returns_valid(): @@ -275,7 +299,7 @@ def test_date_month_is_valid_returns_invalid(): validator = validators.dateMonthIsValid() is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 13 is not a valid month.' + assert error == 'T1 Item item_no friendly_name: 13 is not a valid month.' def test_date_day_is_valid_returns_valid(): @@ -293,7 +317,7 @@ def test_date_day_is_valid_returns_invalid(): validator = validators.dateDayIsValid() is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 32 is not a valid day.' + assert error == 'T1 Item item_no friendly_name: 32 is not a valid day.' def test_olderThan(): @@ -305,7 +329,7 @@ def test_olderThan(): value = 20240101 result = validator(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, (f"T1: {str(value)[:4]} must be less than or equal to {date.today().year - min_age} " + assert result == (False, (f"T1 Item item_no friendly_name: {str(value)[:4]} must be less than or equal to {date.today().year - min_age} " "to meet the minimum age requirement.")) @@ -318,7 +342,7 @@ def test_dateYearIsLargerThan(): value = 18990101 assert validator(value, RowSchema(), "friendly_name", "item_no") == (False, - f"T1: Year {str(value)[:4]} must be larger " + f"T1 Item item_no friendly_name: Year {str(value)[:4]} must be larger " f"than {year}.") @@ -330,7 +354,7 @@ def test_between_returns_invalid_for_string_value(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 047 is not between 100 and 400.' + assert error == 'T1 Item item_no friendly_name: 047 is not between 100 and 400.' def test_recordHasLength_returns_valid(): @@ -352,7 +376,7 @@ def test_recordHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 record length is 7 characters but must be 22.' + assert error == 'T1 Item item_no friendly_name: record length is 7 characters but must be 22.' def test_intHasLength_returns_valid(): @@ -374,7 +398,7 @@ def test_intHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 1a3 does not have exactly 22 digits.' + assert error == 'T1 Item item_no friendly_name: 1a3 does not have exactly 22 digits.' def test_contains_returns_valid(): @@ -396,7 +420,7 @@ def test_contains_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 12345abcde does not contain 6789.' + assert error == 'T1 Item item_no friendly_name: 12345abcde does not contain 6789.' def test_startsWith_returns_valid(): @@ -418,7 +442,7 @@ def test_startsWith_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 12345abcde does not start with abc.' + assert error == 'T1 Item item_no friendly_name: 12345abcde does not start with abc.' def test_notEmpty_returns_valid_full_string(): @@ -440,7 +464,7 @@ def test_notEmpty_returns_invalid_full_string(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: contains blanks between positions 0 and 9.' + assert error == 'T1 Item item_no friendly_name: contains blanks between positions 0 and 9.' def test_notEmpty_returns_valid_substring(): @@ -462,7 +486,7 @@ def test_notEmpty_returns_invalid_substring(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == "T1: 111 333 contains blanks between positions 3 and 5." + assert error == "T1 Item item_no friendly_name: 111 333 contains blanks between positions 3 and 5." def test_notEmpty_returns_nonexistent_substring(): @@ -473,7 +497,7 @@ def test_notEmpty_returns_nonexistent_substring(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == "T1: 111 333 contains blanks between positions 10 and 12." + assert error == "T1 Item item_no friendly_name: 111 333 contains blanks between positions 10 and 12." @pytest.mark.parametrize("test_input", [1, 2, 3, 4]) @@ -493,7 +517,7 @@ def test_quarterIsValid_returns_false_if_invalid(test_input): is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == f"T1: {test_input} is not a valid quarter." + assert error == f"T1 Item item_no friendly_name: {test_input} is not a valid quarter." @pytest.mark.parametrize("value", ["T72020 ", "T720194", "T720200", "T720207", "T72020$"]) def test_calendarQuarterIsValid_returns_invalid(value): @@ -502,7 +526,7 @@ def test_calendarQuarterIsValid_returns_invalid(value): is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error_msg == f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + \ + assert error_msg == f"T1 Item item_no friendly_name: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + \ "representing the Calendar Year and Quarter formatted as YYYYQ" diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 94f44288b..f4ea25cc6 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -1,7 +1,7 @@ """Generic parser validator functions for use in schema definitions.""" from .models import ParserErrorCategoryChoices -from .util import fiscal_to_calendar, year_month_to_year_quarter +from .util import fiscal_to_calendar, year_month_to_year_quarter, clean_options_string import datetime import logging @@ -217,16 +217,53 @@ def sumIsLargerFunc(value, row_schema): return sumIsLargerFunc +def recordHasLength(length): + """Validate that value (string or array) has a length matching length param.""" + return make_validator( + lambda value: len(value) == length, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type}: record length is {len(value)} characters but must be {length}.", + ) + + +def caseNumberNotEmpty(start=0, end=None): + """Validate that string value isn't only blanks.""" + return make_validator( + lambda value: not _is_empty(value, start, end), + lambda value, row_schema, + friendly_name, item_num: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' + ) + + +def calendarQuarterIsValid(start=0, end=None): + """Validate that the calendar quarter value is valid.""" + return make_validator( + lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 + and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " + "representing the Calendar Year and Quarter formatted as YYYYQ", + ) + + # generic validators +def format_error_context(row_schema, friendly_name, item_num): + return f'{row_schema.record_type} Item {item_num} {friendly_name}' + + def matches(option, error_func=None): """Validate that value is equal to option.""" return make_validator( lambda value: value == option, lambda value, row_schema, friendly_name, item_num: error_func(option) if error_func - else f"{row_schema.record_type} {friendly_name}: {value} does not match {option}.", + else f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not match {option}.", ) @@ -234,7 +271,7 @@ def notMatches(option): """Validate that value is not equal to option.""" return make_validator( lambda value: value != option, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} matches {option}." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} matches {option}." ) @@ -242,7 +279,7 @@ def oneOf(options=[]): """Validate that value does not exist in the provided options array.""" return make_validator( lambda value: value in options, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not in {options}." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in {clean_options_string(options)}." ) @@ -250,7 +287,7 @@ def notOneOf(options=[]): """Validate that value exists in the provided options array.""" return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is in {options}." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is in {clean_options_string(options)}." ) @@ -259,18 +296,7 @@ def between(min, max): return make_validator( lambda value: int(value) > min and int(value) < max, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not between {min} and {max}.", - ) - - -def recordHasLength(length): - """Validate that value (string or array) has a length matching length param.""" - return make_validator( - lambda value: len(value) == length, - lambda value, - row_schema, - friendly_name, - item_num: f"{row_schema.record_type} record length is {len(value)} characters but must be {length}.", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", ) @@ -279,7 +305,7 @@ def intHasLength(num_digits): return make_validator( lambda value: sum(c.isdigit() for c in str(value)) == num_digits, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} does not have exactly {num_digits} digits.", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not have exactly {num_digits} digits.", ) @@ -288,7 +314,7 @@ def contains(substring): return make_validator( lambda value: value.find(substring) != -1, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} does not contain {substring}.", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not contain {substring}.", ) @@ -298,15 +324,15 @@ def startsWith(substring, error_func=None): lambda value: value.startswith(substring), lambda value, row_schema, friendly_name, item_num: error_func(substring) if error_func - else f"{row_schema.record_type}: {value} does not start with {substring}.", + else f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not start with {substring}.", ) def isNumber(): """Validate that value can be casted to a number.""" return make_validator( - lambda value: value.isnumeric(), - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not a number." + lambda value: str(value).strip().isnumeric(), + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not a number." ) @@ -314,7 +340,7 @@ def isAlphaNumeric(): """Validate that value is alphanumeric.""" return make_validator( lambda value: value.isalnum(), - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not alphanumeric." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not alphanumeric." ) @@ -322,7 +348,7 @@ def isBlank(): """Validate that string value is blank.""" return make_validator( lambda value: value.isspace(), - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not blank." + lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank." ) @@ -331,7 +357,7 @@ def isInStringRange(lower, upper): return make_validator( lambda value: int(value) >= lower and int(value) <= upper, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not in range [{lower}, {upper}].", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in range [{lower}, {upper}].", ) @@ -340,7 +366,7 @@ def isStringLargerThan(val): return make_validator( lambda value: int(value) > val, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger than {val}.", + friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {val}.", ) @@ -356,39 +382,17 @@ def notEmpty(start=0, end=None): return make_validator( lambda value: not _is_empty(value, start, end), lambda value, row_schema, - friendly_name, item_num: f'{row_schema.record_type}: {str(value)} contains blanks between positions {start} ' + friendly_name, item_num: f'{format_error_context(row_schema, friendly_name, item_num)}: {str(value)} contains blanks between positions {start} ' f'and {end if end else len(str(value))}.' ) -def caseNumberNotEmpty(start=0, end=None): - """Validate that string value isn't only blanks.""" - return make_validator( - lambda value: not _is_empty(value, start, end), - lambda value, row_schema, - friendly_name, item_num: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' - ) - - -def calendarQuarterIsValid(start=0, end=None): - """Validate that the calendar quarter value is valid.""" - return make_validator( - lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 - and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, - lambda value, - row_schema, - friendly_name, - item_num: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " - "representing the Calendar Year and Quarter formatted as YYYYQ", - ) - - def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( lambda value: _is_empty(value, start, end), lambda value, row_schema, - friendly_name, item_num: f'{value} is not blank between positions {start} and {end if end else len(value)}.' + friendly_name, item_num: f'{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not blank between positions {start} and {end if end else len(value)}.' ) @@ -396,7 +400,7 @@ def notZero(number_of_zeros=1): """Validate that value is not zero.""" return make_validator( lambda value: value != "0" * number_of_zeros, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is zero." + lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is zero." ) @@ -405,7 +409,7 @@ def isLargerThan(LowerBound): return make_validator( lambda value: float(value) > LowerBound if value is not None else False, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger than {LowerBound}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger than {LowerBound}.", ) @@ -414,7 +418,7 @@ def isSmallerThan(UpperBound): return make_validator( lambda value: value < UpperBound, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not smaller than {UpperBound}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not smaller than {UpperBound}.", ) @@ -423,7 +427,7 @@ def isLargerThanOrEqualTo(LowerBound): return make_validator( lambda value: value >= LowerBound, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger than {LowerBound}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger than {LowerBound}.", ) @@ -432,7 +436,7 @@ def isSmallerThanOrEqualTo(UpperBound): return make_validator( lambda value: value <= UpperBound, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not smaller than {UpperBound}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not smaller than {UpperBound}.", ) @@ -441,7 +445,7 @@ def isInLimits(LowerBound, UpperBound): return make_validator( lambda value: value >= LowerBound and value <= UpperBound, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger or equal to {LowerBound} and " + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger or equal to {LowerBound} and " f"smaller or equal to {UpperBound}." ) @@ -453,7 +457,7 @@ def dateMonthIsValid(): return make_validator( lambda value: int(str(value)[4:6]) in range(1, 13), lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {str(value)[4:6]} is not a valid month.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[4:6]} is not a valid month.", ) def dateDayIsValid(): @@ -461,7 +465,7 @@ def dateDayIsValid(): return make_validator( lambda value: int(str(value)[6:]) in range(1, 32), lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {str(value)[6:]} is not a valid day.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[6:]} is not a valid day.", ) @@ -470,7 +474,7 @@ def olderThan(min_age): return make_validator( lambda value: datetime.date.today().year - int(str(value)[:4]) > min_age, lambda value, row_schema, - friendly_name, item_num: (f"{row_schema.record_type}: {str(value)[:4]} must be less than or equal to " + friendly_name, item_num: (f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[:4]} must be less than or equal to " f"{datetime.date.today().year - min_age} to meet the minimum age requirement.") ) @@ -480,7 +484,7 @@ def dateYearIsLargerThan(year): return make_validator( lambda value: int(str(value)[:4]) > year, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: Year {str(value)[:4]} must be larger than {year}.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: Year {str(value)[:4]} must be larger than {year}.", ) @@ -489,7 +493,7 @@ def quarterIsValid(): return make_validator( lambda value: int(str(value)[-1]) > 0 and int(str(value)[-1]) < 5, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {str(value)[-1]} is not a valid quarter.", + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[-1]} is not a valid quarter.", ) @@ -498,7 +502,7 @@ def validateSSN(): options = [str(i) * 9 for i in range(0, 10)] return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is in {options}." + lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is in {options}." ) @@ -507,7 +511,7 @@ def validateRace(): return make_validator( lambda value: value >= 0 and value <= 2, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: {value} is not greater than or equal to 0 " + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not greater than or equal to 0 " "or smaller than or equal to 2." ) @@ -519,7 +523,7 @@ def validateRptMonthYear(): "06", "07", "08", "09", "10", "11", "12"}, lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type}: The value: {value[2:8]}, does not follow the YYYYMM " + friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: The value: {value[2:8]}, does not follow the YYYYMM " "format for Reporting Year and Month.", ) From a9789001e7037c13b4a1865392512118cd48da4f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 26 Apr 2024 12:51:21 -0400 Subject: [PATCH 03/52] fix linting --- .../tdpservice/parsers/test/test_parse.py | 4 +- .../parsers/test/test_validators.py | 78 ++++++++---- tdrs-backend/tdpservice/parsers/validators.py | 116 ++++++++++-------- 3 files changed, 120 insertions(+), 78 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index d3ae3f829..8106b261e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -590,7 +590,9 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile, dfs): assert err.row_number == 2 assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == 'M1 Item 11 receives subsidized housing: 3 is not larger or equal to 1 and smaller or equal to 2.' + assert err.error_message == ( + 'M1 Item 11 receives subsidized housing: 3 is not larger or equal to 1 and smaller or equal to 2.' + ) assert err.content_type is not None assert err.object_id is not None assert parser_errors.count() == 32486 diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 76a9a6cf7..03ba4aab7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -47,10 +47,11 @@ def test_or_validators(): validator = validators.or_validators(validators.matches(("2")), validators.matches(("3"))) assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) assert validator("3", RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator("5", RowSchema(), "friendly_name", "item_no") == (False, - "T1 Item item_no friendly_name: 5 does not match 2. or " - "T1 Item item_no friendly_name: 5 does not " - "match 3.") + assert validator("5", RowSchema(), "friendly_name", "item_no") == ( + False, + "T1 Item item_no friendly_name: 5 does not match 2. or " + "T1 Item item_no friendly_name: 5 does not match 3." + ) validator = validators.or_validators(validators.matches(("2")), validators.matches(("3")), validators.matches(("4"))) @@ -63,17 +64,21 @@ def test_or_validators(): assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) value = "5" - assert validator(value, RowSchema(), "friendly_name", "item_no") == (False, - 'T1 Item item_no friendly_name: 5 does not match 2. or ' - 'T1 Item item_no friendly_name: 5 does not match 3. or ' - 'T1 Item item_no friendly_name: 5 does not match 4.') + assert validator(value, RowSchema(), "friendly_name", "item_no") == ( + False, + 'T1 Item item_no friendly_name: 5 does not match 2. or ' + 'T1 Item item_no friendly_name: 5 does not match 3. or ' + 'T1 Item item_no friendly_name: 5 does not match 4.' + ) validator = validators.or_validators(validators.matches((2)), validators.matches((3)), validators.isLargerThan(4)) assert validator(5, RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, - "T1 Item item_no friendly_name: 1 does not match 2. " - "or T1 Item item_no friendly_name: 1 does not " - "match 3. or T1 Item item_no friendly_name: 1 is not larger than 4.") + assert validator(1, RowSchema(), "friendly_name", "item_no") == ( + False, + "T1 Item item_no friendly_name: 1 does not match 2. " + "or T1 Item item_no friendly_name: 1 does not " + "match 3. or T1 Item item_no friendly_name: 1 is not larger than 4." + ) def test_if_validators(): """Test `if_then_validator` gives a valid result.""" @@ -96,7 +101,10 @@ def test_if_validators(): def test_and_validators(): """Test `and_validators` gives a valid result.""" validator = validators.and_validators(validators.isLargerThan(2), validators.isLargerThan(0)) - assert validator(1, RowSchema(), "friendly_name", "item_no") == (False, 'T1 Item item_no friendly_name: 1 is not larger than 2.') + assert validator(1, RowSchema(), "friendly_name", "item_no") == ( + False, + 'T1 Item item_no friendly_name: 1 is not larger than 2.' + ) assert validator(3, RowSchema(), "friendly_name", "item_no") == (True, None) @@ -157,7 +165,10 @@ def test_validateRace(): value = 3 result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: {value} is not greater than or equal to 0 or smaller than or equal to 2.") + assert result == ( + False, + f"T1 Item item_no friendly_name: {value} is not greater than or equal to 0 or smaller than or equal to 2." + ) def test_validateRptMonthYear(): """Test `validateRptMonthYear`.""" @@ -168,18 +179,27 @@ def test_validateRptMonthYear(): value = "T1 " result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " - "Month.") + assert result == ( + False, + f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not " + "follow the YYYYMM format for Reporting Year and Month." + ) value = "T1189912" result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " - "Month.") + assert result == ( + False, + f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow " + "the YYYYMM format for Reporting Year and Month." + ) value = "T1202013" result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow the YYYYMM format for Reporting Year and " - "Month.") + assert result == ( + False, + f"T1 Item item_no friendly_name: The value: {value[2:8]}, does " + "not follow the YYYYMM format for Reporting Year and Month." + ) def test_matches_returns_valid(): """Test `matches` gives a valid result.""" @@ -329,8 +349,11 @@ def test_olderThan(): value = 20240101 result = validator(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, (f"T1 Item item_no friendly_name: {str(value)[:4]} must be less than or equal to {date.today().year - min_age} " - "to meet the minimum age requirement.")) + assert result == ( + False, + f"T1 Item item_no friendly_name: {str(value)[:4]} must be less than or equal to " + f"{date.today().year - min_age} to meet the minimum age requirement." + ) def test_dateYearIsLargerThan(): @@ -341,9 +364,10 @@ def test_dateYearIsLargerThan(): assert validator(value) == (True, None) value = 18990101 - assert validator(value, RowSchema(), "friendly_name", "item_no") == (False, - f"T1 Item item_no friendly_name: Year {str(value)[:4]} must be larger " - f"than {year}.") + assert validator(value, RowSchema(), "friendly_name", "item_no") == ( + False, + f"T1 Item item_no friendly_name: Year {str(value)[:4]} must be larger than {year}." + ) def test_between_returns_invalid_for_string_value(): @@ -526,8 +550,10 @@ def test_calendarQuarterIsValid_returns_invalid(value): is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error_msg == f"T1 Item item_no friendly_name: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + \ + assert error_msg == ( + f"T1 Item item_no friendly_name: {value[2:7]} is invalid. Calendar Quarter must be a numeric " "representing the Calendar Year and Quarter formatted as YYYYQ" + ) @pytest.mark.parametrize("value", ["T720201", "T720202", "T720203", "T720204"]) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index f4ea25cc6..fc17998c4 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -254,6 +254,7 @@ def calendarQuarterIsValid(start=0, end=None): def format_error_context(row_schema, friendly_name, item_num): + """Format the error message for consistency across cat2 validators.""" return f'{row_schema.record_type} Item {item_num} {friendly_name}' @@ -271,7 +272,8 @@ def notMatches(option): """Validate that value is not equal to option.""" return make_validator( lambda value: value != option, - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} matches {option}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} matches {option}." ) @@ -279,7 +281,9 @@ def oneOf(options=[]): """Validate that value does not exist in the provided options array.""" return make_validator( lambda value: value in options, - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in {clean_options_string(options)}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: " + f"{value} is not in {clean_options_string(options)}." ) @@ -287,7 +291,9 @@ def notOneOf(options=[]): """Validate that value exists in the provided options array.""" return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is in {clean_options_string(options)}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: " + f"{value} is in {clean_options_string(options)}." ) @@ -295,8 +301,8 @@ def between(min, max): """Validate value, when casted to int, is greater than min and less than max.""" return make_validator( lambda value: int(value) > min and int(value) < max, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", ) @@ -304,8 +310,9 @@ def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( lambda value: sum(c.isdigit() for c in str(value)) == num_digits, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not have exactly {num_digits} digits.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: " + f"{value} does not have exactly {num_digits} digits.", ) @@ -313,8 +320,8 @@ def contains(substring): """Validate that string value contains the given substring param.""" return make_validator( lambda value: value.find(substring) != -1, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not contain {substring}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not contain {substring}.", ) @@ -332,7 +339,8 @@ def isNumber(): """Validate that value can be casted to a number.""" return make_validator( lambda value: str(value).strip().isnumeric(), - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not a number." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not a number." ) @@ -340,7 +348,8 @@ def isAlphaNumeric(): """Validate that value is alphanumeric.""" return make_validator( lambda value: value.isalnum(), - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not alphanumeric." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not alphanumeric." ) @@ -348,7 +357,8 @@ def isBlank(): """Validate that string value is blank.""" return make_validator( lambda value: value.isspace(), - lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank." ) @@ -356,8 +366,8 @@ def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( lambda value: int(value) >= lower and int(value) <= upper, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in range [{lower}, {upper}].", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in range [{lower}, {upper}].", ) @@ -365,8 +375,8 @@ def isStringLargerThan(val): """Validate that string value is larger than val.""" return make_validator( lambda value: int(value) > val, - lambda value, row_schema, - friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {val}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {val}.", ) @@ -381,9 +391,9 @@ def notEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( lambda value: not _is_empty(value, start, end), - lambda value, row_schema, - friendly_name, item_num: f'{format_error_context(row_schema, friendly_name, item_num)}: {str(value)} contains blanks between positions {start} ' - f'and {end if end else len(str(value))}.' + lambda value, row_schema, friendly_name, item_num: + f'{format_error_context(row_schema, friendly_name, item_num)}: {str(value)} contains blanks ' + f'between positions {start} and {end if end else len(str(value))}.' ) @@ -391,8 +401,9 @@ def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( lambda value: _is_empty(value, start, end), - lambda value, row_schema, - friendly_name, item_num: f'{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not blank between positions {start} and {end if end else len(value)}.' + lambda value, row_schema, friendly_name, item_num: + f'{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank ' + f'between positions {start} and {end if end else len(value)}.' ) @@ -400,7 +411,8 @@ def notZero(number_of_zeros=1): """Validate that value is not zero.""" return make_validator( lambda value: value != "0" * number_of_zeros, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is zero." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is zero." ) @@ -408,8 +420,8 @@ def isLargerThan(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: float(value) > LowerBound if value is not None else False, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger than {LowerBound}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {LowerBound}.", ) @@ -417,8 +429,8 @@ def isSmallerThan(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value < UpperBound, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not smaller than {UpperBound}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not smaller than {UpperBound}.", ) @@ -426,8 +438,8 @@ def isLargerThanOrEqualTo(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: value >= LowerBound, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger than {LowerBound}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {LowerBound}.", ) @@ -435,8 +447,8 @@ def isSmallerThanOrEqualTo(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value <= UpperBound, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not smaller than {UpperBound}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not smaller than {UpperBound}.", ) @@ -444,9 +456,9 @@ def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( lambda value: value >= LowerBound and value <= UpperBound, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not larger or equal to {LowerBound} and " - f"smaller or equal to {UpperBound}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger or equal " + f"to {LowerBound} and smaller or equal to {UpperBound}." ) @@ -456,16 +468,16 @@ def dateMonthIsValid(): """Validate that in a monthyear combination, the month is a valid month.""" return make_validator( lambda value: int(str(value)[4:6]) in range(1, 13), - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[4:6]} is not a valid month.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[4:6]} is not a valid month.", ) def dateDayIsValid(): """Validate that in a monthyearday combination, the day is a valid day.""" return make_validator( lambda value: int(str(value)[6:]) in range(1, 32), - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[6:]} is not a valid day.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[6:]} is not a valid day.", ) @@ -473,9 +485,9 @@ def olderThan(min_age): """Validate that value is larger than min_age.""" return make_validator( lambda value: datetime.date.today().year - int(str(value)[:4]) > min_age, - lambda value, row_schema, - friendly_name, item_num: (f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[:4]} must be less than or equal to " - f"{datetime.date.today().year - min_age} to meet the minimum age requirement.") + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[:4]} must be less " + f"than or equal to {datetime.date.today().year - min_age} to meet the minimum age requirement." ) @@ -483,8 +495,9 @@ def dateYearIsLargerThan(year): """Validate that in a monthyear combination, the year is larger than the given year.""" return make_validator( lambda value: int(str(value)[:4]) > year, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: Year {str(value)[:4]} must be larger than {year}.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: " + f"Year {str(value)[:4]} must be larger than {year}.", ) @@ -492,8 +505,8 @@ def quarterIsValid(): """Validate in a year quarter combination, the quarter is valid.""" return make_validator( lambda value: int(str(value)[-1]) > 0 and int(str(value)[-1]) < 5, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {str(value)[-1]} is not a valid quarter.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[-1]} is not a valid quarter.", ) @@ -502,7 +515,8 @@ def validateSSN(): options = [str(i) * 9 for i in range(0, 10)] return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is in {options}." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is in {options}." ) @@ -510,9 +524,9 @@ def validateRace(): """Validate race.""" return make_validator( lambda value: value >= 0 and value <= 2, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: {value} is not greater than or equal to 0 " - "or smaller than or equal to 2." + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not greater than or equal to 0 " + "or smaller than or equal to 2." ) @@ -522,9 +536,9 @@ def validateRptMonthYear(): lambda value: value[2:8].isdigit() and int(value[2:6]) > 1900 and value[6:8] in {"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"}, - lambda value, row_schema, - friendly_name, item_num: f"{row_schema.record_type} Item {item_num} {friendly_name}: The value: {value[2:8]}, does not follow the YYYYMM " - "format for Reporting Year and Month.", + lambda value, row_schema, friendly_name, item_num: + f"{format_error_context(row_schema, friendly_name, item_num)}: The value: {value[2:8]}, " + "does not follow the YYYYMM format for Reporting Year and Month.", ) From 977a0e8faa418fb9ec264e725e3b636ff37e13e6 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 26 Apr 2024 12:52:37 -0400 Subject: [PATCH 04/52] fix a couple tests --- tdrs-backend/tdpservice/parsers/test/test_validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 03ba4aab7..6a128c606 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -400,7 +400,7 @@ def test_recordHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: record length is 7 characters but must be 22.' + assert error == 'T1: record length is 7 characters but must be 22.' def test_intHasLength_returns_valid(): @@ -551,7 +551,7 @@ def test_calendarQuarterIsValid_returns_invalid(value): assert is_valid is False assert error_msg == ( - f"T1 Item item_no friendly_name: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " "representing the Calendar Year and Quarter formatted as YYYYQ" ) From af563c81e87976851239acf4fe9fc20504beee74 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 2 May 2024 14:13:06 -0400 Subject: [PATCH 05/52] add parens to friendly names --- .../tdpservice/data_files/test/test_api.py | 4 +- .../tdpservice/parsers/test/test_parse.py | 10 +-- .../parsers/test/test_validators.py | 70 +++++++++---------- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 62fb7dad4..3ce127db9 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -103,7 +103,7 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1 Item -1 number of months: 0 is not larger than 0." + + " then number of months T1 Item -1 (number of months): 0 is not larger than 0." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -136,7 +136,7 @@ def assert_error_report_file_content_matches_without_friendly_names(response): + " be in touch when it's ready to use!For now please refer to the reports you receive via email" assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == ( "if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1 Item -1 NBR_MONTHS: 0 is not larger than 0." + "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not larger than 0." ) @staticmethod diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 8106b261e..562aa9723 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -591,7 +591,7 @@ def test_parse_ssp_section1_datafile(ssp_section1_datafile, dfs): assert err.row_number == 2 assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE assert err.error_message == ( - 'M1 Item 11 receives subsidized housing: 3 is not larger or equal to 1 and smaller or equal to 2.' + 'M1 Item 11 (receives subsidized housing): 3 is not larger or equal to 1 and smaller or equal to 2.' ) assert err.content_type is not None assert err.object_id is not None @@ -1573,9 +1573,9 @@ def test_parse_t2_invalid_dob(t2_invalid_dob_file, dfs): year_error = parser_errors[1] digits_error = parser_errors[0] - assert month_error.error_message == "T2 Item 32 date of birth: $9 is not a valid month." - assert year_error.error_message == "T2 Item 32 date of birth: Year Q897 must be larger than 1900." - assert digits_error.error_message == "T2 Item 32 date of birth: Q897$9 3 does not have exactly 8 digits." + assert month_error.error_message == "T2 Item 32 (date of birth): $9 is not a valid month." + assert year_error.error_message == "T2 Item 32 (date of birth): Year Q897 must be larger than 1900." + assert digits_error.error_message == "T2 Item 32 (date of birth): Q897$9 3 does not have exactly 8 digits." @pytest.mark.django_db @@ -1705,7 +1705,7 @@ def test_parse_tanf_section_1_file_with_bad_update_indicator(tanf_section_1_file error = parser_errors.first() assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert error.error_message == "HEADER Item 10 update indicator: U does not match D." + assert error.error_message == "HEADER Item 10 (update indicator): U does not match D." @pytest.fixture def tribal_section_4_bad_quarter(stt_user, stt): diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 6a128c606..3d341a743 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -49,8 +49,8 @@ def test_or_validators(): assert validator("3", RowSchema(), "friendly_name", "item_no") == (True, None) assert validator("5", RowSchema(), "friendly_name", "item_no") == ( False, - "T1 Item item_no friendly_name: 5 does not match 2. or " - "T1 Item item_no friendly_name: 5 does not match 3." + "T1 Item item_no (friendly_name): 5 does not match 2. or " + "T1 Item item_no (friendly_name): 5 does not match 3." ) validator = validators.or_validators(validators.matches(("2")), validators.matches(("3")), @@ -66,18 +66,18 @@ def test_or_validators(): value = "5" assert validator(value, RowSchema(), "friendly_name", "item_no") == ( False, - 'T1 Item item_no friendly_name: 5 does not match 2. or ' - 'T1 Item item_no friendly_name: 5 does not match 3. or ' - 'T1 Item item_no friendly_name: 5 does not match 4.' + 'T1 Item item_no (friendly_name): 5 does not match 2. or ' + 'T1 Item item_no (friendly_name): 5 does not match 3. or ' + 'T1 Item item_no (friendly_name): 5 does not match 4.' ) validator = validators.or_validators(validators.matches((2)), validators.matches((3)), validators.isLargerThan(4)) assert validator(5, RowSchema(), "friendly_name", "item_no") == (True, None) assert validator(1, RowSchema(), "friendly_name", "item_no") == ( False, - "T1 Item item_no friendly_name: 1 does not match 2. " - "or T1 Item item_no friendly_name: 1 does not " - "match 3. or T1 Item item_no friendly_name: 1 is not larger than 4." + "T1 Item item_no (friendly_name): 1 does not match 2. " + "or T1 Item item_no (friendly_name): 1 does not " + "match 3. or T1 Item item_no (friendly_name): 1 is not larger than 4." ) def test_if_validators(): @@ -94,7 +94,7 @@ def test_if_validators(): result_field_name="Field2", result_function=validators.matches('1'), ) result = validator(value, RowSchema()) - assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Item -1 Field2: 2 does not match 1.', + assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Item -1 (Field2): 2 does not match 1.', ['Field1', 'Field2']) @@ -103,7 +103,7 @@ def test_and_validators(): validator = validators.and_validators(validators.isLargerThan(2), validators.isLargerThan(0)) assert validator(1, RowSchema(), "friendly_name", "item_no") == ( False, - 'T1 Item item_no friendly_name: 1 is not larger than 2.' + 'T1 Item item_no (friendly_name): 1 is not larger than 2.' ) assert validator(3, RowSchema(), "friendly_name", "item_no") == (True, None) @@ -141,7 +141,7 @@ def test_quarterIsValid(value, valid): val = validators.quarterIsValid() result = val(value, RowSchema(), "friendly_name", "item_no") - errorText = None if valid else f"T1 Item item_no friendly_name: {value[-1:]} is not a valid quarter." + errorText = None if valid else f"T1 Item item_no (friendly_name): {value[-1:]} is not a valid quarter." assert result == (valid, errorText) def test_validateSSN(): @@ -154,7 +154,7 @@ def test_validateSSN(): value = "111111111" options = [str(i) * 9 for i in range(0, 10)] result = val(value, RowSchema(), "friendly_name", "item_no") - assert result == (False, f"T1 Item item_no friendly_name: {value} is in {options}.") + assert result == (False, f"T1 Item item_no (friendly_name): {value} is in {options}.") def test_validateRace(): """Test `validateRace`.""" @@ -167,7 +167,7 @@ def test_validateRace(): result = val(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: {value} is not greater than or equal to 0 or smaller than or equal to 2." + f"T1 Item item_no (friendly_name): {value} is not greater than or equal to 0 or smaller than or equal to 2." ) def test_validateRptMonthYear(): @@ -181,7 +181,7 @@ def test_validateRptMonthYear(): result = val(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not " + f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does not " "follow the YYYYMM format for Reporting Year and Month." ) @@ -189,7 +189,7 @@ def test_validateRptMonthYear(): result = val(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: The value: {value[2:8]}, does not follow " + f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does not follow " "the YYYYMM format for Reporting Year and Month." ) @@ -197,7 +197,7 @@ def test_validateRptMonthYear(): result = val(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: The value: {value[2:8]}, does " + f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does " "not follow the YYYYMM format for Reporting Year and Month." ) @@ -220,7 +220,7 @@ def test_matches_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: TEST does not match test.' + assert error == 'T1 Item item_no (friendly_name): TEST does not match test.' def test_oneOf_returns_valid(): @@ -244,7 +244,7 @@ def test_oneOf_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 64 is not in [17, 24, 36].' + assert error == 'T1 Item item_no (friendly_name): 64 is not in [17, 24, 36].' def test_between_returns_valid(): @@ -277,7 +277,7 @@ def test_between_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 47 is not between 48 and 400.' + assert error == 'T1 Item item_no (friendly_name): 47 is not between 48 and 400.' @pytest.mark.parametrize('value, expected_is_valid, expected_error', [ @@ -285,16 +285,16 @@ def test_between_returns_invalid(): (77731, True, None), ('7', True, None), ('234897', True, None), - ('a', False, 'T1 Item item_no friendly_name: a is not a number.'), - ('houston, we have a problem', False, 'T1 Item item_no friendly_name: houston, we have a problem is not a number.'), - (' test', False, 'T1 Item item_no friendly_name: test is not a number.'), + ('a', False, 'T1 Item item_no (friendly_name): a is not a number.'), + ('houston, we have a problem', False, 'T1 Item item_no (friendly_name): houston, we have a problem is not a number.'), + (' test', False, 'T1 Item item_no (friendly_name): test is not a number.'), (' 7 ', True, None), (' 8388323', True, None), ('87932875 ', True, None), (' 00 ', True, None), (' 88 ', True, None), (' 088 ', True, None), - (' 8 8 ', False, 'T1 Item item_no friendly_name: 8 8 is not a number.'), + (' 8 8 ', False, 'T1 Item item_no (friendly_name): 8 8 is not a number.'), ]) def test_isNumber(value, expected_is_valid, expected_error): """Test `isNumber` validator.""" @@ -319,7 +319,7 @@ def test_date_month_is_valid_returns_invalid(): validator = validators.dateMonthIsValid() is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 13 is not a valid month.' + assert error == 'T1 Item item_no (friendly_name): 13 is not a valid month.' def test_date_day_is_valid_returns_valid(): @@ -337,7 +337,7 @@ def test_date_day_is_valid_returns_invalid(): validator = validators.dateDayIsValid() is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 32 is not a valid day.' + assert error == 'T1 Item item_no (friendly_name): 32 is not a valid day.' def test_olderThan(): @@ -351,7 +351,7 @@ def test_olderThan(): result = validator(value, RowSchema(), "friendly_name", "item_no") assert result == ( False, - f"T1 Item item_no friendly_name: {str(value)[:4]} must be less than or equal to " + f"T1 Item item_no (friendly_name): {str(value)[:4]} must be less than or equal to " f"{date.today().year - min_age} to meet the minimum age requirement." ) @@ -366,7 +366,7 @@ def test_dateYearIsLargerThan(): value = 18990101 assert validator(value, RowSchema(), "friendly_name", "item_no") == ( False, - f"T1 Item item_no friendly_name: Year {str(value)[:4]} must be larger than {year}." + f"T1 Item item_no (friendly_name): Year {str(value)[:4]} must be larger than {year}." ) @@ -378,7 +378,7 @@ def test_between_returns_invalid_for_string_value(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 047 is not between 100 and 400.' + assert error == 'T1 Item item_no (friendly_name): 047 is not between 100 and 400.' def test_recordHasLength_returns_valid(): @@ -422,7 +422,7 @@ def test_intHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 1a3 does not have exactly 22 digits.' + assert error == 'T1 Item item_no (friendly_name): 1a3 does not have exactly 22 digits.' def test_contains_returns_valid(): @@ -444,7 +444,7 @@ def test_contains_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 12345abcde does not contain 6789.' + assert error == 'T1 Item item_no (friendly_name): 12345abcde does not contain 6789.' def test_startsWith_returns_valid(): @@ -466,7 +466,7 @@ def test_startsWith_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: 12345abcde does not start with abc.' + assert error == 'T1 Item item_no (friendly_name): 12345abcde does not start with abc.' def test_notEmpty_returns_valid_full_string(): @@ -488,7 +488,7 @@ def test_notEmpty_returns_invalid_full_string(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 Item item_no friendly_name: contains blanks between positions 0 and 9.' + assert error == 'T1 Item item_no (friendly_name): contains blanks between positions 0 and 9.' def test_notEmpty_returns_valid_substring(): @@ -510,7 +510,7 @@ def test_notEmpty_returns_invalid_substring(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == "T1 Item item_no friendly_name: 111 333 contains blanks between positions 3 and 5." + assert error == "T1 Item item_no (friendly_name): 111 333 contains blanks between positions 3 and 5." def test_notEmpty_returns_nonexistent_substring(): @@ -521,7 +521,7 @@ def test_notEmpty_returns_nonexistent_substring(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == "T1 Item item_no friendly_name: 111 333 contains blanks between positions 10 and 12." + assert error == "T1 Item item_no (friendly_name): 111 333 contains blanks between positions 10 and 12." @pytest.mark.parametrize("test_input", [1, 2, 3, 4]) @@ -541,7 +541,7 @@ def test_quarterIsValid_returns_false_if_invalid(test_input): is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == f"T1 Item item_no friendly_name: {test_input} is not a valid quarter." + assert error == f"T1 Item item_no (friendly_name): {test_input} is not a valid quarter." @pytest.mark.parametrize("value", ["T72020 ", "T720194", "T720200", "T720207", "T72020$"]) def test_calendarQuarterIsValid_returns_invalid(value): diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index fc17998c4..41f966885 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -255,7 +255,7 @@ def calendarQuarterIsValid(start=0, end=None): def format_error_context(row_schema, friendly_name, item_num): """Format the error message for consistency across cat2 validators.""" - return f'{row_schema.record_type} Item {item_num} {friendly_name}' + return f'{row_schema.record_type} Item {item_num} ({friendly_name})' def matches(option, error_func=None): From 01af1d2dc1f7cc5153aeb5bb48e26e24238a9274 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 2 May 2024 14:24:04 -0400 Subject: [PATCH 06/52] lint --- tdrs-backend/tdpservice/parsers/test/test_validators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 3d341a743..3433e8d9e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -286,7 +286,10 @@ def test_between_returns_invalid(): ('7', True, None), ('234897', True, None), ('a', False, 'T1 Item item_no (friendly_name): a is not a number.'), - ('houston, we have a problem', False, 'T1 Item item_no (friendly_name): houston, we have a problem is not a number.'), + ( + 'houston, we have a problem', False, + 'T1 Item item_no (friendly_name): houston, we have a problem is not a number.' + ), (' test', False, 'T1 Item item_no (friendly_name): test is not a number.'), (' 7 ', True, None), (' 8388323', True, None), From 97aa0ef47ca59d24625ebe86e4a65b76b69a8459 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 16 May 2024 11:10:33 -0400 Subject: [PATCH 07/52] fix tests --- .../tdpservice/parsers/test/test_parse.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index ffe0d2396..a807d303e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -236,7 +236,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER: record length is 24 characters but must be 23.' + assert err.error_message == 'HEADER record length is 24 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -265,7 +265,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER: record length is 14 characters but must be 23.' + assert err.error_message == 'HEADER record length is 14 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -347,7 +347,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): trailer_error = parser_errors.get(row_number=3) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'TRAILER: record length is 11 characters but must be 23.' + assert trailer_error.error_message == 'TRAILER record length is 11 characters but must be 23.' assert trailer_error.content_type is None assert trailer_error.object_id is None @@ -358,7 +358,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors_list.append(row_error) assert row_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.error_message in [ - 'TRAILER: record length is 11 characters but must be 23.', + 'TRAILER record length is 11 characters but must be 23.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.'] assert row_error.content_type is None assert row_error.object_id is None @@ -372,7 +372,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors = list(parser_errors.filter(row_number=2).order_by("id")) length_error = row_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1: record length is 7 characters but must be 156.' + assert length_error.error_message == 'T1 record length is 7 characters but must be 156.' assert length_error.content_type is None assert length_error.object_id is None assert errors == { @@ -404,7 +404,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): trailer_error_1 = trailer_errors[0] assert trailer_error_1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error_1.error_message == 'TRAILER: record length is 7 characters but must be 23.' + assert trailer_error_1.error_message == 'TRAILER record length is 7 characters but must be 23.' assert trailer_error_1.content_type is None assert trailer_error_1.object_id is None @@ -416,7 +416,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert row_2_error.error_message == 'T1: record length is 117 characters but must be 156.' + assert row_2_error.error_message == 'T1 record length is 117 characters but must be 156.' assert row_2_error.content_type is None assert row_2_error.object_id is None @@ -428,10 +428,10 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_error_list.append(row_3_error) assert row_3_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert row_3_error.error_message in { - 'T1: record length is 7 characters but must be 156.', + 'T1 record length is 7 characters but must be 156.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.', 'T1trash does not start with TRAILER.', - 'TRAILER: record length is 7 characters but must be 23.', + 'TRAILER record length is 7 characters but must be 23.', 'T1: Case number T1trash cannot contain blanks.', 'Your file does not end with a TRAILER record.'} assert row_3_error.content_type is None @@ -450,7 +450,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_errors = [trailer_errors[2], trailer_errors[3]] length_error = row_3_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1: record length is 7 characters but must be 156.' + assert length_error.error_message == 'T1 record length is 7 characters but must be 156.' assert length_error.content_type is None assert length_error.object_id is None @@ -513,7 +513,7 @@ def test_parse_empty_file(empty_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER: record length is 0 characters but must be 23.' + assert err.error_message == 'HEADER record length is 0 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -885,7 +885,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel trailer_error = parser_errors.get( row_number=6, - error_message='TRAILER: record length is 15 characters but must be 23.' + error_message='TRAILER record length is 15 characters but must be 23.' ) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.content_type is None @@ -1685,7 +1685,7 @@ def t3_file_zero_filled_second(): ('t3_file', True, 0), ('t3_file_two_child', True, 1), ('t3_file_two_child_with_space_filled', True, 0), - ('two_child_second_filled', True, 9), + ('two_child_second_filled', True, 8), ('t3_file_zero_filled_second', True, 0)]) @pytest.mark.django_db() def test_misformatted_multi_records(file_fixture, result, number_of_errors, request, dfs): From ad87ab166f276dd582031d5588723d0ed47a4fba Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 16 May 2024 11:25:14 -0400 Subject: [PATCH 08/52] fix more tests --- tdrs-backend/tdpservice/data_files/test/test_api.py | 2 +- tdrs-backend/tdpservice/parsers/test/test_validators.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index faacac4d8..6045dc555 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -112,7 +112,7 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \ "but must be 23." @staticmethod diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 13c5c0b65..044c9d864 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -403,7 +403,7 @@ def test_recordHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: record length is 7 characters but must be 22.' + assert error == 'T1 record length is 7 characters but must be 22.' def test_hasLengthGreaterThan_returns_valid(): """Test `hasLengthGreaterThan` gives a valid result.""" From c2c465f8ce06417cdf962c16dfedbcd8d23af7f3 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 16 May 2024 16:40:59 -0400 Subject: [PATCH 09/52] fix merge duplication --- tdrs-backend/tdpservice/parsers/validators.py | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index ed30a19a5..a4ec1c5f2 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -318,28 +318,6 @@ def hasLengthGreaterThan(val, error_func=None): ) -def recordHasLength(length): - """Validate that value (string or array) has a length matching length param.""" - return make_validator( - lambda value: len(value) == length, - lambda value, - row_schema, - friendly_name, - item_num: f"{row_schema.record_type} record length is {len(value)} characters but must be {length}.", - ) - - -def hasLengthGreaterThan(val, error_func=None): - """Validate that value (string or array) has a length greater than val.""" - return make_validator( - lambda value: len(value) >= val, - lambda value, - row_schema, - friendly_name, - item_num: f"Value length {len(value)} is not greater than {val}.", - ) - - def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( From 68789c3d31ebbe35bd566c0374207ac26b7e0365 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 20 May 2024 13:11:04 -0400 Subject: [PATCH 10/52] fix tests --- .../tdpservice/data_files/test/test_api.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 27 ++++++++++--------- .../parsers/test/test_validators.py | 4 +-- tdrs-backend/tdpservice/parsers/validators.py | 21 ++++++++------- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 6045dc555..faacac4d8 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -112,7 +112,7 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \ + assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ "but must be 23." @staticmethod diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index df78d6aff..7ffcb71a1 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -236,7 +236,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 24 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 24 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -265,7 +265,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 14 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 14 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -347,7 +347,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): trailer_error = parser_errors.get(row_number=3) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'TRAILER record length is 11 characters but must be 23.' + assert trailer_error.error_message == 'TRAILER: record length is 11 characters but must be 23.' assert trailer_error.content_type is None assert trailer_error.object_id is None @@ -358,7 +358,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors_list.append(row_error) assert row_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.error_message in [ - 'TRAILER record length is 11 characters but must be 23.', + 'TRAILER: record length is 11 characters but must be 23.', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.'] assert row_error.content_type is None assert row_error.object_id is None @@ -372,7 +372,7 @@ def test_parse_bad_trailer_file(bad_trailer_file, dfs): row_errors = list(parser_errors.filter(row_number=2).order_by("id")) length_error = row_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == "T1 record length of 7 characters is not in the range [117, 156]." + assert length_error.error_message == "T1: record length of 7 characters is not in the range [117, 156]." assert length_error.content_type is None assert length_error.object_id is None assert errors == { @@ -404,7 +404,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): trailer_error_1 = trailer_errors[0] assert trailer_error_1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error_1.error_message == 'TRAILER record length is 7 characters but must be 23.' + assert trailer_error_1.error_message == 'TRAILER: record length is 7 characters but must be 23.' assert trailer_error_1.content_type is None assert trailer_error_1.object_id is None @@ -416,7 +416,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_2_error.error_message == 'T1: 3 is not larger or equal to 1 and smaller or equal to 2.' + assert row_2_error.error_message == 'T1 Item 13 (receives subsidized housing): 3 is not larger or equal to 1 and smaller or equal to 2.' # catch-rpt-month-year-mismatches row_3_errors = parser_errors.filter(row_number=3) @@ -426,10 +426,10 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_error_list.append(row_3_error) assert row_3_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert row_3_error.error_message in { - 'T1 record length of 7 characters is not in the range [117, 156].', + 'T1: record length of 7 characters is not in the range [117, 156].', 'T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.', 'T1trash does not start with TRAILER.', - 'TRAILER record length is 7 characters but must be 23.', + 'TRAILER: record length is 7 characters but must be 23.', 'T1: Case number T1trash cannot contain blanks.', 'Your file does not end with a TRAILER record.'} assert row_3_error.content_type is None @@ -448,7 +448,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_3_errors = [trailer_errors[2], trailer_errors[3]] length_error = row_3_errors[0] assert length_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert length_error.error_message == 'T1 record length of 7 characters is not in the range [117, 156].' + assert length_error.error_message == 'T1: record length of 7 characters is not in the range [117, 156].' assert length_error.content_type is None assert length_error.object_id is None @@ -511,7 +511,7 @@ def test_parse_empty_file(empty_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'HEADER record length is 0 characters but must be 23.' + assert err.error_message == 'HEADER: record length is 0 characters but must be 23.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -883,7 +883,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel trailer_error = parser_errors.get( row_number=6, - error_message='TRAILER record length is 15 characters but must be 23.' + error_message='TRAILER: record length is 15 characters but must be 23.' ) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert trailer_error.content_type is None @@ -1705,7 +1705,7 @@ def t3_file_zero_filled_second(): 'The second child record is too short at 97 characters' + ' and must be at least 101 characters.'), ('t3_file_two_child_with_space_filled', 2, 0, ''), - ('two_child_second_filled', 2, 9, 'T3: Year 6 must be larger than 1900.'), + ('two_child_second_filled', 2, 8, 'T3 Item 68 (date of birth): Year 6 must be larger than 1900.'), ('t3_file_zero_filled_second', 1, 0, '')]) @pytest.mark.django_db() def test_misformatted_multi_records(file_fixture, result, number_of_errors, error_message, request, dfs): @@ -1721,6 +1721,7 @@ def test_misformatted_multi_records(file_fixture, result, number_of_errors, erro assert parser_errors.count() == number_of_errors if number_of_errors > 0: error_messages = [parser_error.error_message for parser_error in parser_errors] + print(error_messages) assert error_message in error_messages parser_errors = ParserError.objects.all().exclude( diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 66b019226..bf819f1f4 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -403,7 +403,7 @@ def test_recordHasLength_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1 record length is 7 characters but must be 22.' + assert error == 'T1: record length is 7 characters but must be 22.' def test_hasLengthGreaterThan_returns_valid(): """Test `hasLengthGreaterThan` gives a valid result.""" @@ -449,7 +449,7 @@ def test_recordHasLengthBetween_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == f"T1 record length of {len(value)} characters is not in the range [{lower}, {upper}]." + assert error == f"T1: record length of {len(value)} characters is not in the range [{lower}, {upper}]." def test_intHasLength_returns_valid(): diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 32a6de39a..a0baf23fd 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -229,6 +229,17 @@ def recordHasLength(length): ) +def recordHasLengthBetween(lower, upper, error_func=None): + """Validate that value (string or array) has a length matching length param.""" + return make_validator( + lambda value: len(value) >= lower and len(value) <= upper, + lambda value, row_schema, friendly_name, item_num: error_func(value, lower, upper) + if error_func + else + f"{row_schema.record_type}: record length of {len(value)} characters is not in the range [{lower}, {upper}].", + ) + + def caseNumberNotEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( @@ -306,16 +317,6 @@ def between(min, max): f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", ) -def recordHasLengthBetween(lower, upper, error_func=None): - """Validate that value (string or array) has a length matching length param.""" - return make_validator( - lambda value: len(value) >= lower and len(value) <= upper, - lambda value, row_schema, friendly_name, item_num: error_func(value, lower, upper) - if error_func - else - f"{row_schema.record_type} record length of {len(value)} characters is not in the range [{lower}, {upper}].", - ) - def hasLengthGreaterThan(val, error_func=None): """Validate that value (string or array) has a length greater than val.""" return make_validator( From 06677a0466a3ba1037139f92ed0a6ab15645c381 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 20 May 2024 13:12:27 -0400 Subject: [PATCH 11/52] lint --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 7ffcb71a1..6993726fb 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -416,7 +416,10 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_2_error.error_message == 'T1 Item 13 (receives subsidized housing): 3 is not larger or equal to 1 and smaller or equal to 2.' + assert row_2_error.error_message == ( + 'T1 Item 13 (receives subsidized housing): 3 is not ' + 'larger or equal to 1 and smaller or equal to 2.' + ) # catch-rpt-month-year-mismatches row_3_errors = parser_errors.filter(row_number=3) @@ -1705,7 +1708,8 @@ def t3_file_zero_filled_second(): 'The second child record is too short at 97 characters' + ' and must be at least 101 characters.'), ('t3_file_two_child_with_space_filled', 2, 0, ''), - ('two_child_second_filled', 2, 8, 'T3 Item 68 (date of birth): Year 6 must be larger than 1900.'), + ('two_child_second_filled', 2, 8, + 'T3 Item 68 (date of birth): Year 6 must be larger than 1900.'), ('t3_file_zero_filled_second', 1, 0, '')]) @pytest.mark.django_db() def test_misformatted_multi_records(file_fixture, result, number_of_errors, error_message, request, dfs): From 9a96a9338437cbac7b600822440620ebb445dc1b Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 22 May 2024 15:02:35 -0400 Subject: [PATCH 12/52] fix tets --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 10 ++++++++-- .../tdpservice/parsers/test/test_validators.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 117c5656b..9423d74b3 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1015,7 +1015,10 @@ def test_parse_tanf_section2_file(tanf_section2_file, dfs): err = parser_errors.first() assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == "T4: 3 is not larger or equal to 1 and smaller or equal to 2." + assert err.error_message == ( + "T4 Item 10 (receives subsidized housing): 3 " + "is not larger or equal to 1 and smaller or equal to 2." + ) assert err.content_type.model == "tanf_t4" assert err.object_id is not None @@ -1768,7 +1771,10 @@ def test_empty_t4_t5_values(t4_t5_empty_values, dfs): assert t4[0].STRATUM is None logger.info(t4[0].__dict__) assert t5.count() == 1 - assert parser_errors[0].error_message == "T4: 3 is not larger or equal to 1 and smaller or equal to 2." + assert parser_errors[0].error_message == ( + "T4 Item 10 (receives subsidized housing): 3 is " + "not larger or equal to 1 and smaller or equal to 2." + ) @pytest.mark.django_db() diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 28b15b1ae..0ee8815e4 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -262,7 +262,7 @@ def test_oneOf_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == 'T1: 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ + assert error == 'T1 Item item_no (friendly_name): 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ '29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55].' From 725a959384c48f0c913f6b4ba5b14075290dbb00 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 29 May 2024 10:26:56 -0400 Subject: [PATCH 13/52] udpate required validator --- tdrs-backend/tdpservice/parsers/row_schema.py | 7 +++++-- tdrs-backend/tdpservice/parsers/test/test_util.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index c16718383..7500ab593 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -1,7 +1,7 @@ """Row schema for datafile.""" from .models import ParserErrorCategoryChoices from .fields import Field, TransformField -from .validators import value_is_empty +from .validators import value_is_empty, format_error_context import logging logger = logging.getLogger(__name__) @@ -150,7 +150,10 @@ def run_field_validators(self, instance, generate_error): generate_error( schema=self, error_category=ParserErrorCategoryChoices.FIELD_VALUE, - error_message=f"{field.name} is required but a value was not provided.", + error_message=( + f"{format_error_context(self, field.friendly_name, field.item)}: " + "field is required but a value was not provided." + ), record=instance, field=field ) diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index a36dacf27..dd4465e9c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -297,8 +297,8 @@ def test_field_validators_blank_and_required_returns_error(first, second): is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is False assert errors == [ - 'first is required but a value was not provided.', - 'second is required but a value was not provided.' + 'T1 Item 1 (first): field is required but a value was not provided.', + 'T1 Item 2 (second): field is required but a value was not provided.' ] From 3ac7d5f092b42bd9d7d4aa5ad911d6baa70f4e06 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Sun, 2 Jun 2024 16:55:48 -0400 Subject: [PATCH 14/52] add upcoming data deadline notification --- tdrs-backend/tdpservice/email/email_enums.py | 1 + tdrs-backend/tdpservice/email/tasks.py | 78 +++++++++ .../upcoming-submission-deadline.html | 27 ++++ .../test/test_upcoming_deadline_email.py | 148 ++++++++++++++++++ tdrs-backend/tdpservice/settings/common.py | 40 +++++ 5 files changed, 294 insertions(+) create mode 100644 tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html create mode 100644 tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py diff --git a/tdrs-backend/tdpservice/email/email_enums.py b/tdrs-backend/tdpservice/email/email_enums.py index 2f56ee6c5..3b7028681 100644 --- a/tdrs-backend/tdpservice/email/email_enums.py +++ b/tdrs-backend/tdpservice/email/email_enums.py @@ -13,3 +13,4 @@ class EmailType(Enum): REQUEST_DENIED = 'request-denied.html' DEACTIVATION_WARNING = 'account-deactivation-warning.html' ACCOUNT_DEACTIVATED = 'account-deactivated.html' + UPCOMING_SUBMISSION_DEADLINE = 'upcoming-submission-deadline.html' diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 3da03d93d..3167180bd 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -11,6 +11,10 @@ import logging from tdpservice.email.helpers.account_access_requests import send_num_access_requests_email from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email +from tdpservice.stts.models import STT +from tdpservice.data_files.models import DataFile +from .email import automated_email +from .email_enums import EmailType logger = logging.getLogger(__name__) @@ -69,3 +73,77 @@ def email_admin_num_access_requests(): subject, email_context, ) + + +@shared_task +def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): + """Send all Data Analysts a reminder to submit if they have not already.""" + def get_fiscal_year(calendar_year, fiscal_quarter): + return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year + + now = datetime.now() + fiscal_year = get_fiscal_year(now.year, fiscal_quarter) + + all_locations = STT.objects.all() + + # + reminder_locations = [] + for loc in all_locations: + submitted_sections = DataFile.objects.all().filter( + stt=loc, + year=fiscal_year, + quarter=fiscal_quarter, + # version=?? + ) + required_sections = loc.filenames.keys() + + submitted_all_sections = True + for s in required_sections: + if not submitted_sections.filter(section=s).exists(): + submitted_all_sections = False + + if not submitted_all_sections: + reminder_locations.append(loc) + + # + # locations_with_no_submission_for_data_period = all_locations.filter() # -> ? + # spike ticket for improvement and/or iterate + + template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value + text_message = 'Your datafiles are due in five days.' + subject = f'Upcoming submission deadline: {due_date}' + + for loc in reminder_locations: + recipients = User.objects.filter( + stt=loc, + account_approval_status=AccountApprovalStatusChoices.APPROVED, + groups=Group.objects.get(name='Data Analyst') + ) + + for rec in recipients: + context = { + 'first_name': rec.first_name, + 'fiscal_year': fiscal_year, + 'fiscal_quarter': fiscal_quarter, + 'submission_deadline': due_date, + 'url': settings.FRONTEND_BASE_URL, + 'subject': subject + } + + logger_context = { + 'user_id': rec.id, + 'object_id': rec.id, + 'object_repr': rec.username, + } + + automated_email( + email_path=template_path, + recipient_email=rec.username, + subject=subject, + email_context=context, + text_message=text_message, + logger_context=logger_context + ) + + if len(recipients) == 0: + print(f'{loc.name} needs a reminder email but has no recipients') diff --git a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html new file mode 100644 index 000000000..72ac24b57 --- /dev/null +++ b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html @@ -0,0 +1,27 @@ +{% extends 'base.html' %} +{% block content %} + + +

Hello {{ first_name }},

+

This is a friendly reminder that your data files are due in 5 days.

+

Please sign in to the TANF Data Portal +to upload and submit data files for Fiscal Year {{ fiscal_year }} - {{ fiscal_quarter}} by {{ submission_deadline }}.

+ + + + +

Submit your data files

+ +Need help?
+We're here for you! Check out the TDP Knowledge Center for specific gudiance on Submitting Data Files and Frequently Asked Questions. +

+TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Reach out to the TDP support team by replying to this email and we will follow up with you directly.

+

Thank you,

+

The TDP Team

+ + + + + + +{% endblock %} diff --git a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py new file mode 100644 index 000000000..eb40906c3 --- /dev/null +++ b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py @@ -0,0 +1,148 @@ +"""Test function for sending upcoming data deadline reminders.""" +import pytest +from django.core import mail +from tdpservice.email.tasks import send_data_submission_reminder +from datetime import datetime +from django.contrib.auth.models import Group + +from tdpservice.stts.models import STT +from tdpservice.users.models import User +from tdpservice.data_files.models import DataFile + + +@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ + ('February 14', 'Oct - Dec', 'Q1'), + ('May 15th', 'Jan - Mar', 'Q2'), + ('August 14th', 'Apr - Jun', 'Q3'), + ('November 14th', 'Jul - Sep', 'Q4'), +]) +@pytest.mark.django_db +def test_upcoming_deadline_sends_no_sections_submitted( + due_date, reporting_period, fiscal_quarter): + """Test that the send_deactivation_warning_email function runs when no sections have been submitted.""" + + stt = STT.objects.create( + name='Arkansas', + filenames={ + "Active Case Data": "test-filename.txt", + "Closed Case Data": "test-filename-closed.txt", + } + ) + + data_analyst = User.objects.create( + username='test@email.com', + stt=stt, + account_approval_status='Approved' + ) + data_analyst.groups.add(Group.objects.get(name='Data Analyst')) + data_analyst.save() + + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + + assert len(mail.outbox) == 1 + assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" + + +@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ + ('February 14', 'Oct - Dec', 'Q1'), + ('May 15th', 'Jan - Mar', 'Q2'), + ('August 14th', 'Apr - Jun', 'Q3'), + ('November 14th', 'Jul - Sep', 'Q4'), +]) +@pytest.mark.django_db +def test_upcoming_deadline_sends_some_sections_submitted( + due_date, reporting_period, fiscal_quarter): + """Test that the send_deactivation_warning_email function runs when some sections have been submitted.""" + + stt = STT.objects.create( + name='Arkansas', + filenames={ + "Active Case Data": "test-filename.txt", + "Closed Case Data": "test-filename-closed.txt", + } + ) + + data_analyst = User.objects.create( + username='test@email.com', + stt=stt, + account_approval_status='Approved' + ) + data_analyst.groups.add(Group.objects.get(name='Data Analyst')) + data_analyst.save() + + now = datetime.now() + fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year + + data_file = DataFile.create_new_version({ + "section": 'Active Case Data', + "quarter": fiscal_quarter, + "year": fiscal_year, + "stt": stt, + "user": data_analyst, + }) + + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + + assert len(mail.outbox) == 1 + assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" + + +"""Test function for sending upcoming data deadline reminders.""" +import pytest +from django.core import mail +from tdpservice.email.tasks import send_data_submission_reminder +from django.contrib.auth.models import Group + +from tdpservice.stts.models import STT +from tdpservice.users.models import User + + +@pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ + ('February 14', 'Oct - Dec', 'Q1'), + ('May 15th', 'Jan - Mar', 'Q2'), + ('August 14th', 'Apr - Jun', 'Q3'), + ('November 14th', 'Jul - Sep', 'Q4'), +]) +@pytest.mark.django_db +def test_upcoming_deadline_no_send_when_all_sections_complete( + due_date, reporting_period, fiscal_quarter): + """Test that the send_deactivation_warning_email function does not run when all sections have been submitted.""" + + stt = STT.objects.create( + name='Arkansas', + filenames={ + "Active Case Data": "test-filename.txt", + "Closed Case Data": "test-filename-closed.txt", + } + ) + + data_analyst = User.objects.create( + username='test@email.com', + stt=stt, + account_approval_status='Approved' + ) + data_analyst.groups.add(Group.objects.get(name='Data Analyst')) + data_analyst.save() + + now = datetime.now() + fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year + + data_file = DataFile.create_new_version({ + "section": 'Active Case Data', + "quarter": fiscal_quarter, + "year": fiscal_year, + "stt": stt, + "user": data_analyst, + }) + + data_file2 = DataFile.create_new_version({ + "section": 'Closed Case Data', + "quarter": fiscal_quarter, + "year": fiscal_year, + "stt": stt, + "user": data_analyst, + }) + + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + + assert len(mail.outbox) == 0 diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 6ca924fe0..13875a52d 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -489,6 +489,46 @@ class Common(Configuration): 'task': 'tdpservice.email.tasks.email_admin_num_access_requests', 'schedule': crontab(minute='0', hour='1', day_of_week='*', day_of_month='*', month_of_year='*'), # Every day at 1am UTC (9pm EST) }, + 'Email Data Analyst Q1 Upcoming Submission Deadline Reminder': { + 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + 'schedule': crontab(month_of_year='2', day_of_month='9', hour='13', minute='0'), + # 'schedule': crontab(minute='*/3'), + 'kwargs': { + 'due_date': 'February 14th', + 'reporting_period': 'Oct - Dec', + 'fiscal_quarter': 'Q1', + } + }, + 'Email Data Analyst Q2 Upcoming Submission Deadline Reminder': { + 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + 'schedule': crontab(month_of_year='5', day_of_month='10', hour='13', minute='0'), + # 'schedule': crontab(minute='*/3'), + 'kwargs': { + 'due_date': 'May 15th', + 'reporting_period': 'Jan - Mar', + 'fiscal_quarter': 'Q2', + } + }, + 'Email Data Analyst Q3 Upcoming Submission Deadline Reminder': { + 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + 'schedule': crontab(month_of_year='8', day_of_month='9', hour='13', minute='0'), + # 'schedule': crontab(minute='*/3'), + 'kwargs': { + 'due_date': 'August 14th', + 'reporting_period': 'Apr - Jun', + 'fiscal_quarter': 'Q3', + } + }, + 'Email Data Analyst Q4 Upcoming Submission Deadline Reminder': { + 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + 'schedule': crontab(month_of_year='11', day_of_month='9', hour='13', minute='0'), + # 'schedule': crontab(minute='*/3'), + 'kwargs': { + 'due_date': 'November 14th', + 'reporting_period': 'Jul - Sep', + 'fiscal_quarter': 'Q4', + } + }, } CYPRESS_TOKEN = os.getenv('CYPRESS_TOKEN', None) From 24bceb9d5e6794a7e6b76f3bcb20790f2f8739f5 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Sun, 2 Jun 2024 17:06:10 -0400 Subject: [PATCH 15/52] lint --- .../test/test_upcoming_deadline_email.py | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py index eb40906c3..b6511505d 100644 --- a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py +++ b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py @@ -1,10 +1,9 @@ """Test function for sending upcoming data deadline reminders.""" import pytest +from datetime import datetime from django.core import mail from tdpservice.email.tasks import send_data_submission_reminder -from datetime import datetime from django.contrib.auth.models import Group - from tdpservice.stts.models import STT from tdpservice.users.models import User from tdpservice.data_files.models import DataFile @@ -20,7 +19,6 @@ def test_upcoming_deadline_sends_no_sections_submitted( due_date, reporting_period, fiscal_quarter): """Test that the send_deactivation_warning_email function runs when no sections have been submitted.""" - stt = STT.objects.create( name='Arkansas', filenames={ @@ -53,7 +51,6 @@ def test_upcoming_deadline_sends_no_sections_submitted( def test_upcoming_deadline_sends_some_sections_submitted( due_date, reporting_period, fiscal_quarter): """Test that the send_deactivation_warning_email function runs when some sections have been submitted.""" - stt = STT.objects.create( name='Arkansas', filenames={ @@ -73,7 +70,7 @@ def test_upcoming_deadline_sends_some_sections_submitted( now = datetime.now() fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year - data_file = DataFile.create_new_version({ + _ = DataFile.create_new_version({ "section": 'Active Case Data', "quarter": fiscal_quarter, "year": fiscal_year, @@ -87,16 +84,6 @@ def test_upcoming_deadline_sends_some_sections_submitted( assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" -"""Test function for sending upcoming data deadline reminders.""" -import pytest -from django.core import mail -from tdpservice.email.tasks import send_data_submission_reminder -from django.contrib.auth.models import Group - -from tdpservice.stts.models import STT -from tdpservice.users.models import User - - @pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ ('February 14', 'Oct - Dec', 'Q1'), ('May 15th', 'Jan - Mar', 'Q2'), @@ -107,7 +94,6 @@ def test_upcoming_deadline_sends_some_sections_submitted( def test_upcoming_deadline_no_send_when_all_sections_complete( due_date, reporting_period, fiscal_quarter): """Test that the send_deactivation_warning_email function does not run when all sections have been submitted.""" - stt = STT.objects.create( name='Arkansas', filenames={ @@ -127,7 +113,7 @@ def test_upcoming_deadline_no_send_when_all_sections_complete( now = datetime.now() fiscal_year = now.year - 1 if fiscal_quarter == 'Q1' else now.year - data_file = DataFile.create_new_version({ + _ = DataFile.create_new_version({ "section": 'Active Case Data', "quarter": fiscal_quarter, "year": fiscal_year, @@ -135,7 +121,7 @@ def test_upcoming_deadline_no_send_when_all_sections_complete( "user": data_analyst, }) - data_file2 = DataFile.create_new_version({ + _ = DataFile.create_new_version({ "section": 'Closed Case Data', "quarter": fiscal_quarter, "year": fiscal_year, From fd351e30b3324556eb61d524fadbefca13c1bfd9 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 13:52:15 -0400 Subject: [PATCH 16/52] query efficiency --- tdrs-backend/tdpservice/email/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 3167180bd..c9d6c5e21 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -94,12 +94,12 @@ def get_fiscal_year(calendar_year, fiscal_quarter): year=fiscal_year, quarter=fiscal_quarter, # version=?? - ) + ).values_list('section', flat=True).distinct() required_sections = loc.filenames.keys() submitted_all_sections = True for s in required_sections: - if not submitted_sections.filter(section=s).exists(): + if s not in submitted_sections: submitted_all_sections = False if not submitted_all_sections: From c34dd0411dc8761a466344707ca6c6a5395437ec Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 14:52:13 -0400 Subject: [PATCH 17/52] query efficiency --- tdrs-backend/tdpservice/email/tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index c9d6c5e21..dd5fbed18 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -88,13 +88,13 @@ def get_fiscal_year(calendar_year, fiscal_quarter): # reminder_locations = [] + year_quarter_files = DataFile.objects.all().filter( + year=fiscal_year, + quarter=fiscal_quarter, + ) + for loc in all_locations: - submitted_sections = DataFile.objects.all().filter( - stt=loc, - year=fiscal_year, - quarter=fiscal_quarter, - # version=?? - ).values_list('section', flat=True).distinct() + submitted_sections = year_quarter_files.filter(stt=loc).values_list('section', flat=True).distinct() required_sections = loc.filenames.keys() submitted_all_sections = True From cb8b37a81a24402819a39062521ab9efb61cad1f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 14:52:19 -0400 Subject: [PATCH 18/52] clarity comment --- tdrs-backend/tdpservice/settings/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 13875a52d..4a12a855c 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -491,6 +491,7 @@ class Common(Configuration): }, 'Email Data Analyst Q1 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + # Feb 9 at 1pm UTC (9am EST) 'schedule': crontab(month_of_year='2', day_of_month='9', hour='13', minute='0'), # 'schedule': crontab(minute='*/3'), 'kwargs': { @@ -501,6 +502,7 @@ class Common(Configuration): }, 'Email Data Analyst Q2 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + # May 10 at 1pm UTC (9am EST) 'schedule': crontab(month_of_year='5', day_of_month='10', hour='13', minute='0'), # 'schedule': crontab(minute='*/3'), 'kwargs': { @@ -511,6 +513,7 @@ class Common(Configuration): }, 'Email Data Analyst Q3 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + # Aug 9 at 1pm UTC (9am EST) 'schedule': crontab(month_of_year='8', day_of_month='9', hour='13', minute='0'), # 'schedule': crontab(minute='*/3'), 'kwargs': { @@ -521,6 +524,7 @@ class Common(Configuration): }, 'Email Data Analyst Q4 Upcoming Submission Deadline Reminder': { 'task': 'tdpservice.email.tasks.send_data_submission_reminder', + # Nov 9 at 1pm UTC (9am EST) 'schedule': crontab(month_of_year='11', day_of_month='9', hour='13', minute='0'), # 'schedule': crontab(minute='*/3'), 'kwargs': { From e1296c74c65a59898005546a374afb13435e79b8 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 14:55:54 -0400 Subject: [PATCH 19/52] add no recips logentry --- tdrs-backend/tdpservice/email/tasks.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index dd5fbed18..20216ba68 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -13,7 +13,7 @@ from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email from tdpservice.stts.models import STT from tdpservice.data_files.models import DataFile -from .email import automated_email +from .email import automated_email, log from .email_enums import EmailType @@ -146,4 +146,13 @@ def get_fiscal_year(calendar_year, fiscal_quarter): ) if len(recipients) == 0: - print(f'{loc.name} needs a reminder email but has no recipients') + system_user, created = User.objects.get_or_create(username='system') + if created: + log('Created reserved system user.') + + logger_context = { + 'user_id': system_user.pk, + 'object_id': loc.id, + 'object_repr': loc.name, + } + log(f"{loc.name} has no recipients for data submission deadline reminder.", logger_context=logger_context) From 29bc6697c199bdeae7f3b9f5dc70bde376f24449 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 17:20:00 -0400 Subject: [PATCH 20/52] rm comments --- tdrs-backend/tdpservice/email/tasks.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 20216ba68..fe0aafd22 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -86,7 +86,6 @@ def get_fiscal_year(calendar_year, fiscal_quarter): all_locations = STT.objects.all() - # reminder_locations = [] year_quarter_files = DataFile.objects.all().filter( year=fiscal_year, @@ -105,13 +104,9 @@ def get_fiscal_year(calendar_year, fiscal_quarter): if not submitted_all_sections: reminder_locations.append(loc) - # - # locations_with_no_submission_for_data_period = all_locations.filter() # -> ? - # spike ticket for improvement and/or iterate - template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value - text_message = 'Your datafiles are due in five days.' - subject = f'Upcoming submission deadline: {due_date}' + text_message = f'Your datafiles are due by {due_date}.' + subject = f'Action Requested: Please submit your {TANF/SSP} data files' for loc in reminder_locations: recipients = User.objects.filter( From 5c52b060283c874820189cce844f3d5f30767781 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 17:22:53 -0400 Subject: [PATCH 21/52] update email subject to include ssp/tanf --- tdrs-backend/tdpservice/email/tasks.py | 4 +++- .../tdpservice/email/test/test_upcoming_deadline_email.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index fe0aafd22..4abd58c89 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -106,9 +106,11 @@ def get_fiscal_year(calendar_year, fiscal_quarter): template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value text_message = f'Your datafiles are due by {due_date}.' - subject = f'Action Requested: Please submit your {TANF/SSP} data files' for loc in reminder_locations: + tanf_ssp_label = 'SSP' if loc.ssp else 'TANF' + subject = f'Action Requested: Please submit your {tanf_ssp_label} data files' + recipients = User.objects.filter( stt=loc, account_approval_status=AccountApprovalStatusChoices.APPROVED, diff --git a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py index b6511505d..1c7b224dd 100644 --- a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py +++ b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py @@ -38,7 +38,7 @@ def test_upcoming_deadline_sends_no_sections_submitted( send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) assert len(mail.outbox) == 1 - assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" + assert mail.outbox[0].subject == "Action Requested: Please submit your TANF data files" @pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ @@ -81,7 +81,7 @@ def test_upcoming_deadline_sends_some_sections_submitted( send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) assert len(mail.outbox) == 1 - assert mail.outbox[0].subject == f"Upcoming submission deadline: {due_date}" + assert mail.outbox[0].subject == "Action Requested: Please submit your TANF data files" @pytest.mark.parametrize('due_date, reporting_period, fiscal_quarter', [ From 9d9b1412f99154adc319152f77101e8ea9087c08 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 3 Jun 2024 17:26:36 -0400 Subject: [PATCH 22/52] SSP -> TANF and SSP --- tdrs-backend/tdpservice/email/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 4abd58c89..92ff0b9e8 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -108,7 +108,7 @@ def get_fiscal_year(calendar_year, fiscal_quarter): text_message = f'Your datafiles are due by {due_date}.' for loc in reminder_locations: - tanf_ssp_label = 'SSP' if loc.ssp else 'TANF' + tanf_ssp_label = 'TANF and SSP' if loc.ssp else 'TANF' subject = f'Action Requested: Please submit your {tanf_ssp_label} data files' recipients = User.objects.filter( From 3be46db469d66c6fb7c5dc6a223312e6b22ebdaf Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 4 Jun 2024 11:35:29 -0400 Subject: [PATCH 23/52] query efficiency --- tdrs-backend/tdpservice/email/tasks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 92ff0b9e8..480c971f7 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -107,15 +107,16 @@ def get_fiscal_year(calendar_year, fiscal_quarter): template_path = EmailType.UPCOMING_SUBMISSION_DEADLINE.value text_message = f'Your datafiles are due by {due_date}.' + all_data_analysts = User.objects.all().filter( + account_approval_status=AccountApprovalStatusChoices.APPROVED, + groups=Group.objects.get(name='Data Analyst') + ) + for loc in reminder_locations: tanf_ssp_label = 'TANF and SSP' if loc.ssp else 'TANF' subject = f'Action Requested: Please submit your {tanf_ssp_label} data files' - recipients = User.objects.filter( - stt=loc, - account_approval_status=AccountApprovalStatusChoices.APPROVED, - groups=Group.objects.get(name='Data Analyst') - ) + recipients = all_data_analysts.filter(stt=loc) for rec in recipients: context = { From 799ce28e96edc215d5f054d8e5ae7d9bfeac19e0 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 5 Jun 2024 12:18:13 -0400 Subject: [PATCH 24/52] Merge branch 'develop' into 2693-cat2-messaging-cleanup --- ...4-configuration-by-environment-variable.md | 4 +- scripts/deploy-backend.sh | 1 + tdrs-backend/docs/api/authentication.md | 11 ++ .../tdpservice/data_files/test/test_api.py | 24 ++-- tdrs-backend/tdpservice/data_files/util.py | 38 ++++-- .../tdpservice/parsers/schema_defs/ssp/m1.py | 13 +- .../tdpservice/parsers/schema_defs/ssp/m2.py | 10 +- .../tdpservice/parsers/schema_defs/ssp/m3.py | 14 +-- .../tdpservice/parsers/schema_defs/ssp/m4.py | 9 +- .../tdpservice/parsers/schema_defs/ssp/m5.py | 8 +- .../tdpservice/parsers/schema_defs/ssp/m6.py | 8 +- .../tdpservice/parsers/schema_defs/ssp/m7.py | 8 +- .../tdpservice/parsers/schema_defs/tanf/t1.py | 10 +- .../tdpservice/parsers/schema_defs/tanf/t2.py | 1 - .../tdpservice/parsers/schema_defs/tanf/t3.py | 4 +- .../tdpservice/parsers/schema_defs/tanf/t4.py | 9 +- .../tdpservice/parsers/schema_defs/tanf/t5.py | 1 - .../parsers/schema_defs/tribal_tanf/t1.py | 14 +-- .../parsers/schema_defs/tribal_tanf/t2.py | 17 +-- .../parsers/schema_defs/tribal_tanf/t3.py | 8 +- .../parsers/schema_defs/tribal_tanf/t4.py | 12 +- .../parsers/schema_defs/tribal_tanf/t5.py | 10 +- .../tdpservice/parsers/test/conftest.py | 64 ++++++++++ .../parsers/test/data/ADS.E2J.FTP2.TS142.txt | 12 +- .../tdpservice/parsers/test/test_parse.py | 84 ++++++++++++- .../parsers/test/test_transforms.py | 22 +++- .../parsers/test/test_validators.py | 28 ----- tdrs-backend/tdpservice/parsers/transforms.py | 6 + tdrs-backend/tdpservice/parsers/validators.py | 112 ++---------------- .../0028_education_level_to_string.py | 23 ++++ .../tdpservice/search_indexes/models/ssp.py | 4 +- .../tdpservice/security/test/test_views.py | 66 +++++++++++ tdrs-backend/tdpservice/security/urls.py | 12 ++ tdrs-backend/tdpservice/security/utils.py | 43 +++++++ tdrs-backend/tdpservice/security/views.py | 36 ++++++ tdrs-backend/tdpservice/settings/common.py | 4 +- tdrs-backend/tdpservice/urls.py | 1 + .../SubmissionHistory/CaseAggregatesTable.jsx | 2 +- .../SubmissionHistory/SubmissionHistory.jsx | 36 ++++-- .../SubmissionHistory.test.js | 4 +- .../TotalAggregatesTable.jsx | 2 +- 41 files changed, 544 insertions(+), 251 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/test/conftest.py create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py create mode 100644 tdrs-backend/tdpservice/security/test/test_views.py create mode 100644 tdrs-backend/tdpservice/security/urls.py create mode 100644 tdrs-backend/tdpservice/security/utils.py create mode 100644 tdrs-backend/tdpservice/security/views.py diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md b/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md index dbd6920c1..95c1a4604 100644 --- a/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md +++ b/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md @@ -9,9 +9,11 @@ Accepted Applications need to be configured differently depending on where they are running. For example, the backend running locally will have different configuration then the backend running in production. +Further, environment variables can be designated "secret" or not; the term "secret key" is often used in place of secret environment variables. Secret keys are sometimes (but not always) shared between different deployment environments, which makes it useful to have a central "single source of truth" where a secret key can be kept and copied out to different environments. CircleCI solves this use case for us, allowing secret keys to be managed by the project's Environment Variables, and accessed in the deployment process to write to cloud.gov applications. + ## Decision -We will use environment variables to configure applications. +We will use environment variables to configure applications. We will use Environment Variables in CircleCI to store and manage secret keys. ## Consequences diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 7a742ad79..3f53b6b59 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -62,6 +62,7 @@ set_cf_envs() "REDIS_URI" "JWT_KEY" "STAGING_JWT_KEY" + "SENDGRID_API_KEY" ) echo "Setting environment variables for $CGAPPNAME_BACKEND" diff --git a/tdrs-backend/docs/api/authentication.md b/tdrs-backend/docs/api/authentication.md index e51f2174d..177c23213 100644 --- a/tdrs-backend/docs/api/authentication.md +++ b/tdrs-backend/docs/api/authentication.md @@ -4,3 +4,14 @@ For clients to authenticate, they have to authenticate with Login.gov via the ba This will allow the backend to identify the browser which requested access and authorize them based on the cookie they provide in their API calls. The secured portion of this authorization is due to the httpOnly cookie being inaccessible to the client's local browser. + +# Generating API token + +In order to use APIs, an activated `OFA Sys Admin` user has to generate a new token and use it in the API request following these steps: +1. User has to first login using frontend and going through the normal login process +2. After user is logged in, user can grab a token at `/v1/security/get-token` +3. The token then can be used in authorization header. As an example: + +```curl --location 'http://{host}/v1/users/' --header 'Authorization: Token {token}'``` + +Note: the authentication token is available for 24 hours by default but this can be overridden using the `TOKEN_EXPIRATION_HOURS` environment variable. diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index faacac4d8..d08ce3f1d 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -96,11 +96,11 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): """Assert the error report file contents match expected with friendly names.""" ws = DataFileAPITestBase.get_spreadsheet(response) - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ + " then number of months T1 Item -1 (number of months): 0 is not larger than 0." @staticmethod @@ -108,11 +108,11 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): """Assert the error report file contents match expected with friendly names.""" ws = DataFileAPITestBase.get_spreadsheet(response) - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=7, column=COL_ERROR_MESSAGE).value == "TRAILER: record length is 15 characters " + \ "but must be 23." @staticmethod @@ -128,11 +128,11 @@ def assert_error_report_file_content_matches_without_friendly_names(response): wb = openpyxl.load_workbook('mycls.xlsx') ws = wb.get_sheet_by_name('Sheet1') - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == ( + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( "if CASH_AMOUNT :873 validator1 passed then " "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not larger than 0." ) diff --git a/tdrs-backend/tdpservice/data_files/util.py b/tdrs-backend/tdpservice/data_files/util.py index 3c5470c13..17beb90aa 100644 --- a/tdrs-backend/tdpservice/data_files/util.py +++ b/tdrs-backend/tdpservice/data_files/util.py @@ -36,27 +36,51 @@ def format_error_msg(x): output = BytesIO() workbook = xlsxwriter.Workbook(output) worksheet = workbook.add_worksheet() + report_columns = [ ('case_number', lambda x: x['case_number']), ('year', lambda x: str(x['rpt_month_year'])[0:4] if x['rpt_month_year'] else None), ('month', lambda x: calendar.month_name[ int(str(x['rpt_month_year'])[4:]) ] if x['rpt_month_year'] else None), - ('error_type', lambda x: x['error_type']), ('error_message', lambda x: format_error_msg(chk(x))), ('item_number', lambda x: x['item_number']), ('item_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].values()])), ('internal_variable_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].keys()])), ('row_number', lambda x: x['row_number']), - ('column_number', lambda x: x['column_number']) ] # write beta banner - worksheet.write(row, col, - "Error reporting in TDP is still in development." + - "We'll be in touch when it's ready to use!" + - "For now please refer to the reports you receive via email") + worksheet.write( + row, col, + "Please refer to the most recent versions of the coding " + + "instructions (linked below) when looking up items " + + "and allowable values during the data revision process" + ) + + row, col = 1, 0 + worksheet.write_url( + row, col, + 'https://www.acf.hhs.gov/ofa/policy-guidance/tribal-tanf-data-coding-instructions', + string='For Tribal TANF data reports: Tribal TANF Instructions', + ) + row, col = 2, 0 + worksheet.write_url( + row, col, + 'https://www.acf.hhs.gov/ofa/policy-guidance/acf-ofa-pi-23-04', + string='For TANF and SSP-MOE data reports: TANF / SSP-MOE (ACF-199 / ACF-209) Instructions' + ) + + row, col = 3, 0 + worksheet.write_url( + row, col, + 'https://tdp-project-updates.app.cloud.gov/knowledge-center/viewing-error-reports.html', + string='Visit the Knowledge Center for further guidance on reviewing error reports' + ) + + row, col = 5, 0 + # write csv header bold = workbook.add_format({'bold': True}) @@ -68,7 +92,7 @@ def format_header(header_list: list): [worksheet.write(row, col, format_header(key[0]), bold) for col, key in enumerate(report_columns)] [ - worksheet.write(row + 3, col, key[1](data_i)) for col, key in enumerate(report_columns) + worksheet.write(row + 6, col, key[1](data_i)) for col, key in enumerate(report_columns) for row, data_i in enumerate(data) ] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 5c2ac878c..338a97216 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -1,7 +1,7 @@ """Schema for SSP M1 record type.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M1DataSubmissionDocument @@ -133,15 +133,16 @@ required=True, validators=[validators.notEmpty()] ), - Field( + TransformField( + zero_pad(3), item="2", - name='COUNTY_FIPS_CODE', + name="COUNTY_FIPS_CODE", friendly_name="county fips code", - type='string', + type="string", startIndex=19, endIndex=22, required=True, - validators=[validators.isNumber(),] + validators=[validators.isNumber()], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 0f72a48e1..c44bfc76a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -370,14 +370,14 @@ item="37", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=55, endIndex=57, required=False, validators=[ validators.or_validators( - validators.isInLimits(0, 16), validators.isInLimits(98, 99) - ) + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) + ), ] ), Field( @@ -388,7 +388,7 @@ startIndex=57, endIndex=58, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="39", @@ -398,7 +398,7 @@ startIndex=58, endIndex=59, required=False, - validators=[validators.oneOf([0, 1, 2, 9])] + validators=[validators.oneOf([1, 2, 9])] ), Field( item="40", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index 19069089b..12376bbfc 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -279,15 +279,15 @@ item="68", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=49, endIndex=51, required=True, validators=[ validators.or_validators( - validators.isInStringRange(0, 16), + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) - ) + ), ] ), Field( @@ -298,7 +298,7 @@ startIndex=51, endIndex=52, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="70A", @@ -593,13 +593,13 @@ item="68", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=90, endIndex=92, required=True, validators=[ validators.or_validators( - validators.isInStringRange(0, 16), + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) ) ] @@ -612,7 +612,7 @@ startIndex=92, endIndex=93, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="70A", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 792ac2ba6..705e2592a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -1,7 +1,7 @@ """Schema for SSP M1 record type.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M4DataSubmissionDocument @@ -54,7 +54,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -62,7 +63,7 @@ startIndex=19, endIndex=22, required=True, - validators=[validators.isInStringRange(0, 999)], + validators=[validators.isNumber()], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index 7ba6aa2c6..080716cb0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -347,7 +347,8 @@ validators.or_validators( validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), - ) + ), + validators.notMatches("00") ], ), Field( @@ -359,10 +360,7 @@ endIndex=57, required=False, validators=[ - validators.or_validators( - validators.isInLimits(0, 3), - validators.matches(9) - ) + validators.oneOf([1, 2, 3, 9]), ], ), Field( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index e85bc8fb1..69d1bda7a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -1,10 +1,10 @@ """Schema for HEADER row of all submission types.""" -from ...transforms import calendar_quarter_to_rpt_month_year -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument s1 = RowSchema( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 8d6664a43..39ecf8f84 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -1,9 +1,9 @@ """Schema for TANF T7 Row.""" -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ...transforms import calendar_quarter_to_rpt_month_year -from ... import validators +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument schemas = [] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index dc5d153cb..371b29df6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -1,6 +1,7 @@ """Schema for t1 record types.""" -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T1DataSubmissionDocument @@ -155,7 +156,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -163,9 +165,7 @@ startIndex=19, endIndex=22, required=True, - validators=[ - validators.isNumber(), - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 47b6e9144..683649a9b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -98,7 +98,6 @@ result_field_name="COOPERATION_CHILD_SUPPORT", result_function=validators.oneOf((1, 2, 9)), ), - validators.validate__FAM_AFF__HOH__Fed_Time(), validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.isInLimits(1, 3), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 4bc9f6195..5cf53bc6a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -295,7 +295,7 @@ startIndex=51, endIndex=52, required=False, - validators=[validators.oneOf([0, 1, 2, 9])], + validators=[validators.oneOf([1, 2, 9])], ), Field( item="77A", @@ -608,7 +608,7 @@ startIndex=92, endIndex=93, required=False, - validators=[validators.oneOf([0, 1, 2, 9])], + validators=[validators.oneOf([1, 2, 9])], ), Field( item="77A", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index 2828e2a8f..2de7ea71c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -1,7 +1,7 @@ """Schema for HEADER row of all submission types.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T4DataSubmissionDocument @@ -55,7 +55,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -63,7 +64,7 @@ startIndex=19, endIndex=22, required=True, - validators=[validators.isInStringRange(1, 999)], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index fa0e1792c..df9bf9ce2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -92,7 +92,6 @@ result_field_name="CITIZENSHIP_STATUS", result_function=validators.isInLimits(1, 2), ), - validators.validate__FAM_AFF__HOH__Count_Fed_Time(), validators.if_then_validator( condition_field_name="DATE_OF_BIRTH", condition_function=validators.olderThan(18), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index 221739dd9..a5d4a45a7 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -1,8 +1,9 @@ """Schema for Tribal TANF T1 record types.""" -from ...fields import Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T1DataSubmissionDocument t1 = SchemaManager( @@ -156,7 +157,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -164,9 +166,7 @@ startIndex=19, endIndex=22, required=False, - validators=[ - validators.isNumber(), - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 396418cba..e815ac849 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T2 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func, zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T2DataSubmissionDocument @@ -98,7 +98,7 @@ result_field_name="COOPERATION_CHILD_SUPPORT", result_function=validators.oneOf((1, 2, 9)), ), - validators.validate__FAM_AFF__HOH__Fed_Time(), + validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.isInLimits(1, 3), @@ -347,9 +347,9 @@ type="string", startIndex=51, endIndex=53, - required=False, + required=True, validators=[ - validators.isInStringRange(0, 10), + validators.isInStringRange(1, 10), ], ), Field( @@ -622,7 +622,8 @@ validators.isInStringRange(0, 99), ], ), - Field( + TransformField( + zero_pad(2), item="61", name="ADD_WORK_ACTIVITIES", friendly_name="additional work activities", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py index 7acbdadaa..4e03bbe61 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T3 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T3DataSubmissionDocument FIRST_CHILD = 1 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index e59cecade..9f1f53802 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -1,8 +1,9 @@ """Schema for Tribal TANF T4 record types.""" -from ...fields import Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T4DataSubmissionDocument @@ -54,7 +55,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -62,7 +64,7 @@ startIndex=19, endIndex=22, required=False, - validators=[validators.matches("000")], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index 553197447..22ea004a8 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T5 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T5DataSubmissionDocument @@ -92,7 +92,7 @@ result_field_name="CITIZENSHIP_STATUS", result_function=validators.isInLimits(1, 2), ), - validators.validate__FAM_AFF__HOH__Count_Fed_Time(), + validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.matches(1), diff --git a/tdrs-backend/tdpservice/parsers/test/conftest.py b/tdrs-backend/tdpservice/parsers/test/conftest.py new file mode 100644 index 000000000..1754e66e3 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/conftest.py @@ -0,0 +1,64 @@ +"""Fixtures for parsing integration tests.""" +import pytest +from .factories import ParsingFileFactory + +@pytest.fixture +def t3_cat2_invalid_citizenship_file(): + """Fixture for T3 file with an invalid CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t3_invalid_citizenship_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20204A06 TAN1ED\n' + b'T320201011111111112420190127WTTTT90W022212222204398000000000\n' + b'T320201011111111112420190127WTTTT90W0222122222043981000000004201001013333333330000000' + b'1100000099998888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.fixture +def m2_cat2_invalid_37_38_39_file(): + """Fixture for M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m2_cat2_invalid_37_38_39_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M2202310111111111275219811103WTTT#PW@W22212222222250122000010119350000000000000000000000000000000' + b'00000000000000000000000000000225300000000000000000000\n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.fixture +def m3_cat2_invalid_68_69_file(): + """Fixture for M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m3_cat2_invalid_68_69_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M320231011111111127420110615WTTTP99B#22212222204300000000000\n' + b'M320231011111111127120110615WTTTP99B#222122222043011000000004201001013333333330000000110000009999' + b'8888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.fixture +def m5_cat2_invalid_23_24_file(): + """Fixture for M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m5_cat2_invalid_23_24_file.txt', + section='SSP Closed Case Data', + file__data=(b'HEADER20184C24 SSP1ED\n' + b'M520181011111111161519791106WTTTY0ZB922212222222210112000112970000\n' + b'TRAILER0000001 ') + ) + return parsing_file diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt index 7ef4b46b0..4078f53ee 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt @@ -1,21 +1,21 @@ HEADER20194C00142TAN1ED -T420191011111111762255 0402451153123 +T420191011111111762 1 0402451153123 T520191011111111762120160102WTTTTT@YB2122222222221 822981 0 03 0 0 T520191011111111762120170526WTTTTTZPW2122221222221 822981 0 03 0 0 T520191011111111762319880112WTTTTTTY#2222212222222 122161 1591 0 0 T520191011111111762319610502WTTTTTT##2222212222222 222161 0601 0 0 -T420191011111112343255 0402451 91113 +T420191011111112343 1 0402451 91113 T520191011111112343119860308WTTTTTTTY2122222222221 122111 44162 0 0 -T420191011111112970255 0403561 91112 +T420191011111112970 1 0403561 91112 T520191011111112970119940807WTTTTT@#Z2122221222221 122121 10501 0 0 -T420191111111111339255 0403561 83113 +T420191111111111339 1 0403561 83113 T520191111111111339119880402WTTTTTZ#B2122221222223 221121 3571 0 0 T520191111111111339119970502WTTTTTTYB2122222222221 111111 8522 0 0 -T420191111111112073255 0403561151123 +T420191111111112073 1 0403561151123 T520191111111112073319900312WTTTTTT0@2122222222222 122121 0601 0 0 T520191111111112073319920507WTTTTT@B02122221222222 222121 0601 0 0 T520191111111112073120090514WTTTTT@@@2122222222221 822 11 0 03 0 0 -T420191111111112472255 0403561183113 +T420191111111112472 1 0403561183113 T520191111111112472120140814WTTTTTZZ02222212222221 422981 0 03 0 0 T520191111111112472119840305WTTTTT90W2122221222221 122101 40202 0 0 TRAILER 19 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 9423d74b3..ef2132b65 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1091,7 +1091,7 @@ def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks, dfs): parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) - assert parser_errors.count() == 23 + assert parser_errors.count() == 22 # Should only be cat3 validator errors for error in parser_errors: @@ -1979,3 +1979,85 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): "representing the Calendar Year and Quarter formatted as YYYYQ" Tribal_TANF_T7.objects.count() == 0 + +@pytest.mark.django_db() +def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs): + """Test parsing a TANF T3 record with an invalid CITIZENSHIP_STATUS.""" + dfs.datafile = t3_cat2_invalid_citizenship_file + t3_cat2_invalid_citizenship_file.year = 2021 + t3_cat2_invalid_citizenship_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(t3_cat2_invalid_citizenship_file, dfs) + + parser_errors = ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 2 + + for e in parser_errors: + assert e.error_message == "T3: 0 is not in [1, 2, 9]." + + +@pytest.mark.django_db() +def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs): + """Test parsing an SSP M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" + dfs.datafile = m2_cat2_invalid_37_38_39_file + m2_cat2_invalid_37_38_39_file.year = 2024 + m2_cat2_invalid_37_38_39_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m2_cat2_invalid_37_38_39_file, dfs) + + parser_errors = ParserError.objects.filter(file=m2_cat2_invalid_37_38_39_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 3 + + error_msgs = {"M2: 00 is not in range [1, 16]. or M2: 00 is not in range [98, 99].", + "M2: 0 is not in [1, 2, 3, 9].", + "M2: 0 is not in [1, 2, 9]."} + for e in parser_errors: + assert e.error_message in error_msgs + +@pytest.mark.django_db() +def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): + """Test parsing an SSP M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m3_cat2_invalid_68_69_file + m3_cat2_invalid_68_69_file.year = 2024 + m3_cat2_invalid_68_69_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m3_cat2_invalid_68_69_file, dfs) + + parser_errors = ParserError.objects.filter(file=m3_cat2_invalid_68_69_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 4 + + error_msgs = {"M3: 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", + "M3: 0 is not in [1, 2, 3, 9]."} + + for e in parser_errors: + assert e.error_message in error_msgs + +@pytest.mark.django_db() +def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): + """Test parsing an SSP M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m5_cat2_invalid_23_24_file + m5_cat2_invalid_23_24_file.year = 2019 + m5_cat2_invalid_23_24_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m5_cat2_invalid_23_24_file, dfs) + + parser_errors = ParserError.objects.filter(file=m5_cat2_invalid_23_24_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 2 + + error_msgs = {"M5: 00 matches 00.", + "M5: 0 is not in [1, 2, 3, 9]."} + + for e in parser_errors: + assert e.error_message in error_msgs diff --git a/tdrs-backend/tdpservice/parsers/test/test_transforms.py b/tdrs-backend/tdpservice/parsers/test/test_transforms.py index ff1188785..1aabe69db 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_transforms.py +++ b/tdrs-backend/tdpservice/parsers/test/test_transforms.py @@ -1,11 +1,31 @@ -"""Tests for the transforms module.""" +"""Test for Transforms.""" +import pytest +import tdpservice.parsers.transforms as transforms from tdpservice.parsers.transforms import ( tanf_ssn_decryption_func, ssp_ssn_decryption_func, ) +@pytest.mark.parametrize("value,digits,expected", [ + ("1", 3, "001"), + ("10", 3, "010"), + ("100", 3, "100"), + ("1000", 3, "1000"), + ("1 ", 3, "01 "), + ("1 ", 3, "1 "), + ("1", 0, "1"), + ("1", -1, "1") +]) +def test_zero_pad(value, digits, expected): + """Test zero_pad returns valid value.""" + transform = transforms.zero_pad(digits) + result = transform(value) + + assert result == expected + + def test_tanf_ssn_decryption_func(): """Test the TANF SSN decryption function.""" assert tanf_ssn_decryption_func(None) is None diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 0ee8815e4..94045acfb 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -922,19 +922,6 @@ def test_validate_cooperation_with_child_support(self, record): result = val(record, RowSchema()) assert result[0] is False - def test_validate_months_federal_time_limit(self, record): - """Test cat3 validator for federal time limit.""" - val = validators.validate__FAM_AFF__HOH__Fed_Time() - record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) - assert result == (True, None, ['FAMILY_AFFILIATION', 'RELATIONSHIP_HOH', 'MONTHS_FED_TIME_LIMIT']) - - record.FAMILY_AFFILIATION = 1 - record.MONTHS_FED_TIME_LIMIT = "000" - record.RELATIONSHIP_HOH = "01" - result = val(record, RowSchema()) - assert result[0] is False - def test_validate_employment_status(self, record): """Test cat3 validator for employment status.""" val = validators.if_then_validator( @@ -1233,21 +1220,6 @@ def test_validate_citizenship_status(self, record): result = val(record, RowSchema()) assert result[0] is False - def test_validate_hoh_fed_time(self, record): - """Test cat3 validator for federal disability.""" - val = validators.validate__FAM_AFF__HOH__Count_Fed_Time() - - record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) - assert result == (True, None, ['FAMILY_AFFILIATION', 'RELATIONSHIP_HOH', 'COUNTABLE_MONTH_FED_TIME']) - - record.FAMILY_AFFILIATION = 1 - record.RELATIONSHIP_HOH = 1 - record.COUNTABLE_MONTH_FED_TIME = 0 - - result = val(record, RowSchema()) - assert result[0] is False - def test_validate_oasdi_insurance(self, record): """Test cat3 validator for OASDI insurance.""" val = validators.if_then_validator( diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index 78d15dcbb..cd51e2012 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -36,3 +36,9 @@ def ssp_ssn_decryption_func(value, **kwargs): decryption_table = str.maketrans(decryption_dict) return value.translate(decryption_table) return value + +def zero_pad(digits): + """Zero pad a string.""" + def transform(value, **kwargs): + return value.lstrip().zfill(digits) + return transform diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 54bf98704..9f99e928c 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -330,6 +330,18 @@ def between(min, max): f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", ) + +def fieldHasLength(length): + """Validate that the field value (string or array) has a length matching length param.""" + return make_validator( + lambda value: len(value) == length, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", + ) + + def hasLengthGreaterThan(val, error_func=None): """Validate that value (string or array) has a length greater than val.""" return make_validator( @@ -490,7 +502,7 @@ def isSmallerThanOrEqualTo(UpperBound): def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( - lambda value: value >= LowerBound and value <= UpperBound, + lambda value: int(value) >= LowerBound and int(value) <= UpperBound, lambda value, row_schema, friendly_name, item_num: f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger or equal " f"to {LowerBound} and smaller or equal to {UpperBound}." @@ -614,104 +626,6 @@ def validate(instance, row_schema): return validate - -def validate__FAM_AFF__HOH__Fed_Time(): - """If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then MONTHS_FED_TIME_LIMIT >= 1.""" - # value is instance - def validate(instance, row_schema): - false_case = (False, - f"{row_schema.record_type}: If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then " - + "MONTHS_FED_TIME_LIMIT >= 1.", - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "MONTHS_FED_TIME_LIMIT",], - ) - true_case = (True, - None, - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "MONTHS_FED_TIME_LIMIT",], - ) - try: - FAMILY_AFFILIATION = ( - instance["FAMILY_AFFILIATION"] - if type(instance) is dict - else getattr(instance, "FAMILY_AFFILIATION") - ) - RELATIONSHIP_HOH = ( - instance["RELATIONSHIP_HOH"] - if type(instance) is dict - else getattr(instance, "RELATIONSHIP_HOH") - ) - RELATIONSHIP_HOH = int(RELATIONSHIP_HOH) - MONTHS_FED_TIME_LIMIT = ( - instance["MONTHS_FED_TIME_LIMIT"] - if type(instance) is dict - else getattr(instance, "MONTHS_FED_TIME_LIMIT") - ) - if FAMILY_AFFILIATION == 1 and (RELATIONSHIP_HOH == 1 or RELATIONSHIP_HOH == 2): - if MONTHS_FED_TIME_LIMIT is None or int(MONTHS_FED_TIME_LIMIT) < 1: - return false_case - else: - return true_case - else: - return true_case - except Exception: - vals = {"FAMILY_AFFILIATION": FAMILY_AFFILIATION, - "RELATIONSHIP_HOH": RELATIONSHIP_HOH, - "MONTHS_FED_TIME_LIMIT": MONTHS_FED_TIME_LIMIT} - logger.debug("Caught exception in validator: validate__FAM_AFF__HOH__Fed_Time. With field values: " + - f"{vals}.") - return false_case - - return validate - - -def validate__FAM_AFF__HOH__Count_Fed_Time(): - """If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then COUNTABLE_MONTH_FED_TIME >= 1.""" - # value is instance - def validate(instance, row_schema): - false_case = (False, - f"{row_schema.record_type}: If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then " - + "COUNTABLE_MONTH_FED_TIME >= 1.", - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "COUNTABLE_MONTH_FED_TIME",], - ) - true_case = (True, - None, - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "COUNTABLE_MONTH_FED_TIME",], - ) - try: - FAMILY_AFFILIATION = ( - instance["FAMILY_AFFILIATION"] - if type(instance) is dict - else getattr(instance, "FAMILY_AFFILIATION") - ) - RELATIONSHIP_HOH = ( - instance["RELATIONSHIP_HOH"] - if type(instance) is dict - else getattr(instance, "RELATIONSHIP_HOH") - ) - RELATIONSHIP_HOH = int(RELATIONSHIP_HOH) - COUNTABLE_MONTH_FED_TIME = ( - instance["COUNTABLE_MONTH_FED_TIME"] - if type(instance) is dict - else getattr(instance, "COUNTABLE_MONTH_FED_TIME") - ) - if FAMILY_AFFILIATION == 1 and (RELATIONSHIP_HOH == 1 or RELATIONSHIP_HOH == 2): - if int(COUNTABLE_MONTH_FED_TIME) < 1: - return false_case - else: - return true_case - else: - return true_case - except Exception: - vals = {"FAMILY_AFFILIATION": FAMILY_AFFILIATION, - "RELATIONSHIP_HOH": RELATIONSHIP_HOH, - "COUNTABLE_MONTH_FED_TIME": COUNTABLE_MONTH_FED_TIME - } - logger.debug("Caught exception in validator: validate__FAM_AFF__HOH__Count_Fed_Time. With field values: " + - f"{vals}.") - return false_case - - return validate - - def validate_header_section_matches_submission(datafile, section, generate_error): """Validate header section matches submission section.""" is_valid = datafile.section == section diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py b/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py new file mode 100644 index 000000000..4a7cf36b4 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.15 on 2024-04-29 18:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0027_tribal_ssp_tanf_dob_to_string'), + ] + + operations = [ + migrations.AlterField( + model_name='ssp_m2', + name='EDUCATION_LEVEL', + field=models.CharField(max_length=2, null=True), + ), + migrations.AlterField( + model_name='ssp_m3', + name='EDUCATION_LEVEL', + field=models.CharField(max_length=2, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index b11e6fff5..bb5840323 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -113,7 +113,7 @@ class SSP_M2(models.Model): RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) NEEDS_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) COOPERATION_CHILD_SUPPORT = models.IntegerField(null=True, blank=False) EMPLOYMENT_STATUS = models.IntegerField(null=True, blank=False) @@ -194,7 +194,7 @@ class SSP_M3(models.Model): RECEIVE_SSI = models.IntegerField(null=True, blank=False) RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) UNEARNED_SSI = models.IntegerField(null=True, blank=False) OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/security/test/test_views.py b/tdrs-backend/tdpservice/security/test/test_views.py new file mode 100644 index 000000000..b602ca577 --- /dev/null +++ b/tdrs-backend/tdpservice/security/test/test_views.py @@ -0,0 +1,66 @@ +"""Tests for the views in the security app.""" + +import pytest +import logging +from rest_framework.authtoken.models import Token +from tdpservice.users.models import User, AccountApprovalStatusChoices +from tdpservice.security.views import token_is_valid +from django.test import Client +from django.urls import reverse +from django.contrib.auth.models import Group + +client = Client() + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def token(): + """Return a DRF token.""" + user = User.objects.create(username="testuser") + token = Token.objects.create(user=user) + return token + + +@pytest.mark.django_db +def test_token_is_valid(token): + """Test token_is_valid function.""" + logger.info(token.__dict__) + assert token_is_valid(token) is True + token.created = token.created.replace(year=2000) + # token.save() + assert token_is_valid(token) is False + + +@pytest.mark.django_db +def test_generate_new_token(client): + """Test generate_new_token function.""" + url = reverse("get-new-token") + # assert if user is not authenticated + response = client.get(url) + assert response.status_code == 302 + + # assert if user is not ofa_sys_admin + user = User.objects.create_user(username="testuser", password="testpassword") + user.save() + client.login(username="testuser", password="testpassword") + response = client.get(url) + assert response.status_code == 302 + + # assert if user is not approved + user.account_approval_status = AccountApprovalStatusChoices.PENDING + user.groups.add(Group.objects.get(name="OFA System Admin")) + user.save() + client.login(username="testuser", password="testpassword") + response = client.get(url) + assert response.status_code == 302 + + # assert if token is valid + user.account_approval_status = AccountApprovalStatusChoices.APPROVED + user.save() + + client.login(username="testuser", password="testpassword") + url = reverse("get-new-token") + response = client.get(url) + assert response.status_code == 200 + assert response.data == str(Token.objects.get(user=user)) diff --git a/tdrs-backend/tdpservice/security/urls.py b/tdrs-backend/tdpservice/security/urls.py new file mode 100644 index 000000000..ef62d5e44 --- /dev/null +++ b/tdrs-backend/tdpservice/security/urls.py @@ -0,0 +1,12 @@ +"""URL patterns for the security app.""" + +from . import views +from django.urls import path + +urlpatterns = [ + path( + "get-token", + views.generate_new_token, + name="get-new-token", + ), +] diff --git a/tdrs-backend/tdpservice/security/utils.py b/tdrs-backend/tdpservice/security/utils.py new file mode 100644 index 000000000..873dc3aa4 --- /dev/null +++ b/tdrs-backend/tdpservice/security/utils.py @@ -0,0 +1,43 @@ +"""Utility classes and functions for security.""" + +from rest_framework import exceptions +from rest_framework.authentication import TokenAuthentication +from django.utils.translation import gettext_lazy as _ +from datetime import datetime +import pytz +from datetime import timedelta +from django.conf import settings +import logging + +logger = logging.getLogger(__name__) + +def token_is_valid(token): + """Check if token is valid.""" + utc_now = datetime.now() + utc_now = utc_now.replace(tzinfo=pytz.utc) + if token.created < (utc_now - timedelta(hours=settings.TOKEN_EXPIRATION_HOURS)): + logger.info("API auth Token expired") + return False + return token is not None + +# have to use ExpTokenAuthentication in settings.py instead of TokenAuthentication +class ExpTokenAuthentication(TokenAuthentication): + """Custom token authentication class that checks if token is expired.""" + + # see https://github.com/encode/django-rest-framework/blob/master/rest_framework/authentication.py + + def authenticate_credentials(self, key): + """Authenticate the credentials.""" + model = self.get_model() + try: + token = model.objects.select_related("user").get(key=key) + except model.DoesNotExist: + raise exceptions.AuthenticationFailed(_("Invalid token.")) + + if not token.user.is_active: + raise exceptions.AuthenticationFailed(_("User inactive or deleted.")) + + if not token_is_valid(token): + raise exceptions.AuthenticationFailed(_("Token expired.")) + + return (token.user, token) diff --git a/tdrs-backend/tdpservice/security/views.py b/tdrs-backend/tdpservice/security/views.py new file mode 100644 index 000000000..64377332c --- /dev/null +++ b/tdrs-backend/tdpservice/security/views.py @@ -0,0 +1,36 @@ +"""Views for the security app.""" + +from rest_framework.decorators import api_view +from rest_framework.response import Response +from django.contrib.auth.decorators import user_passes_test +from tdpservice.users.models import User, AccountApprovalStatusChoices +from rest_framework.authtoken.models import Token +from tdpservice.security.utils import token_is_valid + +import logging + +logger = logging.getLogger(__name__) + + +def can_get_new_token(user): + """Check if user can get a new token.""" + return ( + user.is_authenticated + and user.is_ofa_sys_admin + and user.account_approval_status == AccountApprovalStatusChoices.APPROVED + ) + + +@user_passes_test(can_get_new_token, login_url="/login/") +@api_view(["GET"]) +def generate_new_token(request): + """Generate new token for the API user.""" + if request.method == "GET": + user = User.objects.get(username=request.user) + token, created = Token.objects.get_or_create(user=user) + if token_is_valid(token): + return Response(str(token)) + else: + token.delete() + token = Token.objects.create(user=user) + return Response(str(token)) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 50bc6cb90..6ca924fe0 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -297,7 +297,7 @@ class Common(Configuration): "DEFAULT_AUTHENTICATION_CLASSES": ( "tdpservice.users.authentication.CustomAuthentication", "rest_framework.authentication.SessionAuthentication", - "rest_framework.authentication.TokenAuthentication", + "tdpservice.security.utils.ExpTokenAuthentication", ), "DEFAULT_FILTER_BACKENDS": [ "django_filters.rest_framework.DjangoFilterBackend", @@ -311,6 +311,8 @@ class Common(Configuration): "django.contrib.auth.backends.ModelBackend", ) + TOKEN_EXPIRATION_HOURS = int(os.getenv("TOKEN_EXPIRATION_HOURS", 24)) + # CORS CORS_ALLOW_CREDENTIALS = True diff --git a/tdrs-backend/tdpservice/urls.py b/tdrs-backend/tdpservice/urls.py index f19e26b98..d40e64651 100755 --- a/tdrs-backend/tdpservice/urls.py +++ b/tdrs-backend/tdpservice/urls.py @@ -38,6 +38,7 @@ path("stts/", include("tdpservice.stts.urls")), path("data_files/", include("tdpservice.data_files.urls")), path("logs/", write_logs), + path("security/", include("tdpservice.security.urls")), ] if settings.DEBUG: diff --git a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx index 9800206c6..3ddfe7365 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx @@ -121,7 +121,7 @@ export const CaseAggregatesTable = ({ files }) => ( Status - Error Reports (In development) + Error Reports diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index d61ce85ec..654339ed6 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -72,17 +72,31 @@ const SubmissionHistory = ({ filterValues }) => { }, [hasFetchedFiles, files, dispatch, filterValues]) return ( -
- {fileUploadSections.map((section, index) => ( - f.section.includes(section))} - /> - ))} -
+ <> + +
+ {fileUploadSections.map((section, index) => ( + f.section.includes(section))} + /> + ))} +
+ ) } diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js index 06baeb730..325c7d898 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js @@ -241,9 +241,7 @@ describe('SubmissionHistory', () => { expect(screen.queryByText('test5.txt')).not.toBeInTheDocument() expect(screen.queryByText('test6.txt')).toBeInTheDocument() - expect( - screen.queryByText('Error Reports (In development)') - ).toBeInTheDocument() + expect(screen.queryByText('Error Reports')).toBeInTheDocument() }) it('Shows SSP results when SSP-MOE file type selected', () => { diff --git a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx index 8a8e6fd5c..3f4ba24a4 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx @@ -109,7 +109,7 @@ export const TotalAggregatesTable = ({ files }) => ( Status - Error Reports (In development) + Error Reports From 20021d4d36bd85c0dbcb58ae3d51cf9ac114edf1 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 5 Jun 2024 14:26:39 -0400 Subject: [PATCH 25/52] fix tests --- .../tdpservice/parsers/test/test_parse.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index ef2132b65..dc5e75567 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1463,15 +1463,15 @@ def test_parse_tribal_section_2_file(tribal_section_2_file, dfs): dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 0, 'months': [ - {'accepted_without_errors': 0, - 'accepted_with_errors': 3, 'month': 'Oct'}, - {'accepted_without_errors': 0, - 'accepted_with_errors': 3, 'month': 'Nov'}, + {'accepted_without_errors': 3, + 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 3, + 'accepted_with_errors': 0, 'month': 'Nov'}, {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} ]} - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED assert Tribal_TANF_T4.objects.all().count() == 6 assert Tribal_TANF_T5.objects.all().count() == 13 @@ -1996,7 +1996,7 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs assert parser_errors.count() == 2 for e in parser_errors: - assert e.error_message == "T3: 0 is not in [1, 2, 9]." + assert e.error_message == "T3 Item 76 (citizenship status): 0 is not in [1, 2, 9]." @pytest.mark.django_db() @@ -2014,9 +2014,9 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2: 00 is not in range [1, 16]. or M2: 00 is not in range [98, 99].", - "M2: 0 is not in [1, 2, 3, 9].", - "M2: 0 is not in [1, 2, 9]."} + error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. or M2 Item 37 (education level): 00 is not in range [98, 99].", + "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", + "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} for e in parser_errors: assert e.error_message in error_msgs @@ -2035,8 +2035,8 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3: 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", - "M3: 0 is not in [1, 2, 3, 9]."} + error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", + "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: assert e.error_message in error_msgs @@ -2056,8 +2056,8 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): assert parser_errors.count() == 2 - error_msgs = {"M5: 00 matches 00.", - "M5: 0 is not in [1, 2, 3, 9]."} + error_msgs = {"M5 Item 23 (education level): 00 matches 00.", + "M5 Item 24 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: assert e.error_message in error_msgs From e2f13fac8c039c8696989cf58ade54949479bcfb Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 5 Jun 2024 14:35:18 -0400 Subject: [PATCH 26/52] fix test --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index dc5e75567..563c179a1 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2035,7 +2035,7 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", + error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3 Item 68 (education level): 00 is not in range [98, 99].", "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: From 6c50bdc434cf67ac2a2ffd2e61617dcf354691f6 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 11 Jun 2024 12:14:38 -0400 Subject: [PATCH 27/52] abs import --- tdrs-backend/tdpservice/email/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 480c971f7..972c7eef6 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -13,8 +13,8 @@ from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email from tdpservice.stts.models import STT from tdpservice.data_files.models import DataFile -from .email import automated_email, log -from .email_enums import EmailType +from tdpservice.email import automated_email, log +from tdpservice.email.email_enums import EmailType logger = logging.getLogger(__name__) From cb82c56dbd882af6ace3d1caf05720cc0c340d03 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 11 Jun 2024 12:16:15 -0400 Subject: [PATCH 28/52] move calendar_to_fiscal to util --- tdrs-backend/tdpservice/email/tasks.py | 5 ++--- tdrs-backend/tdpservice/parsers/util.py | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 972c7eef6..15b255ad8 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -15,6 +15,7 @@ from tdpservice.data_files.models import DataFile from tdpservice.email import automated_email, log from tdpservice.email.email_enums import EmailType +from tdpservice.parsers.util import calendar_to_fiscal logger = logging.getLogger(__name__) @@ -78,11 +79,9 @@ def email_admin_num_access_requests(): @shared_task def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): """Send all Data Analysts a reminder to submit if they have not already.""" - def get_fiscal_year(calendar_year, fiscal_quarter): - return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year now = datetime.now() - fiscal_year = get_fiscal_year(now.year, fiscal_quarter) + fiscal_year = calendar_to_fiscal(now.year, fiscal_quarter) all_locations = STT.objects.all() diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 287c58cff..839f9f142 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -130,6 +130,11 @@ def fiscal_to_calendar(year, fiscal_quarter): ind_qtr = array.index(int_qtr) # get the index so we can easily wrap-around end of array return year, "Q{}".format(array[ind_qtr - 1]) # return the previous quarter + +def calendar_to_fiscal(calendar_year, fiscal_quarter): + return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year + + def transform_to_months(quarter): """Return a list of months in a quarter depending the quarter's format.""" match quarter: From dd65bbe0e4761187883da905228539010f59ecd3 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 11 Jun 2024 14:35:21 -0400 Subject: [PATCH 29/52] fix tests --- tdrs-backend/tdpservice/email/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 15b255ad8..84e5f1415 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -13,7 +13,7 @@ from tdpservice.email.helpers.account_deactivation_warning import send_deactivation_warning_email from tdpservice.stts.models import STT from tdpservice.data_files.models import DataFile -from tdpservice.email import automated_email, log +from tdpservice.email.email import automated_email, log from tdpservice.email.email_enums import EmailType from tdpservice.parsers.util import calendar_to_fiscal From 2cfcfb30ef5c47b6965d13934a6c5b78840e3383 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 11 Jun 2024 14:45:56 -0400 Subject: [PATCH 30/52] lint --- tdrs-backend/tdpservice/email/tasks.py | 1 - tdrs-backend/tdpservice/parsers/util.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 84e5f1415..73b2158e9 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -79,7 +79,6 @@ def email_admin_num_access_requests(): @shared_task def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): """Send all Data Analysts a reminder to submit if they have not already.""" - now = datetime.now() fiscal_year = calendar_to_fiscal(now.year, fiscal_quarter) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 839f9f142..76c29dd37 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -132,6 +132,7 @@ def fiscal_to_calendar(year, fiscal_quarter): def calendar_to_fiscal(calendar_year, fiscal_quarter): + """Decrement the calendar year if in Q1.""" return calendar_year - 1 if fiscal_quarter == 'Q1' else calendar_year From 884da8e0df87306380d4f089854ba7c604f4865f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 12 Jun 2024 09:37:07 -0400 Subject: [PATCH 31/52] pass error_context_format for different formatting cat 3 messages --- tdrs-backend/tdpservice/parsers/row_schema.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 4 +- .../parsers/test/test_validators.py | 952 +++++++++++++++--- tdrs-backend/tdpservice/parsers/validators.py | 192 ++-- 4 files changed, 916 insertions(+), 234 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index 7500ab593..39f3814f3 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -151,7 +151,7 @@ def run_field_validators(self, instance, generate_error): schema=self, error_category=ParserErrorCategoryChoices.FIELD_VALUE, error_message=( - f"{format_error_context(self, field.friendly_name, field.item)}: " + f"{format_error_context(self, field.friendly_name, field.item)} " "field is required but a value was not provided." ), record=instance, diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 563c179a1..b9a5eb895 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2014,7 +2014,7 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. or M2 Item 37 (education level): 00 is not in range [98, 99].", + error_msgs = {"Item 37 (education level) 00 is not in range [1, 16]. or Item 37 (education level) 00 is not in range [98, 99].", "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} for e in parser_errors: @@ -2035,7 +2035,7 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3 Item 68 (education level): 00 is not in range [98, 99].", + error_msgs = {"Item 68 (education level) 00 is not in range [1, 16]. or Item 68 (education level) 00 is not in range [98, 99].", "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 94045acfb..4f6bd0772 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -6,6 +6,7 @@ from .. import validators from .. import schema_defs, util from ..row_schema import RowSchema +from ..fields import Field from tdpservice.parsers.test.factories import TanfT1Factory, TanfT2Factory, TanfT3Factory, TanfT5Factory, TanfT6Factory from tdpservice.parsers.test.factories import SSPM5Factory @@ -45,39 +46,39 @@ def test_or_validators(): """Test `or_validators` gives a valid result.""" value = "2" validator = validators.or_validators(validators.matches(("2")), validators.matches(("3"))) - assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator("3", RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator("5", RowSchema(), "friendly_name", "item_no") == ( + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) + assert validator("3", RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) + assert validator("5", RowSchema(), "friendly_name", "item_no", 'inline') == ( False, - "T1 Item item_no (friendly_name): 5 does not match 2. or " - "T1 Item item_no (friendly_name): 5 does not match 3." + "Item item_no (friendly_name) 5 does not match 2. or " + "Item item_no (friendly_name) 5 does not match 3." ) validator = validators.or_validators(validators.matches(("2")), validators.matches(("3")), validators.matches(("4"))) - assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) value = "3" - assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) value = "4" - assert validator(value, RowSchema(), "friendly_name", "item_no") == (True, None) + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) value = "5" - assert validator(value, RowSchema(), "friendly_name", "item_no") == ( + assert validator(value, RowSchema(), "friendly_name", "item_no", 'inline') == ( False, - 'T1 Item item_no (friendly_name): 5 does not match 2. or ' - 'T1 Item item_no (friendly_name): 5 does not match 3. or ' - 'T1 Item item_no (friendly_name): 5 does not match 4.' + "Item item_no (friendly_name) 5 does not match 2. or " + "Item item_no (friendly_name) 5 does not match 3. or " + "Item item_no (friendly_name) 5 does not match 4." ) validator = validators.or_validators(validators.matches((2)), validators.matches((3)), validators.isLargerThan(4)) - assert validator(5, RowSchema(), "friendly_name", "item_no") == (True, None) - assert validator(1, RowSchema(), "friendly_name", "item_no") == ( + assert validator(5, RowSchema(), "friendly_name", "item_no", 'inline') == (True, None) + assert validator(1, RowSchema(), "friendly_name", "item_no", 'inline') == ( False, - "T1 Item item_no (friendly_name): 1 does not match 2. " - "or T1 Item item_no (friendly_name): 1 does not " - "match 3. or T1 Item item_no (friendly_name): 1 is not larger than 4." + "Item item_no (friendly_name) 1 does not match 2. or " + "Item item_no (friendly_name) 1 does not match 3. or " + "Item item_no (friendly_name) 1 is not larger than 4." ) def test_if_validators(): @@ -87,14 +88,28 @@ def test_if_validators(): condition_field_name="Field1", condition_function=validators.matches('1'), result_field_name="Field2", result_function=validators.matches('2'), ) - assert validator(value, RowSchema()) == (True, None, ['Field1', 'Field2']) + assert validator(value, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='Field1', friendly_name='field 1'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='Field2', friendly_name='field 2'), + ] + )) == (True, None, ['Field1', 'Field2']) validator = validator = validators.if_then_validator( condition_field_name="Field1", condition_function=validators.matches('1'), result_field_name="Field2", result_function=validators.matches('1'), ) - result = validator(value, RowSchema()) - assert result == (False, 'if Field1 :1 validator1 passed then Field2 T1 Item -1 (Field2): 2 does not match 1.', + result = validator(value, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='Field1', friendly_name='field 1'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='Field2', friendly_name='field 2'), + ] + )) + assert result == (False, 'if Field1 :1 validator1 passed then Field2 Item 2 (field 2) 2 does not match 1.', ['Field1', 'Field2']) @@ -103,7 +118,7 @@ def test_and_validators(): validator = validators.and_validators(validators.isLargerThan(2), validators.isLargerThan(0)) assert validator(1, RowSchema(), "friendly_name", "item_no") == ( False, - 'T1 Item item_no (friendly_name): 1 is not larger than 2.' + 'Item item_no (friendly_name) 1 is not larger than 2.' ) assert validator(3, RowSchema(), "friendly_name", "item_no") == (True, None) @@ -139,7 +154,7 @@ def test_validate__FAM_AFF__SSN(): def test_quarterIsValid(value, valid): """Test `quarterIsValid`.""" val = validators.quarterIsValid() - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) errorText = None if valid else f"T1 Item item_no (friendly_name): {value[-1:]} is not a valid quarter." assert result == (valid, errorText) @@ -153,7 +168,7 @@ def test_validateSSN(): value = "111111111" options = [str(i) * 9 for i in range(0, 10)] - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == (False, f"T1 Item item_no (friendly_name): {value} is in {options}.") def test_validateRace(): @@ -164,7 +179,7 @@ def test_validateRace(): assert result == (True, None) value = 3 - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): {value} is not greater than or equal to 0 or smaller than or equal to 2." @@ -178,7 +193,7 @@ def test_validateRptMonthYear(): assert result == (True, None) value = "T1 " - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does not " @@ -186,7 +201,7 @@ def test_validateRptMonthYear(): ) value = "T1189912" - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does not follow " @@ -194,7 +209,7 @@ def test_validateRptMonthYear(): ) value = "T1202013" - result = val(value, RowSchema(), "friendly_name", "item_no") + result = val(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): The value: {value[2:8]}, does " @@ -217,7 +232,7 @@ def test_matches_returns_invalid(): value = 'TEST' validator = validators.matches('test') - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): TEST does not match test.' @@ -238,7 +253,7 @@ def test_oneOf_returns_valid(): options = ["17-55"] validator = validators.oneOf(options) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is True assert error is None @@ -250,7 +265,7 @@ def test_oneOf_returns_invalid(): options = [17, 24, 36] validator = validators.oneOf(options) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 64 is not in [17, 24, 36].' @@ -259,7 +274,7 @@ def test_oneOf_returns_invalid(): options = ["17-55"] validator = validators.oneOf(options) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ @@ -293,7 +308,7 @@ def test_between_returns_invalid(): value = 47 validator = validators.between(48, 400) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 47 is not between 48 and 400.' @@ -321,7 +336,7 @@ def test_between_returns_invalid(): def test_isNumber(value, expected_is_valid, expected_error): """Test `isNumber` validator.""" validator = validators.isNumber() - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid == expected_is_valid assert error == expected_error @@ -339,7 +354,7 @@ def test_date_month_is_valid_returns_invalid(): """Test `dateMonthIsValid` gives an invalid result.""" value = '20191327' validator = validators.dateMonthIsValid() - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 13 is not a valid month.' @@ -357,7 +372,7 @@ def test_date_day_is_valid_returns_invalid(): """Test `dateDayIsValid` gives an invalid result.""" value = '20191132' validator = validators.dateDayIsValid() - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 32 is not a valid day.' @@ -370,7 +385,7 @@ def test_olderThan(): assert validator(value) == (True, None) value = 20240101 - result = validator(value, RowSchema(), "friendly_name", "item_no") + result = validator(value, RowSchema(), "friendly_name", "item_no", None) assert result == ( False, f"T1 Item item_no (friendly_name): {str(value)[:4]} must be less than or equal to " @@ -386,7 +401,7 @@ def test_dateYearIsLargerThan(): assert validator(value) == (True, None) value = 18990101 - assert validator(value, RowSchema(), "friendly_name", "item_no") == ( + assert validator(value, RowSchema(), "friendly_name", "item_no", None) == ( False, f"T1 Item item_no (friendly_name): Year {str(value)[:4]} must be larger than {year}." ) @@ -397,7 +412,7 @@ def test_between_returns_invalid_for_string_value(): value = '047' validator = validators.between(100, 400) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 047 is not between 100 and 400.' @@ -419,7 +434,7 @@ def test_recordHasLength_returns_invalid(): value = 'abcd123' validator = validators.recordHasLength(22) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1: record length is 7 characters but must be 22.' @@ -429,7 +444,7 @@ def test_hasLengthGreaterThan_returns_valid(): value = 'abcd123' validator = validators.hasLengthGreaterThan(6) - is_valid, error = validator(value, None, "friendly_name", "item_no") + is_valid, error = validator(value, None, "friendly_name", "item_no", None) assert is_valid is True assert error is None @@ -452,7 +467,7 @@ def test_recordHasLengthBetween_returns_valid(): upper = 15 validator = validators.recordHasLengthBetween(lower, upper) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is True assert error is None @@ -465,7 +480,7 @@ def test_recordHasLengthBetween_returns_invalid(): upper = 1 validator = validators.recordHasLengthBetween(lower, upper) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == f"T1: record length of {len(value)} characters is not in the range [{lower}, {upper}]." @@ -487,7 +502,7 @@ def test_intHasLength_returns_invalid(): value = '1a3' validator = validators.intHasLength(22) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 1a3 does not have exactly 22 digits.' @@ -509,7 +524,7 @@ def test_contains_returns_invalid(): value = '12345abcde' validator = validators.contains('6789') - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 12345abcde does not contain 6789.' @@ -531,7 +546,7 @@ def test_startsWith_returns_invalid(): value = '12345abcde' validator = validators.startsWith('abc') - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): 12345abcde does not start with abc.' @@ -553,7 +568,7 @@ def test_notEmpty_returns_invalid_full_string(): value = ' ' validator = validators.notEmpty() - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == 'T1 Item item_no (friendly_name): contains blanks between positions 0 and 9.' @@ -575,7 +590,7 @@ def test_notEmpty_returns_invalid_substring(): value = '111 333' validator = validators.notEmpty(start=3, end=5) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == "T1 Item item_no (friendly_name): 111 333 contains blanks between positions 3 and 5." @@ -586,7 +601,7 @@ def test_notEmpty_returns_nonexistent_substring(): value = '111 333' validator = validators.notEmpty(start=10, end=12) - is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error == "T1 Item item_no (friendly_name): 111 333 contains blanks between positions 10 and 12." @@ -596,7 +611,7 @@ def test_notEmpty_returns_nonexistent_substring(): def test_quarterIsValid_returns_true_if_valid(test_input): """Test `quarterIsValid` gives a valid result for values 1-4.""" validator = validators.quarterIsValid() - is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no", None) assert is_valid is True assert error is None @@ -606,7 +621,7 @@ def test_quarterIsValid_returns_true_if_valid(test_input): def test_quarterIsValid_returns_false_if_invalid(test_input): """Test `quarterIsValid` gives an invalid result for values not 1-4.""" validator = validators.quarterIsValid() - is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") + is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no", 'prefix') assert is_valid is False assert error == f"T1 Item item_no (friendly_name): {test_input} is not a valid quarter." @@ -615,7 +630,7 @@ def test_quarterIsValid_returns_false_if_invalid(test_input): def test_calendarQuarterIsValid_returns_invalid(value): """Test `calendarQuarterIsValid` returns false on invalid input.""" val = validators.calendarQuarterIsValid(2, 7) - is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") + is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False assert error_msg == ( @@ -628,7 +643,7 @@ def test_calendarQuarterIsValid_returns_invalid(value): def test_calendarQuarterIsValid_returns_valid(value): """Test `calendarQuarterIsValid` returns false on invalid input.""" val = validators.calendarQuarterIsValid(2, 7) - is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") + is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is True assert error_msg is None @@ -662,11 +677,25 @@ def test_validate_food_stamps(self, record): ) record.RECEIVES_FOOD_STAMPS = 1 record.AMT_FOOD_STAMP_ASSISTANCE = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='RECEIVES_FOOD_STAMPS', friendly_name='receives food stamps'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_FOOD_STAMP_ASSISTANCE', friendly_name='amt food stamps'), + ] + )) assert result == (True, None, ['RECEIVES_FOOD_STAMPS', 'AMT_FOOD_STAMP_ASSISTANCE']) record.AMT_FOOD_STAMP_ASSISTANCE = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='RECEIVES_FOOD_STAMPS', friendly_name='receives food stamps'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_FOOD_STAMP_ASSISTANCE', friendly_name='amt food stamps'), + ] + )) assert result[0] is False def test_validate_subsidized_child_care(self, record): @@ -677,12 +706,26 @@ def test_validate_subsidized_child_care(self, record): ) record.RECEIVES_SUB_CC = 4 record.AMT_SUB_CC = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='RECEIVES_SUB_CC', friendly_name='receives sub cc'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_SUB_CC', friendly_name='amt sub cc'), + ] + )) assert result == (True, None, ['RECEIVES_SUB_CC', 'AMT_SUB_CC']) record.RECEIVES_SUB_CC = 4 record.AMT_SUB_CC = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='RECEIVES_SUB_CC', friendly_name='receives sub cc'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_SUB_CC', friendly_name='amt sub cc'), + ] + )) assert result[0] is False def test_validate_cash_amount_and_nbr_months(self, record): @@ -691,12 +734,26 @@ def test_validate_cash_amount_and_nbr_months(self, record): condition_field_name='CASH_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='NBR_MONTHS', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CASH_AMOUNT', friendly_name='cash amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NBR_MONTHS', friendly_name='nbr months'), + ] + )) assert result == (True, None, ['CASH_AMOUNT', 'NBR_MONTHS']) record.CASH_AMOUNT = 1 record.NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CASH_AMOUNT', friendly_name='cash amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NBR_MONTHS', friendly_name='nbr months'), + ] + )) assert result[0] is False def test_validate_child_care(self, record): @@ -705,12 +762,26 @@ def test_validate_child_care(self, record): condition_field_name='CC_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='CHILDREN_COVERED', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CC_AMOUNT', friendly_name='cc amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CHILDREN_COVERED', friendly_name='chldrn coverd'), + ] + )) assert result == (True, None, ['CC_AMOUNT', 'CHILDREN_COVERED']) record.CC_AMOUNT = 1 record.CHILDREN_COVERED = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CC_AMOUNT', friendly_name='cc amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CHILDREN_COVERED', friendly_name='chldrn coverd'), + ] + )) assert result[0] is False val = validators.if_then_validator( @@ -719,7 +790,14 @@ def test_validate_child_care(self, record): ) record.CC_AMOUNT = 10 record.CC_NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='CC_AMOUNT', friendly_name='cc amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CC_NBR_MONTHS', friendly_name='cc nbr mnths'), + ] + )) assert result[0] is False def test_validate_transportation(self, record): @@ -728,12 +806,26 @@ def test_validate_transportation(self, record): condition_field_name='TRANSP_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='TRANSP_NBR_MONTHS', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='TRANSP_AMOUNT', friendly_name='transp amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='TRANSP_NBR_MONTHS', friendly_name='transp nbr months'), + ] + )) assert result == (True, None, ['TRANSP_AMOUNT', 'TRANSP_NBR_MONTHS']) record.TRANSP_AMOUNT = 1 record.TRANSP_NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='TRANSP_AMOUNT', friendly_name='transp amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='TRANSP_NBR_MONTHS', friendly_name='transp nbr months'), + ] + )) assert result[0] is False def test_validate_transitional_services(self, record): @@ -742,12 +834,26 @@ def test_validate_transitional_services(self, record): condition_field_name='TRANSITION_SERVICES_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='TRANSITION_NBR_MONTHS', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='TRANSITION_SERVICES_AMOUNT', friendly_name='transition serv amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='TRANSITION_NBR_MONTHS', friendly_name='transition nbr months'), + ] + )) assert result == (True, None, ['TRANSITION_SERVICES_AMOUNT', 'TRANSITION_NBR_MONTHS']) record.TRANSITION_SERVICES_AMOUNT = 1 record.TRANSITION_NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='TRANSITION_SERVICES_AMOUNT', friendly_name='transition serv amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='TRANSITION_NBR_MONTHS', friendly_name='transition nbr months'), + ] + )) assert result[0] is False def test_validate_other(self, record): @@ -756,12 +862,26 @@ def test_validate_other(self, record): condition_field_name='OTHER_AMOUNT', condition_function=validators.isLargerThan(0), result_field_name='OTHER_NBR_MONTHS', result_function=validators.isLargerThan(0), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='OTHER_AMOUNT', friendly_name='other amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='OTHER_NBR_MONTHS', friendly_name='other nbr months'), + ] + )) assert result == (True, None, ['OTHER_AMOUNT', 'OTHER_NBR_MONTHS']) record.OTHER_AMOUNT = 1 record.OTHER_NBR_MONTHS = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='OTHER_AMOUNT', friendly_name='other amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='OTHER_NBR_MONTHS', friendly_name='other nbr months'), + ] + )) assert result[0] is False def test_validate_reasons_for_amount_of_assistance_reductions(self, record): @@ -771,12 +891,26 @@ def test_validate_reasons_for_amount_of_assistance_reductions(self, record): result_field_name='WORK_REQ_SANCTION', result_function=validators.oneOf((1, 2)), ) record.SANC_REDUCTION_AMT = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='SANC_REDUCTION_AMT', friendly_name='sanc reduction amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_REQ_SANCTION', friendly_name='work req sanction'), + ] + )) assert result == (True, None, ['SANC_REDUCTION_AMT', 'WORK_REQ_SANCTION']) record.SANC_REDUCTION_AMT = 10 record.WORK_REQ_SANCTION = -1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='SANC_REDUCTION_AMT', friendly_name='sanc reduction amt'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_REQ_SANCTION', friendly_name='work req sanction'), + ] + )) assert result[0] is False def test_validate_sum(self, record): @@ -793,7 +927,22 @@ def test_validate_sum(self, record): record.TRANSP_AMOUNT = 0 record.TRANSITION_SERVICES_AMOUNT = 0 record.OTHER_AMOUNT = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='AMT_FOOD_STAMP_ASSISTANCE', friendly_name='amt food stamp assis'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='AMT_SUB_CC', friendly_name='amt sub cc'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='CC_AMOUNT', friendly_name='cc amt'), + Field(item=4, startIndex=5, endIndex=6, type='string', + name='TRANSP_AMOUNT', friendly_name='transp amt'), + Field(item=5, startIndex=6, endIndex=7, type='string', + name='TRANSITION_SERVICES_AMOUNT', friendly_name='transition serv amt'), + Field(item=6, startIndex=7, endIndex=8, type='string', + name='OTHER_AMOUNT', friendly_name='other amt'), + ] + )) assert result[0] is False @@ -816,12 +965,26 @@ def test_validate_ssn(self, record): ) record.SSN = "999989999" record.FAMILY_AFFILIATION = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='ssn'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'SSN']) record.FAMILY_AFFILIATION = 1 record.SSN = "999999999" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='ssn'), + ] + )) assert result[0] is False def test_validate_race_ethnicity(self, record): @@ -833,7 +996,14 @@ def test_validate_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2, 3)), result_field_name=race, result_function=validators.isInLimits(1, 2), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='race'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) record.FAMILY_AFFILIATION = 0 @@ -842,7 +1012,14 @@ def test_validate_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2, 3)), result_field_name=race, result_function=validators.isInLimits(1, 2) ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='race'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) def test_validate_marital_status(self, record): @@ -852,12 +1029,26 @@ def test_validate_marital_status(self, record): result_field_name='MARITAL_STATUS', result_function=validators.isInLimits(1, 5), ) record.FAMILY_AFFILIATION = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='married?'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'MARITAL_STATUS']) record.FAMILY_AFFILIATION = 3 record.MARITAL_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='married?'), + ] + )) assert result[0] is False def test_validate_parent_with_minor(self, record): @@ -866,11 +1057,25 @@ def test_validate_parent_with_minor(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), result_field_name='PARENT_MINOR_CHILD', result_function=validators.isInLimits(1, 3), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) record.PARENT_MINOR_CHILD = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result[0] is False def test_validate_education_level(self, record): @@ -884,12 +1089,26 @@ def test_validate_education_level(self, record): "98", "99")), ) record.FAMILY_AFFILIATION = 3 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EDUCATION_LEVEL']) record.FAMILY_AFFILIATION = 1 record.EDUCATION_LEVEL = "00" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result[0] is False def test_validate_citizenship(self, record): @@ -899,12 +1118,26 @@ def test_validate_citizenship(self, record): result_field_name='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2)), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS']) record.FAMILY_AFFILIATION = 1 record.CITIZENSHIP_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result[0] is False def test_validate_cooperation_with_child_support(self, record): @@ -914,12 +1147,26 @@ def test_validate_cooperation_with_child_support(self, record): result_field_name='COOPERATION_CHILD_SUPPORT', result_function=validators.oneOf((1, 2, 9)), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='COOPERATION_CHILD_SUPPORT', friendly_name='cooperation child support'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'COOPERATION_CHILD_SUPPORT']) record.FAMILY_AFFILIATION = 1 record.COOPERATION_CHILD_SUPPORT = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='COOPERATION_CHILD_SUPPORT', friendly_name='cooperation child support'), + ] + )) assert result[0] is False def test_validate_employment_status(self, record): @@ -929,12 +1176,26 @@ def test_validate_employment_status(self, record): result_field_name='EMPLOYMENT_STATUS', result_function=validators.isInLimits(1, 3), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EMPLOYMENT_STATUS', friendly_name='employment status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EMPLOYMENT_STATUS']) record.FAMILY_AFFILIATION = 3 record.EMPLOYMENT_STATUS = 4 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EMPLOYMENT_STATUS', friendly_name='employment status'), + ] + )) assert result[0] is False def test_validate_work_eligible_indicator(self, record): @@ -947,12 +1208,26 @@ def test_validate_work_eligible_indicator(self, record): ), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_ELIGIBLE_INDICATOR', friendly_name='work eligible indicator'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'WORK_ELIGIBLE_INDICATOR']) record.FAMILY_AFFILIATION = 1 record.WORK_ELIGIBLE_INDICATOR = "00" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_ELIGIBLE_INDICATOR', friendly_name='work eligible indicator'), + ] + )) assert result[0] is False def test_validate_work_participation(self, record): @@ -964,12 +1239,26 @@ def test_validate_work_participation(self, record): '19', '99']), ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_PART_STATUS', friendly_name='work part status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'WORK_PART_STATUS']) record.FAMILY_AFFILIATION = 2 record.WORK_PART_STATUS = "04" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_PART_STATUS', friendly_name='work part status'), + ] + )) assert result[0] is False val = validators.if_then_validator( @@ -980,7 +1269,14 @@ def test_validate_work_participation(self, record): ) record.WORK_PART_STATUS = "99" record.WORK_ELIGIBLE_INDICATOR = "01" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='WORK_ELIGIBLE_INDICATOR', friendly_name='work eligible indicator'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='WORK_PART_STATUS', friendly_name='work part status'), + ] + )) assert result[0] is False @@ -1000,12 +1296,26 @@ def test_validate_ssn(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.matches(1), result_field_name='SSN', result_function=validators.notOneOf(("999999999", "000000000")), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'SSN']) record.FAMILY_AFFILIATION = 1 record.SSN = "999999999" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result[0] is False def test_validate_t3_race_ethnicity(self, record): @@ -1017,7 +1327,14 @@ def test_validate_t3_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), result_field_name=race, result_function=validators.oneOf((1, 2)), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='race'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) record.FAMILY_AFFILIATION = 0 @@ -1026,7 +1343,14 @@ def test_validate_t3_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.oneOf((1, 2)), result_field_name=race, result_function=validators.oneOf((1, 2)), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='race'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) def test_validate_relationship_hoh(self, record): @@ -1037,12 +1361,26 @@ def test_validate_relationship_hoh(self, record): ) record.FAMILY_AFFILIATION = 0 record.RELATIONSHIP_HOH = "04" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='RELATIONSHIP_HOH', friendly_name='relationship hoh'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'RELATIONSHIP_HOH']) record.FAMILY_AFFILIATION = 1 record.RELATIONSHIP_HOH = "01" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='RELATIONSHIP_HOH', friendly_name='relationship hoh'), + ] + )) assert result[0] is False def test_validate_t3_education_level(self, record): @@ -1052,12 +1390,26 @@ def test_validate_t3_education_level(self, record): result_field_name='EDUCATION_LEVEL', result_function=validators.notMatches("99"), ) record.FAMILY_AFFILIATION = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='ed lev'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EDUCATION_LEVEL']) record.FAMILY_AFFILIATION = 1 record.EDUCATION_LEVEL = "99" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='ed lev'), + ] + )) assert result[0] is False def test_validate_t3_citizenship(self, record): @@ -1067,12 +1419,26 @@ def test_validate_t3_citizenship(self, record): result_field_name='CITIZENSHIP_STATUS', result_function=validators.oneOf((1, 2)), ) record.FAMILY_AFFILIATION = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS']) record.FAMILY_AFFILIATION = 1 record.CITIZENSHIP_STATUS = 3 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result[0] is False val = validators.if_then_validator( @@ -1081,7 +1447,14 @@ def test_validate_t3_citizenship(self, record): ) record.FAMILY_AFFILIATION = 2 record.CITIZENSHIP_STATUS = 3 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result[0] is False @@ -1100,26 +1473,58 @@ def test_validate_ssn(self, record): result_field_name='SSN', result_function=validators.isNumber() ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'SSN']) record.SSN = "abc" record.FAMILY_AFFILIATION = 2 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result[0] is False def test_validate_ssn_citizenship(self, record): """Test cat3 validator for SSN/citizenship.""" val = validators.validate__FAM_AFF__SSN() - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS', 'SSN']) record.FAMILY_AFFILIATION = 2 record.SSN = "000000000" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='CITIZENSHIP_STATUS', friendly_name='cit stat'), + ] + )) assert result[0] is False def test_validate_race_ethnicity(self, record): @@ -1129,10 +1534,17 @@ def test_validate_race_ethnicity(self, record): for race in races: val = validators.if_then_validator( condition_field_name='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), - result_field_name='RACE_HISPANIC', result_function=validators.isInLimits(1, 2) + result_field_name=race, result_function=validators.isInLimits(1, 2) ) - result = val(record, RowSchema()) - assert result == (True, None, ['FAMILY_AFFILIATION', 'RACE_HISPANIC']) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='social'), + ] + )) + assert result == (True, None, ['FAMILY_AFFILIATION', race]) record.FAMILY_AFFILIATION = 1 record.RACE_HISPANIC = 0 @@ -1146,7 +1558,14 @@ def test_validate_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), result_field_name=race, result_function=validators.isInLimits(1, 2) ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='social'), + ] + )) assert result[0] is False def test_validate_marital_status(self, record): @@ -1157,13 +1576,27 @@ def test_validate_marital_status(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='marital status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'MARITAL_STATUS']) record.FAMILY_AFFILIATION = 2 record.MARITAL_STATUS = 6 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='marital status'), + ] + )) assert result[0] is False def test_validate_parent_minor(self, record): @@ -1174,13 +1607,27 @@ def test_validate_parent_minor(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) record.FAMILY_AFFILIATION = 2 record.PARENT_MINOR_CHILD = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result[0] is False def test_validate_education(self, record): @@ -1194,13 +1641,27 @@ def test_validate_education(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EDUCATION_LEVEL']) record.FAMILY_AFFILIATION = 2 record.EDUCATION_LEVEL = "0" - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result[0] is False def test_validate_citizenship_status(self, record): @@ -1211,13 +1672,27 @@ def test_validate_citizenship_status(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS']) record.FAMILY_AFFILIATION = 1 record.CITIZENSHIP_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result[0] is False def test_validate_oasdi_insurance(self, record): @@ -1228,13 +1703,27 @@ def test_validate_oasdi_insurance(self, record): ) record.DATE_OF_BIRTH = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='DATE_OF_BIRTH', friendly_name='dob'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_OASDI_INSURANCE', friendly_name='rec oasdi insurance'), + ] + )) assert result == (True, None, ['DATE_OF_BIRTH', 'REC_OASDI_INSURANCE']) record.DATE_OF_BIRTH = 200001 record.REC_OASDI_INSURANCE = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='DATE_OF_BIRTH', friendly_name='dob'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_OASDI_INSURANCE', friendly_name='rec oasdi insurance'), + ] + )) assert result[0] is False def test_validate_federal_disability(self, record): @@ -1245,13 +1734,27 @@ def test_validate_federal_disability(self, record): ) record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_FEDERAL_DISABILITY', friendly_name='rec fed disability'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'REC_FEDERAL_DISABILITY']) record.FAMILY_AFFILIATION = 1 record.REC_FEDERAL_DISABILITY = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_FEDERAL_DISABILITY', friendly_name='rec fed disability'), + ] + )) assert result[0] is False @@ -1268,12 +1771,30 @@ def test_sum_of_applications(self, record): val = validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]) record.NUM_APPLICATIONS = 2 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_APPLICATIONS', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_APPROVED', friendly_name='num approved'), + Field(item=2, startIndex=4, endIndex=5, type='string', + name='NUM_DENIED', friendly_name='num denied'), + ] + )) assert result == (True, None, ['NUM_APPLICATIONS', 'NUM_APPROVED', 'NUM_DENIED']) record.NUM_APPLICATIONS = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_APPLICATIONS', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_APPROVED', friendly_name='num approved'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_DENIED', friendly_name='num denied'), + ] + )) assert result[0] is False @@ -1282,12 +1803,34 @@ def test_sum_of_families(self, record): val = validators.sumIsEqual("NUM_FAMILIES", ["NUM_2_PARENTS", "NUM_1_PARENTS", "NUM_NO_PARENTS"]) record.NUM_FAMILIES = 3 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_FAMILIES', friendly_name='num fam'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_2_PARENTS', friendly_name='num 2 parent'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_1_PARENTS', friendly_name='num 2 parent'), + Field(item=4, startIndex=5, endIndex=6, type='string', + name='NUM_NO_PARENTS', friendly_name='num 0 parent'), + ] + )) assert result == (True, None, ['NUM_FAMILIES', 'NUM_2_PARENTS', 'NUM_1_PARENTS', 'NUM_NO_PARENTS']) record.NUM_FAMILIES = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_FAMILIES', friendly_name='num fam'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_2_PARENTS', friendly_name='num 2 parent'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_1_PARENTS', friendly_name='num 2 parent'), + Field(item=4, startIndex=5, endIndex=6, type='string', + name='NUM_NO_PARENTS', friendly_name='num 0 parent'), + ] + )) assert result[0] is False @@ -1296,12 +1839,30 @@ def test_sum_of_recipients(self, record): val = validators.sumIsEqual("NUM_RECIPIENTS", ["NUM_ADULT_RECIPIENTS", "NUM_CHILD_RECIPIENTS"]) record.NUM_RECIPIENTS = 2 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_RECIPIENTS', friendly_name='num recip'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_ADULT_RECIPIENTS', friendly_name='num adult recip'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_CHILD_RECIPIENTS', friendly_name='num child recip'), + ] + )) assert result == (True, None, ['NUM_RECIPIENTS', 'NUM_ADULT_RECIPIENTS', 'NUM_CHILD_RECIPIENTS']) record.NUM_RECIPIENTS = 1 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='NUM_RECIPIENTS', friendly_name='num recip'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='NUM_ADULT_RECIPIENTS', friendly_name='num adult recip'), + Field(item=3, startIndex=4, endIndex=5, type='string', + name='NUM_CHILD_RECIPIENTS', friendly_name='num child recip'), + ] + )) assert result[0] is False @@ -1320,11 +1881,25 @@ def test_fam_affil_ssn(self, record): result_field_name='SSN', result_function=validators.validateSSN(), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result == (True, None, ["FAMILY_AFFILIATION", "SSN"]) record.SSN = '111111111' - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='SSN', friendly_name='social'), + ] + )) assert result[0] is False @@ -1336,7 +1911,14 @@ def test_validate_race_ethnicity(self, record): condition_field_name='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3), result_field_name=race, result_function=validators.isInLimits(1, 2), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name=race, friendly_name='social'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', race]) def test_fam_affil_marital_stat(self, record): @@ -1346,12 +1928,26 @@ def test_fam_affil_marital_stat(self, record): result_field_name='MARITAL_STATUS', result_function=validators.isInLimits(1, 5), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='marital status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'MARITAL_STATUS']) record.MARITAL_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='MARITAL_STATUS', friendly_name='marital status'), + ] + )) assert result[0] is False def test_fam_affil_parent_with_minor(self, record): @@ -1361,12 +1957,26 @@ def test_fam_affil_parent_with_minor(self, record): result_field_name='PARENT_MINOR_CHILD', result_function=validators.isInLimits(1, 3), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'PARENT_MINOR_CHILD']) record.PARENT_MINOR_CHILD = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='PARENT_MINOR_CHILD', friendly_name='parent minor child'), + ] + )) assert result[0] is False def test_fam_affil_ed_level(self, record): @@ -1377,12 +1987,26 @@ def test_fam_affil_ed_level(self, record): validators.isInStringRange(1, 16), validators.isInStringRange(98, 99)), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'EDUCATION_LEVEL']) record.EDUCATION_LEVEL = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='EDUCATION_LEVEL', friendly_name='education level'), + ] + )) assert result[0] is False def test_fam_affil_citz_stat(self, record): @@ -1392,12 +2016,26 @@ def test_fam_affil_citz_stat(self, record): result_field_name='CITIZENSHIP_STATUS', result_function=validators.isInLimits(1, 3), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS']) record.CITIZENSHIP_STATUS = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='CITIZENSHIP_STATUS', friendly_name='citizenship status'), + ] + )) assert result[0] is False def test_dob_oasdi_insur(self, record): @@ -1407,12 +2045,26 @@ def test_dob_oasdi_insur(self, record): result_field_name='REC_OASDI_INSURANCE', result_function=validators.isInLimits(1, 2), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='DATE_OF_BIRTH', friendly_name='dob'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_OASDI_INSURANCE', friendly_name='rec oasdi insurance'), + ] + )) assert result == (True, None, ['DATE_OF_BIRTH', 'REC_OASDI_INSURANCE']) record.REC_OASDI_INSURANCE = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='DATE_OF_BIRTH', friendly_name='dob'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_OASDI_INSURANCE', friendly_name='rec oasdi insurance'), + ] + )) assert result[0] is False def test_fam_affil_fed_disability(self, record): @@ -1422,11 +2074,25 @@ def test_fam_affil_fed_disability(self, record): result_field_name='REC_FEDERAL_DISABILITY', result_function=validators.isInLimits(1, 2), ) - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_FEDERAL_DISABILITY', friendly_name='rec fed disability'), + ] + )) assert result == (True, None, ['FAMILY_AFFILIATION', 'REC_FEDERAL_DISABILITY']) record.REC_FEDERAL_DISABILITY = 0 - result = val(record, RowSchema()) + result = val(record, RowSchema( + fields=[ + Field(item=1, startIndex=0, endIndex=2, type='string', + name='FAMILY_AFFILIATION', friendly_name='fam affil'), + Field(item=2, startIndex=2, endIndex=4, type='string', + name='REC_FEDERAL_DISABILITY', friendly_name='rec fed disability'), + ] + )) assert result[0] is False def test_is_quiet_preparser_errors(): diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 9f99e928c..d8583cd90 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -22,27 +22,41 @@ def value_is_empty(value, length, extra_vals={}): return value is None or value in empty_values + +def format_error_context(row_schema, friendly_name, item_num, format='prefix'): + """Format the error message for consistency across cat2 validators.""" + match format: + case 'inline': + return f'Item {item_num} ({friendly_name})' + + case 'prefix' | _: + return f'{row_schema.record_type} Item {item_num} ({friendly_name}):' + + # higher order validator functions def make_validator(validator_func, error_func): """Return a function accepting a value input and returning (bool, string) to represent validation state.""" - def validator(value, row_schema=None, friendly_name=None, item_num=None): + + # struct + # or kwargs + def validator(value, row_schema=None, friendly_name=None, item_num=None, error_context_format='prefix'): try: if validator_func(value): return (True, None) - return (False, error_func(value, row_schema, friendly_name, item_num)) + return (False, error_func(value, row_schema, friendly_name, item_num, error_context_format)) except Exception as e: logger.debug(f"Caught exception in validator. Exception: {e}") - return (False, error_func(value, row_schema, friendly_name, item_num)) + return (False, error_func(value, row_schema, friendly_name, item_num, error_context_format)) return validator def or_validators(*args, **kwargs): """Return a validator that is true only if one of the validators is true.""" return ( - lambda value, row_schema, friendly_name, item_num: (True, None) - if any([validator(value, row_schema, friendly_name, item_num)[0] for validator in args]) - else (False, " or ".join([validator(value, row_schema, friendly_name, item_num)[1] for validator in args])) + lambda value, row_schema, friendly_name, item_num, error_context_format='inline': (True, None) + if any([validator(value, row_schema, friendly_name, item_num, error_context_format)[0] for validator in args]) + else (False, " or ".join([validator(value, row_schema, friendly_name, item_num, error_context_format)[1] for validator in args])) ) @@ -50,14 +64,14 @@ def and_validators(validator1, validator2): """Return a validator that is true only if both validators are true.""" return ( lambda value, row_schema, friendly_name, item_num: (True, None) - if (validator1(value, row_schema, friendly_name, item_num)[0] and validator2(value, row_schema, - friendly_name, item_num)[0]) + if (validator1(value, row_schema, friendly_name, item_num, 'inline')[0] and validator2(value, row_schema, + friendly_name, item_num, 'inline')[0]) else ( False, - (validator1(value, row_schema, friendly_name, item_num)[1]) - if validator1(value, row_schema, friendly_name, item_num)[1] is not None + (validator1(value, row_schema, friendly_name, item_num, 'inline')[1]) + if validator1(value, row_schema, friendly_name, item_num, 'inline')[1] is not None else "" + " and " + validator2(value)[1] - if validator2(value, row_schema, friendly_name, item_num)[1] is not None + if validator2(value, row_schema, friendly_name, item_num, 'inline')[1] is not None else "", ) ) @@ -69,9 +83,9 @@ def or_priority_validators(validators=[]): """ def or_priority_validators_func(value, rows_schema, friendly_name=None, item_num=None): for validator in validators: - if not validator(value, rows_schema, friendly_name, item_num)[0]: + if not validator(value, rows_schema, friendly_name, item_num, 'inline')[0]: return (False, validator(value, rows_schema, - friendly_name, item_num)[1]) + friendly_name, item_num, 'inline')[1]) return (True, None) return or_priority_validators_func @@ -80,14 +94,14 @@ def or_priority_validators_func(value, rows_schema, friendly_name=None, item_num def extended_and_validators(*args, **kwargs): """Return a validator that is true only if all validators are true.""" def returned_func(value, row_schema, friendly_name, item_num): - if all([validator(value, row_schema, friendly_name, item_num)[0] for validator in args]): + if all([validator(value, row_schema, friendly_name, item_num, 'inline')[0] for validator in args]): return (True, None) else: return (False, "".join( [ " and " + validator(value, row_schema, - friendly_name, item_num)[1] if validator(value, row_schema, - friendly_name, item_num)[0] else "" + friendly_name, item_num, 'inline')[1] if validator(value, row_schema, + friendly_name, item_num, 'inline')[0] else "" for validator in args ] )) @@ -115,9 +129,12 @@ def if_then_validator_func(value, row_schema): value[result_field_name] if type(value) is dict else getattr(value, result_field_name) ) + condition_field = row_schema.get_field_by_name(condition_field_name) + result_field = row_schema.get_field_by_name(result_field_name) + # TODO: There is some work to be done here to get the actual friendly name and item numbers of the fields - validator1_result = condition_function(value1, row_schema, condition_field_name, "-1") - validator2_result = result_function(value2, row_schema, result_field_name, "-1") + validator1_result = condition_function(value1, row_schema, condition_field.friendly_name, condition_field.item, 'inline') + validator2_result = result_function(value2, row_schema, result_field.friendly_name, result_field.item, 'inline') if not validator1_result[0]: returned_value = (True, None, [condition_field_name, result_field_name]) @@ -179,7 +196,7 @@ def sumIsEqualFunc(value, row_schema): def field_year_month_with_header_year_quarter(): """Validate that the field year and month match the header year and quarter.""" - def validate_reporting_month_year_fields_header(line, row_schema, friendly_name, item_num): + def validate_reporting_month_year_fields_header(line, row_schema, friendly_name, item_num, error_context_format=None): field_month_year = row_schema.get_field_values_by_names(line, ['RPT_MONTH_YEAR']).get('RPT_MONTH_YEAR') df_quarter = row_schema.datafile.quarter @@ -225,7 +242,8 @@ def recordHasLength(length): lambda value, row_schema, friendly_name, - item_num: f"{row_schema.record_type}: record length is {len(value)} characters but must be {length}.", + item_num, + error_context_format: f"{row_schema.record_type}: record length is {len(value)} characters but must be {length}.", ) @@ -233,7 +251,7 @@ def recordHasLengthBetween(lower, upper, error_func=None): """Validate that value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) >= lower and len(value) <= upper, - lambda value, row_schema, friendly_name, item_num: error_func(value, lower, upper) + lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(value, lower, upper) if error_func else f"{row_schema.record_type}: record length of {len(value)} characters is not in the range [{lower}, {upper}].", @@ -245,7 +263,7 @@ def caseNumberNotEmpty(start=0, end=None): return make_validator( lambda value: not _is_empty(value, start, end), lambda value, row_schema, - friendly_name, item_num: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' + friendly_name, item_num, error_context_format: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' ) @@ -257,7 +275,8 @@ def calendarQuarterIsValid(start=0, end=None): lambda value, row_schema, friendly_name, - item_num: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " + item_num, + error_context_format: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " "representing the Calendar Year and Quarter formatted as YYYYQ", ) @@ -265,18 +284,13 @@ def calendarQuarterIsValid(start=0, end=None): # generic validators -def format_error_context(row_schema, friendly_name, item_num): - """Format the error message for consistency across cat2 validators.""" - return f'{row_schema.record_type} Item {item_num} ({friendly_name})' - - def matches(option, error_func=None): """Validate that value is equal to option.""" return make_validator( lambda value: value == option, - lambda value, row_schema, friendly_name, item_num: error_func(option) + lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(option) if error_func - else f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not match {option}.", + else f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not match {option}.", ) @@ -284,8 +298,8 @@ def notMatches(option): """Validate that value is not equal to option.""" return make_validator( lambda value: value != option, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} matches {option}." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} matches {option}." ) @@ -306,8 +320,8 @@ def check_option(value, options): return make_validator( lambda value: check_option(value, options), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " f"{value} is not in {clean_options_string(options)}." ) @@ -316,8 +330,8 @@ def notOneOf(options=[]): """Validate that value exists in the provided options array.""" return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " f"{value} is in {clean_options_string(options)}." ) @@ -326,8 +340,8 @@ def between(min, max): """Validate value, when casted to int, is greater than min and less than max.""" return make_validator( lambda value: int(value) > min and int(value) < max, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not between {min} and {max}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not between {min} and {max}.", ) @@ -338,7 +352,8 @@ def fieldHasLength(length): lambda value, row_schema, friendly_name, - item_num: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", + item_num, + error_context_format: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", ) @@ -349,7 +364,8 @@ def hasLengthGreaterThan(val, error_func=None): lambda value, row_schema, friendly_name, - item_num: f"Value length {len(value)} is not greater than {val}.", + item_num, + error_context_format: f"Value length {len(value)} is not greater than {val}.", ) @@ -357,8 +373,8 @@ def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( lambda value: sum(c.isdigit() for c in str(value)) == num_digits, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " f"{value} does not have exactly {num_digits} digits.", ) @@ -367,8 +383,8 @@ def contains(substring): """Validate that string value contains the given substring param.""" return make_validator( lambda value: value.find(substring) != -1, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not contain {substring}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not contain {substring}.", ) @@ -376,9 +392,9 @@ def startsWith(substring, error_func=None): """Validate that string value starts with the given substring param.""" return make_validator( lambda value: value.startswith(substring), - lambda value, row_schema, friendly_name, item_num: error_func(substring) + lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(substring) if error_func - else f"{format_error_context(row_schema, friendly_name, item_num)}: {value} does not start with {substring}.", + else f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not start with {substring}.", ) @@ -386,8 +402,8 @@ def isNumber(): """Validate that value can be casted to a number.""" return make_validator( lambda value: str(value).strip().isnumeric(), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not a number." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not a number." ) @@ -395,8 +411,8 @@ def isAlphaNumeric(): """Validate that value is alphanumeric.""" return make_validator( lambda value: value.isalnum(), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not alphanumeric." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not alphanumeric." ) @@ -404,8 +420,8 @@ def isBlank(): """Validate that string value is blank.""" return make_validator( lambda value: value.isspace(), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not blank." ) @@ -413,8 +429,8 @@ def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( lambda value: int(value) >= lower and int(value) <= upper, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not in range [{lower}, {upper}].", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not in range [{lower}, {upper}].", ) @@ -422,8 +438,8 @@ def isStringLargerThan(val): """Validate that string value is larger than val.""" return make_validator( lambda value: int(value) > val, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {val}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {val}.", ) @@ -438,8 +454,8 @@ def notEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( lambda value: not _is_empty(value, start, end), - lambda value, row_schema, friendly_name, item_num: - f'{format_error_context(row_schema, friendly_name, item_num)}: {str(value)} contains blanks ' + lambda value, row_schema, friendly_name, item_num, error_context_format: + f'{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)} contains blanks ' f'between positions {start} and {end if end else len(str(value))}.' ) @@ -448,8 +464,8 @@ def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( lambda value: _is_empty(value, start, end), - lambda value, row_schema, friendly_name, item_num: - f'{format_error_context(row_schema, friendly_name, item_num)}: {value} is not blank ' + lambda value, row_schema, friendly_name, item_num, error_context_format: + f'{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not blank ' f'between positions {start} and {end if end else len(value)}.' ) @@ -458,8 +474,8 @@ def notZero(number_of_zeros=1): """Validate that value is not zero.""" return make_validator( lambda value: value != "0" * number_of_zeros, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is zero." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is zero." ) @@ -467,8 +483,8 @@ def isLargerThan(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: float(value) > LowerBound if value is not None else False, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {LowerBound}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {LowerBound}.", ) @@ -476,8 +492,8 @@ def isSmallerThan(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value < UpperBound, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not smaller than {UpperBound}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not smaller than {UpperBound}.", ) @@ -485,8 +501,8 @@ def isLargerThanOrEqualTo(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: value >= LowerBound, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger than {LowerBound}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {LowerBound}.", ) @@ -494,8 +510,8 @@ def isSmallerThanOrEqualTo(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value <= UpperBound, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not smaller than {UpperBound}.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not smaller than {UpperBound}.", ) @@ -503,8 +519,8 @@ def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( lambda value: int(value) >= LowerBound and int(value) <= UpperBound, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not larger or equal " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger or equal " f"to {LowerBound} and smaller or equal to {UpperBound}." ) @@ -514,16 +530,16 @@ def dateMonthIsValid(): """Validate that in a monthyear combination, the month is a valid month.""" return make_validator( lambda value: int(str(value)[4:6]) in range(1, 13), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[4:6]} is not a valid month.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[4:6]} is not a valid month.", ) def dateDayIsValid(): """Validate that in a monthyearday combination, the day is a valid day.""" return make_validator( lambda value: int(str(value)[6:]) in range(1, 32), - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[6:]} is not a valid day.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[6:]} is not a valid day.", ) @@ -531,8 +547,8 @@ def olderThan(min_age): """Validate that value is larger than min_age.""" return make_validator( lambda value: datetime.date.today().year - int(str(value)[:4]) > min_age, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[:4]} must be less " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[:4]} must be less " f"than or equal to {datetime.date.today().year - min_age} to meet the minimum age requirement." ) @@ -541,8 +557,8 @@ def dateYearIsLargerThan(year): """Validate that in a monthyear combination, the year is larger than the given year.""" return make_validator( lambda value: int(str(value)[:4]) > year, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " f"Year {str(value)[:4]} must be larger than {year}.", ) @@ -551,8 +567,8 @@ def quarterIsValid(): """Validate in a year quarter combination, the quarter is valid.""" return make_validator( lambda value: int(str(value)[-1]) > 0 and int(str(value)[-1]) < 5, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {str(value)[-1]} is not a valid quarter.", + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[-1]} is not a valid quarter.", ) @@ -561,8 +577,8 @@ def validateSSN(): options = [str(i) * 9 for i in range(0, 10)] return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is in {options}." + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is in {options}." ) @@ -570,8 +586,8 @@ def validateRace(): """Validate race.""" return make_validator( lambda value: value >= 0 and value <= 2, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: {value} is not greater than or equal to 0 " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not greater than or equal to 0 " "or smaller than or equal to 2." ) @@ -582,8 +598,8 @@ def validateRptMonthYear(): lambda value: value[2:8].isdigit() and int(value[2:6]) > 1900 and value[6:8] in {"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"}, - lambda value, row_schema, friendly_name, item_num: - f"{format_error_context(row_schema, friendly_name, item_num)}: The value: {value[2:8]}, " + lambda value, row_schema, friendly_name, item_num, error_context_format: + f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} The value: {value[2:8]}, " "does not follow the YYYYMM format for Reporting Year and Month.", ) From 1ecc38ceaf2874ba2c8885008aceb8059768215a Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 14 Jun 2024 11:08:08 -0400 Subject: [PATCH 32/52] create ValidationErrorArgs dto --- tdrs-backend/tdpservice/parsers/row_schema.py | 12 +- tdrs-backend/tdpservice/parsers/validators.py | 181 ++++++++---------- 2 files changed, 94 insertions(+), 99 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index 39f3814f3..01a87f046 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -1,7 +1,7 @@ """Row schema for datafile.""" from .models import ParserErrorCategoryChoices from .fields import Field, TransformField -from .validators import value_is_empty, format_error_context +from .validators import value_is_empty, format_error_context, ValidationErrorArgs import logging logger = logging.getLogger(__name__) @@ -146,12 +146,20 @@ def run_field_validators(self, instance, generate_error): ) elif field.required: is_valid = False + eargs = ValidationErrorArgs( + value=value, + row_schema=self, + friendly_name=field.friendly_name, + item_num=field.item, + error_context_format='prefix' + ) + errors.append( generate_error( schema=self, error_category=ParserErrorCategoryChoices.FIELD_VALUE, error_message=( - f"{format_error_context(self, field.friendly_name, field.item)} " + f"{format_error_context(eargs)} " "field is required but a value was not provided." ), record=instance, diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index d8583cd90..f330670a1 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -1,9 +1,12 @@ """Generic parser validator functions for use in schema definitions.""" -from .models import ParserErrorCategoryChoices -from .util import fiscal_to_calendar, year_month_to_year_quarter, clean_options_string import datetime import logging +from dataclasses import dataclass +from typing import Any +# from tdpservice.parsers.row_schema import RowSchema +from tdpservice.parsers.models import ParserErrorCategoryChoices +from tdpservice.parsers.util import fiscal_to_calendar, year_month_to_year_quarter, clean_options_string logger = logging.getLogger(__name__) @@ -23,31 +26,49 @@ def value_is_empty(value, length, extra_vals={}): return value is None or value in empty_values -def format_error_context(row_schema, friendly_name, item_num, format='prefix'): +@dataclass +class ValidationErrorArgs: + value: Any + row_schema: object # RowSchema causes circular import + friendly_name: str + item_num: str + error_context_format: str = 'prefix' + + +def format_error_context(eargs: ValidationErrorArgs): """Format the error message for consistency across cat2 validators.""" - match format: + match eargs.error_context_format: case 'inline': - return f'Item {item_num} ({friendly_name})' + return f'Item {eargs.item_num} ({eargs.friendly_name})' case 'prefix' | _: - return f'{row_schema.record_type} Item {item_num} ({friendly_name}):' + return f'{eargs.row_schema.record_type} Item {eargs.item_num} ({eargs.friendly_name}):' # higher order validator functions + def make_validator(validator_func, error_func): """Return a function accepting a value input and returning (bool, string) to represent validation state.""" # struct # or kwargs def validator(value, row_schema=None, friendly_name=None, item_num=None, error_context_format='prefix'): + eargs = ValidationErrorArgs( + value=value, + row_schema=row_schema, + friendly_name=friendly_name, + item_num=item_num, + error_context_format=error_context_format + ) + try: if validator_func(value): return (True, None) - return (False, error_func(value, row_schema, friendly_name, item_num, error_context_format)) + return (False, error_func(eargs)) except Exception as e: logger.debug(f"Caught exception in validator. Exception: {e}") - return (False, error_func(value, row_schema, friendly_name, item_num, error_context_format)) + return (False, error_func(eargs)) return validator @@ -239,11 +260,8 @@ def recordHasLength(length): """Validate that value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) == length, - lambda value, - row_schema, - friendly_name, - item_num, - error_context_format: f"{row_schema.record_type}: record length is {len(value)} characters but must be {length}.", + lambda eargs: f"{eargs.row_schema.record_type}: record length is " + f"{len(eargs.value)} characters but must be {length}.", ) @@ -251,10 +269,11 @@ def recordHasLengthBetween(lower, upper, error_func=None): """Validate that value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) >= lower and len(value) <= upper, - lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(value, lower, upper) + lambda eargs: error_func(eargs.value, lower, upper) if error_func else - f"{row_schema.record_type}: record length of {len(value)} characters is not in the range [{lower}, {upper}].", + f"{eargs.row_schema.record_type}: record length of {len(eargs.value)} " + f"characters is not in the range [{lower}, {upper}].", ) @@ -262,8 +281,7 @@ def caseNumberNotEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( lambda value: not _is_empty(value, start, end), - lambda value, row_schema, - friendly_name, item_num, error_context_format: f'{row_schema.record_type}: Case number {str(value)} cannot contain blanks.' + lambda eargs: f'{eargs.row_schema.record_type}: Case number {str(eargs.value)} cannot contain blanks.' ) @@ -272,12 +290,8 @@ def calendarQuarterIsValid(start=0, end=None): return make_validator( lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, - lambda value, - row_schema, - friendly_name, - item_num, - error_context_format: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " - "representing the Calendar Year and Quarter formatted as YYYYQ", + lambda eargs: f"{eargs.row_schema.record_type}: {eargs.value[start:end]} is invalid. " + "Calendar Quarter must be a numeric representing the Calendar Year and Quarter formatted as YYYYQ", ) @@ -288,9 +302,9 @@ def matches(option, error_func=None): """Validate that value is equal to option.""" return make_validator( lambda value: value == option, - lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(option) + lambda eargs: error_func(option) if error_func - else f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not match {option}.", + else f"{format_error_context(eargs)} {eargs.value} does not match {option}.", ) @@ -298,8 +312,7 @@ def notMatches(option): """Validate that value is not equal to option.""" return make_validator( lambda value: value != option, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} matches {option}." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} matches {option}." ) @@ -320,9 +333,8 @@ def check_option(value, options): return make_validator( lambda value: check_option(value, options), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " - f"{value} is not in {clean_options_string(options)}." + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is not in {clean_options_string(options)}." ) @@ -330,9 +342,8 @@ def notOneOf(options=[]): """Validate that value exists in the provided options array.""" return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " - f"{value} is in {clean_options_string(options)}." + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is in {clean_options_string(options)}." ) @@ -340,8 +351,8 @@ def between(min, max): """Validate value, when casted to int, is greater than min and less than max.""" return make_validator( lambda value: int(value) > min and int(value) < max, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not between {min} and {max}.", + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is not between {min} and {max}.", ) @@ -349,11 +360,8 @@ def fieldHasLength(length): """Validate that the field value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) == length, - lambda value, - row_schema, - friendly_name, - item_num, - error_context_format: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", + lambda eargs: + f"{eargs.row_schema.record_type} field length is {len(eargs.value)} characters but must be {length}.", ) @@ -361,11 +369,8 @@ def hasLengthGreaterThan(val, error_func=None): """Validate that value (string or array) has a length greater than val.""" return make_validator( lambda value: len(value) >= val, - lambda value, - row_schema, - friendly_name, - item_num, - error_context_format: f"Value length {len(value)} is not greater than {val}.", + lambda eargs: + f"Value length {len(eargs.value)} is not greater than {val}.", ) @@ -373,9 +378,8 @@ def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( lambda value: sum(c.isdigit() for c in str(value)) == num_digits, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " - f"{value} does not have exactly {num_digits} digits.", + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} does not have exactly {num_digits} digits.", ) @@ -383,8 +387,7 @@ def contains(substring): """Validate that string value contains the given substring param.""" return make_validator( lambda value: value.find(substring) != -1, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not contain {substring}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} does not contain {substring}.", ) @@ -392,9 +395,9 @@ def startsWith(substring, error_func=None): """Validate that string value starts with the given substring param.""" return make_validator( lambda value: value.startswith(substring), - lambda value, row_schema, friendly_name, item_num, error_context_format: error_func(substring) + lambda eargs: error_func(substring) if error_func - else f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} does not start with {substring}.", + else f"{format_error_context(eargs)} {eargs.value} does not start with {substring}.", ) @@ -402,8 +405,7 @@ def isNumber(): """Validate that value can be casted to a number.""" return make_validator( lambda value: str(value).strip().isnumeric(), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not a number." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not a number." ) @@ -411,8 +413,7 @@ def isAlphaNumeric(): """Validate that value is alphanumeric.""" return make_validator( lambda value: value.isalnum(), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not alphanumeric." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not alphanumeric." ) @@ -420,8 +421,7 @@ def isBlank(): """Validate that string value is blank.""" return make_validator( lambda value: value.isspace(), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not blank." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not blank." ) @@ -429,8 +429,7 @@ def isInStringRange(lower, upper): """Validate that string value is in a specific range.""" return make_validator( lambda value: int(value) >= lower and int(value) <= upper, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not in range [{lower}, {upper}].", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not in range [{lower}, {upper}].", ) @@ -438,8 +437,7 @@ def isStringLargerThan(val): """Validate that string value is larger than val.""" return make_validator( lambda value: int(value) > val, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {val}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not larger than {val}.", ) @@ -454,9 +452,9 @@ def notEmpty(start=0, end=None): """Validate that string value isn't only blanks.""" return make_validator( lambda value: not _is_empty(value, start, end), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f'{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)} contains blanks ' - f'between positions {start} and {end if end else len(str(value))}.' + lambda eargs: + f'{format_error_context(eargs)} {str(eargs.value)} contains blanks ' + f'between positions {start} and {end if end else len(str(eargs.value))}.' ) @@ -464,9 +462,9 @@ def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( lambda value: _is_empty(value, start, end), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f'{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not blank ' - f'between positions {start} and {end if end else len(value)}.' + lambda eargs: + f'{format_error_context(eargs)} {eargs.value} is not blank ' + f'between positions {start} and {end if end else len(eargs.value)}.' ) @@ -474,8 +472,7 @@ def notZero(number_of_zeros=1): """Validate that value is not zero.""" return make_validator( lambda value: value != "0" * number_of_zeros, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is zero." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is zero." ) @@ -483,8 +480,7 @@ def isLargerThan(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: float(value) > LowerBound if value is not None else False, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {LowerBound}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not larger than {LowerBound}.", ) @@ -492,8 +488,7 @@ def isSmallerThan(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value < UpperBound, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not smaller than {UpperBound}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not smaller than {UpperBound}.", ) @@ -501,8 +496,7 @@ def isLargerThanOrEqualTo(LowerBound): """Validate that value is larger than the given value.""" return make_validator( lambda value: value >= LowerBound, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger than {LowerBound}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not larger than {LowerBound}.", ) @@ -510,8 +504,7 @@ def isSmallerThanOrEqualTo(UpperBound): """Validate that value is smaller than the given value.""" return make_validator( lambda value: value <= UpperBound, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not smaller than {UpperBound}.", + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is not smaller than {UpperBound}.", ) @@ -519,8 +512,8 @@ def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( lambda value: int(value) >= LowerBound and int(value) <= UpperBound, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not larger or equal " + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is not larger or equal " f"to {LowerBound} and smaller or equal to {UpperBound}." ) @@ -530,16 +523,14 @@ def dateMonthIsValid(): """Validate that in a monthyear combination, the month is a valid month.""" return make_validator( lambda value: int(str(value)[4:6]) in range(1, 13), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[4:6]} is not a valid month.", + lambda eargs: f"{format_error_context(eargs)} {str(eargs.value)[4:6]} is not a valid month.", ) def dateDayIsValid(): """Validate that in a monthyearday combination, the day is a valid day.""" return make_validator( lambda value: int(str(value)[6:]) in range(1, 32), - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[6:]} is not a valid day.", + lambda eargs: f"{format_error_context(eargs)} {str(eargs.value)[6:]} is not a valid day.", ) @@ -547,8 +538,8 @@ def olderThan(min_age): """Validate that value is larger than min_age.""" return make_validator( lambda value: datetime.date.today().year - int(str(value)[:4]) > min_age, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[:4]} must be less " + lambda eargs: + f"{format_error_context(eargs)} {str(eargs.value)[:4]} must be less " f"than or equal to {datetime.date.today().year - min_age} to meet the minimum age requirement." ) @@ -557,9 +548,7 @@ def dateYearIsLargerThan(year): """Validate that in a monthyear combination, the year is larger than the given year.""" return make_validator( lambda value: int(str(value)[:4]) > year, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} " - f"Year {str(value)[:4]} must be larger than {year}.", + lambda eargs: f"{format_error_context(eargs)} Year {str(eargs.value)[:4]} must be larger than {year}.", ) @@ -567,8 +556,7 @@ def quarterIsValid(): """Validate in a year quarter combination, the quarter is valid.""" return make_validator( lambda value: int(str(value)[-1]) > 0 and int(str(value)[-1]) < 5, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {str(value)[-1]} is not a valid quarter.", + lambda eargs: f"{format_error_context(eargs)} {str(eargs.value)[-1]} is not a valid quarter.", ) @@ -577,8 +565,7 @@ def validateSSN(): options = [str(i) * 9 for i in range(0, 10)] return make_validator( lambda value: value not in options, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is in {options}." + lambda eargs: f"{format_error_context(eargs)} {eargs.value} is in {options}." ) @@ -586,8 +573,8 @@ def validateRace(): """Validate race.""" return make_validator( lambda value: value >= 0 and value <= 2, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} {value} is not greater than or equal to 0 " + lambda eargs: + f"{format_error_context(eargs)} {eargs.value} is not greater than or equal to 0 " "or smaller than or equal to 2." ) @@ -598,8 +585,8 @@ def validateRptMonthYear(): lambda value: value[2:8].isdigit() and int(value[2:6]) > 1900 and value[6:8] in {"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"}, - lambda value, row_schema, friendly_name, item_num, error_context_format: - f"{format_error_context(row_schema, friendly_name, item_num, error_context_format)} The value: {value[2:8]}, " + lambda eargs: + f"{format_error_context(eargs)} The value: {eargs.value[2:8]}, " "does not follow the YYYYMM format for Reporting Year and Month.", ) From a67c232e4fbd1bbe7509aac0e2fe262a02b63ae4 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 14 Jun 2024 11:08:11 -0400 Subject: [PATCH 33/52] fix test --- tdrs-backend/tdpservice/data_files/test/test_api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index d08ce3f1d..500fabc94 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -97,11 +97,13 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): ws = DataFileAPITestBase.get_spreadsheet(response) COL_ERROR_MESSAGE = 4 + COL_ITEM_NAME = 6 assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1 Item -1 (number of months): 0 is not larger than 0." + + " then number of months Item 21B (number of months) 0 is not larger than 0." + assert ws.cell(row=8, column=COL_ITEM_NAME).value == 'number of months,cash amount' @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -129,13 +131,15 @@ def assert_error_report_file_content_matches_without_friendly_names(response): ws = wb.get_sheet_by_name('Sheet1') COL_ERROR_MESSAGE = 4 + COL_ITEM_NAME = 6 assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( "if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not larger than 0." + "NBR_MONTHS Item 21B (number of months) 0 is not larger than 0." ) + assert ws.cell(row=8, column=COL_ITEM_NAME).value == '[CASH_AMOUNT(55-59), NBR_MONTHS(59-62)]' @staticmethod def assert_data_file_exists(data_file_data, version, user): From 887c17005664b807b38700021d620bcd599351ec Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 14 Jun 2024 11:27:28 -0400 Subject: [PATCH 34/52] lint --- .../tdpservice/parsers/test/test_parse.py | 6 ++- tdrs-backend/tdpservice/parsers/validators.py | 44 +++++++++++++------ 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index b9a5eb895..7dac97e8b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2014,7 +2014,8 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"Item 37 (education level) 00 is not in range [1, 16]. or Item 37 (education level) 00 is not in range [98, 99].", + error_msgs = {"Item 37 (education level) 00 is not in range [1, 16]. or Item 37 " + "(education level) 00 is not in range [98, 99].", "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} for e in parser_errors: @@ -2035,7 +2036,8 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"Item 68 (education level) 00 is not in range [1, 16]. or Item 68 (education level) 00 is not in range [98, 99].", + error_msgs = {"Item 68 (education level) 00 is not in range [1, 16]. or Item 68 " + "(education level) 00 is not in range [98, 99].", "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index f330670a1..5d05e2ea1 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -28,6 +28,8 @@ def value_is_empty(value, length, extra_vals={}): @dataclass class ValidationErrorArgs: + """Dataclass for args to `make_validator` `error_func`s.""" + value: Any row_schema: object # RowSchema causes circular import friendly_name: str @@ -50,9 +52,6 @@ def format_error_context(eargs: ValidationErrorArgs): def make_validator(validator_func, error_func): """Return a function accepting a value input and returning (bool, string) to represent validation state.""" - - # struct - # or kwargs def validator(value, row_schema=None, friendly_name=None, item_num=None, error_context_format='prefix'): eargs = ValidationErrorArgs( value=value, @@ -75,9 +74,14 @@ def validator(value, row_schema=None, friendly_name=None, item_num=None, error_c def or_validators(*args, **kwargs): """Return a validator that is true only if one of the validators is true.""" return ( - lambda value, row_schema, friendly_name, item_num, error_context_format='inline': (True, None) - if any([validator(value, row_schema, friendly_name, item_num, error_context_format)[0] for validator in args]) - else (False, " or ".join([validator(value, row_schema, friendly_name, item_num, error_context_format)[1] for validator in args])) + lambda value, row_schema, friendly_name, + item_num, error_context_format='inline': (True, None) + if any([ + validator(value, row_schema, friendly_name, item_num, error_context_format)[0] for validator in args + ]) + else (False, " or ".join([ + validator(value, row_schema, friendly_name, item_num, error_context_format)[1] for validator in args + ])) ) @@ -85,8 +89,8 @@ def and_validators(validator1, validator2): """Return a validator that is true only if both validators are true.""" return ( lambda value, row_schema, friendly_name, item_num: (True, None) - if (validator1(value, row_schema, friendly_name, item_num, 'inline')[0] and validator2(value, row_schema, - friendly_name, item_num, 'inline')[0]) + if (validator1(value, row_schema, friendly_name, item_num, 'inline')[0] + and validator2(value, row_schema, friendly_name, item_num, 'inline')[0]) else ( False, (validator1(value, row_schema, friendly_name, item_num, 'inline')[1]) @@ -120,9 +124,8 @@ def returned_func(value, row_schema, friendly_name, item_num): else: return (False, "".join( [ - " and " + validator(value, row_schema, - friendly_name, item_num, 'inline')[1] if validator(value, row_schema, - friendly_name, item_num, 'inline')[0] else "" + " and " + validator(value, row_schema, friendly_name, item_num, 'inline')[1] + if validator(value, row_schema, friendly_name, item_num, 'inline')[0] else "" for validator in args ] )) @@ -154,8 +157,20 @@ def if_then_validator_func(value, row_schema): result_field = row_schema.get_field_by_name(result_field_name) # TODO: There is some work to be done here to get the actual friendly name and item numbers of the fields - validator1_result = condition_function(value1, row_schema, condition_field.friendly_name, condition_field.item, 'inline') - validator2_result = result_function(value2, row_schema, result_field.friendly_name, result_field.item, 'inline') + validator1_result = condition_function( + value1, + row_schema, + condition_field.friendly_name, + condition_field.item, + 'inline' + ) + validator2_result = result_function( + value2, + row_schema, + result_field.friendly_name, + result_field.item, + 'inline' + ) if not validator1_result[0]: returned_value = (True, None, [condition_field_name, result_field_name]) @@ -217,7 +232,8 @@ def sumIsEqualFunc(value, row_schema): def field_year_month_with_header_year_quarter(): """Validate that the field year and month match the header year and quarter.""" - def validate_reporting_month_year_fields_header(line, row_schema, friendly_name, item_num, error_context_format=None): + def validate_reporting_month_year_fields_header( + line, row_schema, friendly_name, item_num, error_context_format=None): field_month_year = row_schema.get_field_values_by_names(line, ['RPT_MONTH_YEAR']).get('RPT_MONTH_YEAR') df_quarter = row_schema.datafile.quarter From 3b0a1b862cd7a408354143eee3e70aa080b64c2e Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 24 Jun 2024 10:06:59 -0400 Subject: [PATCH 35/52] rm comment --- tdrs-backend/tdpservice/parsers/validators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 5d05e2ea1..5e617c8c7 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -156,7 +156,6 @@ def if_then_validator_func(value, row_schema): condition_field = row_schema.get_field_by_name(condition_field_name) result_field = row_schema.get_field_by_name(result_field_name) - # TODO: There is some work to be done here to get the actual friendly name and item numbers of the fields validator1_result = condition_function( value1, row_schema, From 640fe342eeee997bf2430caa9db95287f7b82721 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 27 Jun 2024 09:14:23 -0400 Subject: [PATCH 36/52] fix tests --- tdrs-backend/tdpservice/data_files/test/test_api.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index d08ce3f1d..5f177721d 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -100,8 +100,8 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ - + " then number of months T1 Item -1 (number of months): 0 is not larger than 0." + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "Every T1 record should have at least one " + \ + "corresponding T2 or T3 record with the same RPT_MONTH_YEAR and CASE_NUMBER." @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -132,10 +132,9 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( - "if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not larger than 0." - ) + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("Every T1 record should have at least one " + "corresponding T2 or T3 record with the same " + "RPT_MONTH_YEAR and CASE_NUMBER.") @staticmethod def assert_data_file_exists(data_file_data, version, user): From 89c0f2da05baad42c382442485be6efcf41126d3 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 27 Jun 2024 09:15:57 -0400 Subject: [PATCH 37/52] fix lint --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 29b3b9176..c354cf11f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1646,7 +1646,8 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. or M2 Item 37 (education level): 00 is not in range [98, 99].", + error_msgs = {"M2 Item 37 (education level): 00 is not in range [1, 16]. " + "or M2 Item 37 (education level): 00 is not in range [98, 99].", "M2 Item 38 (citizenship status): 0 is not in [1, 2, 3, 9].", "M2 Item 39 (cooperation with child support): 0 is not in [1, 2, 9]."} for e in parser_errors: @@ -1669,7 +1670,8 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. or M3 Item 68 (education level): 00 is not in range [98, 99].", + error_msgs = {"M3 Item 68 (education level): 00 is not in range [1, 16]. " + "or M3 Item 68 (education level): 00 is not in range [98, 99].", "M3 Item 69 (citizenship status): 0 is not in [1, 2, 3, 9]."} for e in parser_errors: From fb0e539298344d06526afc0c015b81f2c3a18cf7 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Thu, 27 Jun 2024 09:30:15 -0400 Subject: [PATCH 38/52] lint --- tdrs-backend/tdpservice/data_files/test/test_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 1910fa581..5f177721d 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -97,7 +97,6 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): ws = DataFileAPITestBase.get_spreadsheet(response) COL_ERROR_MESSAGE = 4 - COL_ITEM_NAME = 6 assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" @@ -130,7 +129,6 @@ def assert_error_report_file_content_matches_without_friendly_names(response): ws = wb.get_sheet_by_name('Sheet1') COL_ERROR_MESSAGE = 4 - COL_ITEM_NAME = 6 assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" From 9b76e219477a34a857049a255c964299c785d161 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 8 Jul 2024 10:54:21 -0400 Subject: [PATCH 39/52] fix confl --- tdrs-backend/tdpservice/parsers/test/test_validators.py | 4 ---- tdrs-backend/tdpservice/parsers/validators.py | 5 ----- 2 files changed, 9 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 2b2c575f0..c39ff545b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -279,7 +279,6 @@ def test_oneOf_returns_invalid(): is_valid, error = validator(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False - assert error == 'T1 Item item_no (friendly_name): 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ assert error == 'T1 Item item_no (friendly_name): 65 is not in [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ' \ '29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55].' @@ -648,13 +647,10 @@ def test_calendarQuarterIsValid_returns_invalid(value): is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no", None) assert is_valid is False - assert error_msg == ( - f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " assert error_msg == ( f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " "representing the Calendar Year and Quarter formatted as YYYYQ" ) - ) @pytest.mark.parametrize("value", ["T720201", "T720202", "T720203", "T720204"]) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index e56b7f550..029317be4 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -313,11 +313,6 @@ def calendarQuarterIsValid(start=0, end=None): # generic validators -def format_error_context(row_schema, friendly_name, item_num): - """Format the error message for consistency across cat2 validators.""" - return f'{row_schema.record_type} Item {item_num} ({friendly_name})' - - def matches(option, error_func=None): """Validate that value is equal to option.""" return make_validator( From 1c15c36ee1aca41037568e2675b385593d13cf18 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 8 Jul 2024 10:55:57 -0400 Subject: [PATCH 40/52] lint --- tdrs-backend/tdpservice/parsers/validators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 029317be4..5e617c8c7 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -371,7 +371,6 @@ def between(min, max): ) - def fieldHasLength(length): """Validate that the field value (string or array) has a length matching length param.""" return make_validator( From ed8f67e8acb7dff5c1bd9f6d45013f6a7571a960 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 09:43:31 -0400 Subject: [PATCH 41/52] dynamic url to email template --- .../email/templates/upcoming-submission-deadline.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html index 72ac24b57..55519c07a 100644 --- a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html +++ b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html @@ -4,13 +4,13 @@

Hello {{ first_name }},

This is a friendly reminder that your data files are due in 5 days.

-

Please sign in to the TANF Data Portal +

Please sign in to the TANF Data Portal to upload and submit data files for Fiscal Year {{ fiscal_year }} - {{ fiscal_quarter}} by {{ submission_deadline }}.

-

Submit your data files

+

Submit your data files

Need help?
We're here for you! Check out the TDP Knowledge Center for specific gudiance on Submitting Data Files and Frequently Asked Questions. From 57ca5f621fe888de06d80743188279f71c4d2006 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 09:46:05 -0400 Subject: [PATCH 42/52] no-reply language change --- .../email/templates/upcoming-submission-deadline.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html index 55519c07a..59f0c5235 100644 --- a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html +++ b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html @@ -15,7 +15,7 @@ Need help?
We're here for you! Check out the TDP Knowledge Center for specific gudiance on Submitting Data Files and Frequently Asked Questions.

-TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Reach out to the TDP support team by replying to this email and we will follow up with you directly.

+TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Reach out to the TDP support team if you have any questions or concerns, and we will follow up with you directly.

Thank you,

The TDP Team

From 3c36b8172928e44864ec51c87c5001bd1ca0a8aa Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:47:32 -0400 Subject: [PATCH 43/52] 3025 As an STT user, I need an accurate error report when I space-fill `COUNTY_FIPS_CODE` (#3045) * 3025 Corrected the None case and added tests * added import for from DataFileSummaryFactory * linting * correct the failing test * fixed linting * corrected validator message --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> --- .../tdpservice/parsers/test/test_parse.py | 79 ++++++++++++++++++- tdrs-backend/tdpservice/parsers/transforms.py | 2 + 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 0c3cf0113..f762e6f91 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -12,7 +12,9 @@ from tdpservice.search_indexes.models.tribal import Tribal_TANF_T5, Tribal_TANF_T6, Tribal_TANF_T7 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3, SSP_M4, SSP_M5, SSP_M6, SSP_M7 from tdpservice.search_indexes import documents +from tdpservice.parsers.test.factories import DataFileSummaryFactory, ParsingFileFactory from tdpservice.data_files.models import DataFile +from tdpservice.parsers import util from .. import schema_defs, aggregates from elasticsearch.helpers.errors import BulkIndexError import logging @@ -24,7 +26,41 @@ settings.GENERATE_TRAILER_ERRORS = True -# TODO: the name of this test doesn't make perfect sense anymore since it will always have errors and no records now. +@pytest.fixture +def test_datafile(stt_user, stt): + """Fixture for small_correct_file.""" + return util.create_test_datafile('small_correct_file.txt', stt_user, stt) + + +@pytest.fixture +def test_header_datafile(stt_user, stt): + """Fixture for header test.""" + return util.create_test_datafile('tanf_section1_header_test.txt', stt_user, stt) + + +@pytest.fixture +def dfs(): + """Fixture for DataFileSummary.""" + return DataFileSummaryFactory.build() + + +@pytest.fixture +def t2_invalid_dob_file(): + """Fixture for T2 file with an invalid DOB.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t2_invalid_dob_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20204A25 TAN1ED\n' + b'T22020101111111111212Q897$9 3WTTTTTY@W222122222222101221211001472201140000000000000000000000000' + b'0000000000000000000000000000000000000000000000000000000000291\n' + b'TRAILER0000001 ') + ) + return parsing_file + +# TODO: the name of this test doesn't make perfect sense anymore since it will always have errors now. +# TODO: parametrize and merge with test_zero_filled_fips_code_file @pytest.mark.django_db def test_parse_small_correct_file(small_correct_file, dfs): """Test parsing of small_correct_file.""" @@ -1695,6 +1731,47 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): for e in parser_errors: assert e.error_message in error_msgs + +@pytest.fixture +def test_file_zero_filled_fips_code(): + """Fixture for T1 file with an invalid CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t3_invalid_citizenship_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20241A01000TAN2ED\n' + b'T1202401 2132333 0140951112 43312 03 0 0 2 554145' + + b' 0 0 0 0 0 0 0 0 0 0222222 0 02229 22 \n' + + b'T2202401 21323333219550117WT@TB9BT92122222222223 1329911 34' + + b' 32 699 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0' + + b' 0 0 0 0 0 01623 0 0 0\n' + + b'T2202401 21323333219561102WTT@WBP992122221222222 2329911 28' + + b' 32 699 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0' + + b' 0 0 0 0 0 01432 0 0 0\n' + + b'T3202401 2132333120070906WT@@#ZY@W212222122 63981 0 012' + + b'0050201WTTYT#TT0212222122 63981 0 0 \n' + + b'TRAILER 4 ') + ) + return parsing_file + + +@pytest.mark.django_db() +def test_zero_filled_fips_code_file(test_file_zero_filled_fips_code, dfs): + """Test parsing a file with zero filled FIPS code.""" + # TODO: this test can be merged as parametrized test with "test_parse_small_correct_file" + dfs.datafile = test_file_zero_filled_fips_code + test_file_zero_filled_fips_code.year = 2024 + test_file_zero_filled_fips_code.quarter = 'Q2' + test_file_zero_filled_fips_code.save() + + parse.parse_datafile(test_file_zero_filled_fips_code, dfs) + + parser_errors = ParserError.objects.filter(file=test_file_zero_filled_fips_code) + assert 'T1 Item 2 (County FIPS Code): field is required but a value was not' + \ + ' provided.' in [i.error_message for i in parser_errors] + + @pytest.mark.parametrize("file, batch_size, model, record_type, num_errors", [ ('tanf_s1_exact_dup_file', 10000, TANF_T1, "T1", 3), ('tanf_s1_exact_dup_file', 1, TANF_T1, "T1", 3), # This forces an in memory and database deletion of records. diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index cd51e2012..6a3717817 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -40,5 +40,7 @@ def ssp_ssn_decryption_func(value, **kwargs): def zero_pad(digits): """Zero pad a string.""" def transform(value, **kwargs): + if value is None: + return None return value.lstrip().zfill(digits) return transform From 2618dd612f5651037669e125e591e270bda81fd9 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 09:48:50 -0400 Subject: [PATCH 44/52] turn down log sensitivity in dev --- tdrs-backend/tdpservice/email/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index aaf46c0e4..089ea4e8d 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -157,4 +157,7 @@ def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): 'object_id': loc.id, 'object_repr': loc.name, } - log(f"{loc.name} has no recipients for data submission deadline reminder.", logger_context=logger_context) + log( + f"{loc.name} has no recipients for data submission deadline reminder.", + logger_context=logger_context if not settings.DEBUG else None + ) From 05dbc4669a62e3aaabc161691df971120bddd935 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 12:14:31 -0400 Subject: [PATCH 45/52] email reply language --- .../email/templates/upcoming-submission-deadline.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html index 59f0c5235..191e099cd 100644 --- a/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html +++ b/tdrs-backend/tdpservice/email/templates/upcoming-submission-deadline.html @@ -15,7 +15,7 @@ Need help?
We're here for you! Check out the TDP Knowledge Center for specific gudiance on Submitting Data Files and Frequently Asked Questions.

-TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Reach out to the TDP support team if you have any questions or concerns, and we will follow up with you directly.

+TDP is now the only method for data submissions; you should not be submitting data through SFTP or Cyberfusion. Please reach out to the TDP support team at TANFData@acf.hhs.gov if you have any questions or need assistance.

Thank you,

The TDP Team

From d532be0e1934703a5c1faaf7a63ce9364c10473b Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 13:06:37 -0400 Subject: [PATCH 46/52] fix tests --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index f762e6f91..d82124894 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1674,10 +1674,12 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - error_msgs = {"M2 Item 37 (Educational Level): 00 is not in range [1, 16]. or M2 Item 37 (Educational Level): " + - "00 is not in range [98, 99].", - "M2 Item 38 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9].", - "M2 Item 39 (Cooperated with Child Support): 0 is not in [1, 2, 9]."} + error_msgs = { + "Item 37 (Educational Level) 00 is not in range [1, 16]. or " + "Item 37 (Educational Level) 00 is not in range [98, 99].", + "M2 Item 38 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9].", + "M2 Item 39 (Cooperated with Child Support): 0 is not in [1, 2, 9]." + } for e in parser_errors: assert e.error_message in error_msgs @@ -1698,10 +1700,10 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 4 - error_msgs = {"M3 Item 68 (Educational Level): 00 is not in range [1, 16]. or M3 Item 68 (Educational Level): " + + error_msgs = {"Item 68 (Educational Level) 00 is not in range [1, 16]. or Item 68 (Educational Level) " + "00 is not in range [98, 99].", "M3 Item 69 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9].", - "M3 Item 68 (Educational Level): 00 is not in range [1, 16]. or M3 Item 68 (Educational Level): " + + "Item 68 (Educational Level) 00 is not in range [1, 16]. or Item 68 (Educational Level) " + "00 is not in range [98, 99].", "M3 Item 69 (Citizenship/Immigration Status): 0 is not in [1, 2, 3, 9]."} From d519b033f71dbbe114e2682e8ab2ce47d2301217 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 14:09:27 -0400 Subject: [PATCH 47/52] fix tests --- .../tdpservice/data_files/test/test_api.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 9ae1a408e..cea6547f4 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -100,10 +100,11 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("if Cash Amount :873 validator1 passed then Cash and " - "Cash Equivalents: Number of Months T1 Item -1 (Cash " - "and Cash Equivalents: Number of Months): 0 is not " - "larger than 0.") + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( + "if Cash Amount :873 validator1 passed then Cash " + "and Cash Equivalents: Number of Months Item 21B " + "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." + ) @staticmethod def assert_error_report_ssp_file_content_matches_with_friendly_names(response): @@ -134,9 +135,10 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" - assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then " - "NBR_MONTHS T1 Item -1 (NBR_MONTHS): 0 is not " - "larger than 0.") + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( + "if CASH_AMOUNT :873 validator1 passed then NBR_MONTHS Item 21B " + "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." + ) @staticmethod def assert_data_file_exists(data_file_data, version, user): From 5431881edcfb875ff9fbd655dd00b32cd27338ad Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Tue, 16 Jul 2024 14:19:13 -0400 Subject: [PATCH 48/52] test --- tdrs-backend/tdpservice/data_files/test/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index cea6547f4..58ed6363c 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -136,8 +136,8 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( - "if CASH_AMOUNT :873 validator1 passed then NBR_MONTHS Item 21B " - "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." + "Every T1 record should have at least one corresponding T2 or T3 " + "record with the same RPT_MONTH_YEAR and CASE_NUMBER." ) @staticmethod From 66542e84faf4b8e4d7b1bc495b628e09d89afd7b Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:01:34 -0400 Subject: [PATCH 49/52] 2950 updated search index (#2964) * 2950 updated model names * 2950 added changes for creating the filter * revert unused changes back * linting * removed unused file * 2950 removed unnecessary change * Fix linting error * 2950 added Territory to TANF * 2950 added Territory to TANF corrected linting --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> --- .../search_indexes/admin/filters.py | 16 ++- ...029_tanf_tribal_ssp_alter_verbose_names.py | 97 +++++++++++++++++++ .../tdpservice/search_indexes/models/ssp.py | 35 +++++++ .../tdpservice/search_indexes/models/tanf.py | 35 +++++++ .../search_indexes/models/tribal.py | 35 +++++++ 5 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0029_tanf_tribal_ssp_alter_verbose_names.py diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py index 517d27e48..36e1e66da 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py @@ -61,9 +61,23 @@ def _get_lookup_choices(self, request): else: type_query = Query(type=STT.EntityType.STATE) | Query(type=STT.EntityType.TERRITORY) queryset = queryset.filter(type_query) - return (queryset.distinct().order_by('name').values_list('name', flat=True)) + def lookups(self, request, model_admin): + """Available options in dropdown.""" + listing_record_type = [i for i in str(request.path).lower().split('/') if i not in ['/', '']][-1] + options = [] + if 'tribal' in listing_record_type: + objects = STT.objects.filter(type=STT.EntityType.TRIBE) + elif 'ssp' in listing_record_type: + objects = STT.objects.filter(ssp=True) + else: + objects = STT.objects.filter(Query(type=STT.EntityType.STATE) | Query(type=STT.EntityType.TERRITORY)) + for obj in objects: + options.append((obj.stt_code, _(obj.name))) + + return options + class FiscalPeriodFilter(SimpleListFilter): """Simple filter class to filter records based on datafile fiscal year.""" diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0029_tanf_tribal_ssp_alter_verbose_names.py b/tdrs-backend/tdpservice/search_indexes/migrations/0029_tanf_tribal_ssp_alter_verbose_names.py new file mode 100644 index 000000000..768c91d1c --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0029_tanf_tribal_ssp_alter_verbose_names.py @@ -0,0 +1,97 @@ +# Generated by Django 3.2.15 on 2024-05-15 16:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0028_education_level_to_string'), + ] + + operations = [ + migrations.AlterModelOptions( + name='ssp_m1', + options={'verbose_name': 'SSP M1'}, + ), + migrations.AlterModelOptions( + name='ssp_m2', + options={'verbose_name': 'SSP M2'}, + ), + migrations.AlterModelOptions( + name='ssp_m3', + options={'verbose_name': 'SSP M3'}, + ), + migrations.AlterModelOptions( + name='ssp_m4', + options={'verbose_name': 'SSP M4'}, + ), + migrations.AlterModelOptions( + name='ssp_m5', + options={'verbose_name': 'SSP M5'}, + ), + migrations.AlterModelOptions( + name='ssp_m6', + options={'verbose_name': 'SSP M6'}, + ), + migrations.AlterModelOptions( + name='ssp_m7', + options={'verbose_name': 'SSP M7'}, + ), + migrations.AlterModelOptions( + name='tanf_t1', + options={'verbose_name': 'TANF T1'}, + ), + migrations.AlterModelOptions( + name='tanf_t2', + options={'verbose_name': 'TANF T2'}, + ), + migrations.AlterModelOptions( + name='tanf_t3', + options={'verbose_name': 'TANF T3'}, + ), + migrations.AlterModelOptions( + name='tanf_t4', + options={'verbose_name': 'TANF T4'}, + ), + migrations.AlterModelOptions( + name='tanf_t5', + options={'verbose_name': 'TANF T5'}, + ), + migrations.AlterModelOptions( + name='tanf_t6', + options={'verbose_name': 'TANF T6'}, + ), + migrations.AlterModelOptions( + name='tanf_t7', + options={'verbose_name': 'TANF T7'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t1', + options={'verbose_name': 'Tribal TANF T1'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t2', + options={'verbose_name': 'Tribal TANF T2'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t3', + options={'verbose_name': 'Tribal TANF T3'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t4', + options={'verbose_name': 'Tribal TANF T4'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t5', + options={'verbose_name': 'Tribal TANF T5'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t6', + options={'verbose_name': 'Tribal TANF T6'}, + ), + migrations.AlterModelOptions( + name='tribal_tanf_t7', + options={'verbose_name': 'Tribal TANF T7'}, + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index bb5840323..78c035db2 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -12,6 +12,11 @@ class SSP_M1(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M1' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -78,6 +83,11 @@ class SSP_M2(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M2' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -165,6 +175,11 @@ class SSP_M3(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M3' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -206,6 +221,11 @@ class SSP_M4(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M4' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -240,6 +260,11 @@ class SSP_M5(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M5' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -286,6 +311,11 @@ class SSP_M6(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M6' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -318,6 +348,11 @@ class SSP_M7(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'SSP M7' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index bac3f008d..c57ab89ed 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -12,6 +12,11 @@ class TANF_T1(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T1' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -81,6 +86,11 @@ class TANF_T2(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T2' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -170,6 +180,11 @@ class TANF_T3(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T3' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -211,6 +226,11 @@ class TANF_T4(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T4' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -243,6 +263,11 @@ class TANF_T5(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T5' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -292,6 +317,11 @@ class TANF_T6(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T6' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -329,6 +359,11 @@ class TANF_T7(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'TANF T7' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, diff --git a/tdrs-backend/tdpservice/search_indexes/models/tribal.py b/tdrs-backend/tdpservice/search_indexes/models/tribal.py index fd5ee805d..d42818336 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tribal.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tribal.py @@ -14,6 +14,11 @@ class Tribal_TANF_T1(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T1' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -83,6 +88,11 @@ class Tribal_TANF_T2(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T2' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -155,6 +165,11 @@ class Tribal_TANF_T3(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T3' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -195,6 +210,11 @@ class Tribal_TANF_T4(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T4' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -226,6 +246,11 @@ class Tribal_TANF_T5(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T5' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -274,6 +299,11 @@ class Tribal_TANF_T6(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T6' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, @@ -310,6 +340,11 @@ class Tribal_TANF_T7(models.Model): Mapped to an elastic search index. """ + class Meta: + """Meta class for the model.""" + + verbose_name = 'Tribal TANF T7' + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) datafile = models.ForeignKey( DataFile, From ff50f9e2bbde8216a7519800a66f301f606cb8d9 Mon Sep 17 00:00:00 2001 From: robgendron <163159602+robgendron@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:28:36 -0400 Subject: [PATCH 50/52] Create sprint-101-summary.md (#3052) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-101-summary.md | 87 ++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 docs/Sprint-Review/sprint-101-summary.md diff --git a/docs/Sprint-Review/sprint-101-summary.md b/docs/Sprint-Review/sprint-101-summary.md new file mode 100644 index 000000000..1071e6298 --- /dev/null +++ b/docs/Sprint-Review/sprint-101-summary.md @@ -0,0 +1,87 @@ +# sprint-101-summary + +6/5/2024 - 6/18/2024 + +**Dev:** + +_**Prioritized DAC and Notifications Work**_ + +* As sys admin, I want to be able to reparse datafile sets #2978 +* As a software engineer, I want to be able to test django-admin-508 #3008 +* As tech lead, I need the STT filter for search\_indexes to be updated #2950 +* As a data analyst I want to be notified of approaching data deadlines #2473 +* add `SENDGRID_API_KEY` to deploy.backend.sh #2677 +* Implement (small) data lifecycle (backup/archive ES) #3004 +* As a developer I want to test django-508 repo #2980\ + + +**DevOps:** + +_**Successful deployments across environments and pipeline stability investments**_ + +* Application health monitoring #831 + +**Design:** + +_**Close out error guide work, coordinate with dev on a plan for Cat 3 problems introduced by Cat 2 work, support spec-writing for upcoming work, and continued error audit dev ticket refinement.**_ + +* Error Report Guide #2847 is going through final edits + * Walk-on Dear Colleague letter link update to this PR (or spin up a separate ticket if deployment of the letter to OFA's website doesn't align to this) +* Deliver spec for #3014 (Blanked-out values in Submission History) +* \#3021 Updated KC Release Notes & Update Indicator FAQ - stretch goal for this sprint +* Write follow-on / spec tickets from #2909 findings - stretch/ongoing lift +* Category 3 error messages clean-up #2792 - stretch/ongoing lift + +## Tickets + +### Completed/Merged + +* [#2980 As a developer I want to test django-508 repo](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2980) +* [#2892 Correct misleading error message for unaligned reporting year/q against header year/q](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2892) +* [#2909 \[Research Spike\] OOtB OFA Kibana Experience & DIGIT Data Access](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2909) +* [#2991 As tech lead, I need the sftp file transfer feature to be deprecated](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2991) +* [#2847 \[Design Deliverable\] Error Report Knowledge Center Explainer](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2847) +* [#3024 2897 follow-on for a11y-related enhancement ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3024) +* [#2897 As a data analyst I want finalized language and guidance resources in Submission History & Error Reports ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2897) + +### Submitted (QASP Review, OCIO Review) + +* [#2133 \[Dev\] Enhancement for Request Access form (Tribe discoverability) ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2133) +* [#3023 as STT approved user, I need my IP address whitelisted so i can access TDP](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3023) +* [#3000 \[Design Deliverable\] TDP Poster for summer 2024 conferences](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3000) +* [#2795 As tech lead, I need TDP to detect duplicate records within a file and not store them in the db. ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2795) +* [#2693 \[Error Audit\] Category 2 error messages clean-up ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2693) +* [#2801 Friendly name cleanup ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2801) +* [#2883 Pre-Made Reporting Dashboards on Kibana](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2883) +* [#3021 \[Design Deliverable\] Updated KC Release Notes & Update Indicator FAQ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3021) +* [#2954 Extend SESSION\_COOKIE\_AGE](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2954) + +### Ready to Merge + +* + +### Closed (Not Merged) + +* [#2491 Create root-level docker-compose configuration file(s)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2491) +* [#1690 As a system admin, I need a way to be redirected to frontend from DAC](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1690) +* [#2351 As a user I want to be notified when the files are being scanned or uploaded when I push upload button](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2351) +* [#2591 Allow `manage.py` commands to be run by circleci](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2591) + +### Moved to Next Sprint + +**In Progress** + +* [#3004 Implement (small) data lifecycle (backup/archive ES)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3004) +* [#831 \[Spike\] As a Tech Lead, I want to get alerts when there is a backend or frontend error that affects an STT user ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/831) +* [#2978 As sys admin, I want to be able to reparse datafile sets](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2978) + +#### Blocked + +* + +**Raft Review** + +* [#2950 As tech lead, I need the STT filter for search\_indexes to be updated ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2950) +* [#3008 As a software engineer, I want to be able to test django-admin-508](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3008) +* [#3016 Spike - Cat2 Validator Improvement](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/3016) +* [#2473 As a data analyst I want to be notified of approaching data deadlines](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2473) From e29f07cb5eb335f6419afeab0c113864deed4451 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 17 Jul 2024 09:01:27 -0400 Subject: [PATCH 51/52] remove extraneous item names --- tdrs-backend/tdpservice/data_files/test/test_api.py | 7 +++---- tdrs-backend/tdpservice/parsers/test/test_validators.py | 2 +- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 58ed6363c..55dba626e 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -101,8 +101,7 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( - "if Cash Amount :873 validator1 passed then Cash " - "and Cash Equivalents: Number of Months Item 21B " + "if Cash Amount :873 validator1 passed then Item 21B " "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." ) @@ -136,8 +135,8 @@ def assert_error_report_file_content_matches_without_friendly_names(response): assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + "instructions (linked below) when looking up items and allowable values during the data revision process" assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ( - "Every T1 record should have at least one corresponding T2 or T3 " - "record with the same RPT_MONTH_YEAR and CASE_NUMBER." + "if CASH_AMOUNT :873 validator1 passed then Item 21B " + "(Cash and Cash Equivalents: Number of Months) 0 is not larger than 0." ) @staticmethod diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index c39ff545b..d729efc6e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -108,7 +108,7 @@ def test_if_validators(): name='Field2', friendly_name='field 2'), ] )) - assert result == (False, 'if Field1 :1 validator1 passed then Field2 Item 2 (field 2) 2 does not match 1.', + assert result == (False, 'if Field1 :1 validator1 passed then Item 2 (field 2) 2 does not match 1.', ['Field1', 'Field2']) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 5e617c8c7..53726ee4c 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -189,7 +189,7 @@ def if_then_validator_func(value, row_schema): ending_error = "validator2 passed" error_message = (f"if {condition_field_name} " + (center_error) + - f" then {result_field_name} " + ending_error) + f" then " + ending_error) else: error_message = None From e9c02c8ff7fe174da5b3e65c6aaf0848b413ce9c Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Wed, 17 Jul 2024 09:22:30 -0400 Subject: [PATCH 52/52] lint --- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 53726ee4c..e8320055a 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -189,7 +189,7 @@ def if_then_validator_func(value, row_schema): ending_error = "validator2 passed" error_message = (f"if {condition_field_name} " + (center_error) + - f" then " + ending_error) + " then " + ending_error) else: error_message = None