From 06c291622af5cf3937ca0f39650729f0518af19e Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Fri, 15 Mar 2024 09:50:24 -0400 Subject: [PATCH 01/20] date update + catching invalid quarter error --- tdrs-backend/tdpservice/parsers/fields.py | 3 +- tdrs-backend/tdpservice/parsers/parse.py | 52 ++++++++++++------- .../tdpservice/parsers/schema_defs/ssp/m6.py | 6 +-- .../tdpservice/parsers/schema_defs/ssp/m7.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t6.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t7.py | 2 +- .../parsers/schema_defs/tribal_tanf/t6.py | 2 +- .../parsers/schema_defs/tribal_tanf/t7.py | 2 +- .../tribal_section_4_fake_bad_quarter.txt | 3 ++ .../tdpservice/parsers/test/test_parse.py | 23 ++++++++ .../parsers/test/test_validators.py | 22 ++++++++ 11 files changed, 92 insertions(+), 27 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index 076743096..b283f13d3 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -80,4 +80,5 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex, def parse_value(self, line): """Parse and transform the value for a field given a line, startIndex, endIndex, and field type.""" value = super().parse_value(line) - return self.transform_func(value, **self.kwargs) + # wrap in try catch return none to avoid db write + return self.transform_func(value, **self.kwargs) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 9e7af9ce9..a448ba0d4 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -236,24 +236,40 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): schema_manager = get_schema_manager(line, section, program_type) - records = manager_parse_line(line, schema_manager, generate_error, is_encrypted) - - record_number = 0 - for i in range(len(records)): - r = records[i] - record_number += 1 - record, record_is_valid, record_errors = r - if not record_is_valid: - logger.debug(f"Record #{i} from line {line_number} is invalid.") - line_errors = errors.get(f"{line_number}_{i}", {}) - line_errors.update({record_number: record_errors}) - errors.update({f"{line_number}_{i}": record_errors}) - unsaved_parser_errors.update({f"{line_number}_{i}": record_errors}) - num_errors += len(record_errors) - if record: - s = schema_manager.schemas[i] - record.datafile = datafile - unsaved_records.setdefault(s.document, []).append(record) + try: + records = manager_parse_line(line, schema_manager, generate_error, is_encrypted) + + # this bubbles up when uyploading a file on port 3000 in the history (XLS) + # get the errors in this structure + record_number = 0 + for i in range(len(records)): + r = records[i] + record_number += 1 + record, record_is_valid, record_errors = r + if not record_is_valid: + logger.debug(f"Record #{i} from line {line_number} is invalid.") + line_errors = errors.get(f"{line_number}_{i}", {}) + line_errors.update({record_number: record_errors}) + errors.update({f"{line_number}_{i}": record_errors}) + unsaved_parser_errors.update({f"{line_number}_{i}": record_errors}) + num_errors += len(record_errors) + if record: + s = schema_manager.schemas[i] + record.datafile = datafile + unsaved_records.setdefault(s.document, []).append(record) + + except ValueError as ex: + err = generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message=ex, + record=None, + field="Record_Type" + ) + + errors.update({f"{line_number}_0": [err]}) + unsaved_parser_errors.update({f"{line_number}_0": [err]}) + num_errors += 1 all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count, datafile) unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 6f7f67d34..94d77673f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -47,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), @@ -208,7 +208,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), @@ -369,7 +369,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 24a680b24..806b4251a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -47,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index e800aafc4..f0e5f5621 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -54,7 +54,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index 80ec21f20..81e0f70e5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -47,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index 2d6d230b8..41980bb9b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -42,7 +42,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index 1212f300d..a17820514 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -47,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt new file mode 100644 index 000000000..c4b6851d7 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt @@ -0,0 +1,3 @@ +HEADER20194S00142TAN1EU +T72020 101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 24e49f5a7..ab1d083ae 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1467,3 +1467,26 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d assert first.FAMILIES_MONTH == 0 assert sixth.FAMILIES_MONTH == 446 + +@pytest.fixture +def tribal_section_4_bad_quarter(stt_user, stt): + """Fixture for tribal_section_4_bad_quarter.""" + return util.create_test_datafile('tribal_section_4_fake_bad_quarter.txt', stt_user, stt, "Tribal Stratum Data") + +@pytest.mark.django_db() +def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): + """Test parsing invalid quarter value""" + tribal_section_4_bad_quarter.year = 2020 + tribal_section_4_bad_quarter.quarter = 'Q1' + dfs.datafile = tribal_section_4_bad_quarter + + parse.parse_datafile(tribal_section_4_bad_quarter) + parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter) + + assert parser_errors.count() == 1 + error = parser_errors.first() + + print(error.error_message) + + assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert error.error_message == "Invalid quarter value." diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index bd3eb88ce..7f3488883 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -336,6 +336,28 @@ def test_notEmpty_returns_nonexistent_substring(): assert is_valid is False assert error == "111 333 contains blanks between positions 10 and 12." + +@pytest.mark.parametrize("test_input", [1,2,3,4]) +def test_quarterIsValid_returns_true_if_valid(test_input): + """Test `quarterIsValid` gives a valid result for values 1-4""" + + validator = validators.quarterIsValid() + is_valid, error = validator(test_input) + + assert is_valid is True + assert error is None + + +@pytest.mark.parametrize("test_input", [" ", 0, 5, "A"]) +def test_quarterIsValid_returns_false_if_invalid(test_input): + """Test `quarterIsValid` gives an invalid result for values not 1-4""" + + validator = validators.quarterIsValid() + is_valid, error = validator(test_input) + + assert is_valid is False + assert error == f"{test_input} is not a valid quarter." + @pytest.mark.usefixtures('db') class TestCat3ValidatorsBase: """A base test class for tests that evaluate category three validators.""" From d20225cb94e0e8dfe80427c1c9538c3fc3b0f8f4 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Fri, 15 Mar 2024 09:50:24 -0400 Subject: [PATCH 02/20] date update + catching invalid quarter error --- tdrs-backend/tdpservice/parsers/fields.py | 3 +- tdrs-backend/tdpservice/parsers/parse.py | 52 ++++++++++++------- .../tdpservice/parsers/schema_defs/ssp/m6.py | 6 +-- .../tdpservice/parsers/schema_defs/ssp/m7.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t6.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t7.py | 2 +- .../parsers/schema_defs/tribal_tanf/t6.py | 2 +- .../parsers/schema_defs/tribal_tanf/t7.py | 2 +- .../tribal_section_4_fake_bad_quarter.txt | 3 ++ .../tdpservice/parsers/test/test_parse.py | 23 ++++++++ .../parsers/test/test_validators.py | 22 ++++++++ 11 files changed, 92 insertions(+), 27 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index 076743096..b283f13d3 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -80,4 +80,5 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex, def parse_value(self, line): """Parse and transform the value for a field given a line, startIndex, endIndex, and field type.""" value = super().parse_value(line) - return self.transform_func(value, **self.kwargs) + # wrap in try catch return none to avoid db write + return self.transform_func(value, **self.kwargs) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 9e7af9ce9..a448ba0d4 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -236,24 +236,40 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): schema_manager = get_schema_manager(line, section, program_type) - records = manager_parse_line(line, schema_manager, generate_error, is_encrypted) - - record_number = 0 - for i in range(len(records)): - r = records[i] - record_number += 1 - record, record_is_valid, record_errors = r - if not record_is_valid: - logger.debug(f"Record #{i} from line {line_number} is invalid.") - line_errors = errors.get(f"{line_number}_{i}", {}) - line_errors.update({record_number: record_errors}) - errors.update({f"{line_number}_{i}": record_errors}) - unsaved_parser_errors.update({f"{line_number}_{i}": record_errors}) - num_errors += len(record_errors) - if record: - s = schema_manager.schemas[i] - record.datafile = datafile - unsaved_records.setdefault(s.document, []).append(record) + try: + records = manager_parse_line(line, schema_manager, generate_error, is_encrypted) + + # this bubbles up when uyploading a file on port 3000 in the history (XLS) + # get the errors in this structure + record_number = 0 + for i in range(len(records)): + r = records[i] + record_number += 1 + record, record_is_valid, record_errors = r + if not record_is_valid: + logger.debug(f"Record #{i} from line {line_number} is invalid.") + line_errors = errors.get(f"{line_number}_{i}", {}) + line_errors.update({record_number: record_errors}) + errors.update({f"{line_number}_{i}": record_errors}) + unsaved_parser_errors.update({f"{line_number}_{i}": record_errors}) + num_errors += len(record_errors) + if record: + s = schema_manager.schemas[i] + record.datafile = datafile + unsaved_records.setdefault(s.document, []).append(record) + + except ValueError as ex: + err = generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message=ex, + record=None, + field="Record_Type" + ) + + errors.update({f"{line_number}_0": [err]}) + unsaved_parser_errors.update({f"{line_number}_0": [err]}) + num_errors += 1 all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count, datafile) unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 6f7f67d34..94d77673f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -47,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), @@ -208,7 +208,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), @@ -369,7 +369,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 24a680b24..806b4251a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -47,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index e800aafc4..f0e5f5621 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -54,7 +54,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index 80ec21f20..81e0f70e5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -47,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index 2d6d230b8..41980bb9b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -42,7 +42,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index 1212f300d..a17820514 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -47,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(1998), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt new file mode 100644 index 000000000..c4b6851d7 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt @@ -0,0 +1,3 @@ +HEADER20194S00142TAN1EU +T72020 101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 2aaabcc19..4f34b5015 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1489,3 +1489,26 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d assert first.FAMILIES_MONTH == 0 assert sixth.FAMILIES_MONTH == 446 + +@pytest.fixture +def tribal_section_4_bad_quarter(stt_user, stt): + """Fixture for tribal_section_4_bad_quarter.""" + return util.create_test_datafile('tribal_section_4_fake_bad_quarter.txt', stt_user, stt, "Tribal Stratum Data") + +@pytest.mark.django_db() +def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): + """Test parsing invalid quarter value""" + tribal_section_4_bad_quarter.year = 2020 + tribal_section_4_bad_quarter.quarter = 'Q1' + dfs.datafile = tribal_section_4_bad_quarter + + parse.parse_datafile(tribal_section_4_bad_quarter) + parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter) + + assert parser_errors.count() == 1 + error = parser_errors.first() + + print(error.error_message) + + assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert error.error_message == "Invalid quarter value." diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 7d6a6011a..238f5edb7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -449,6 +449,28 @@ def test_notEmpty_returns_nonexistent_substring(): assert is_valid is False assert error == "111 333 contains blanks between positions 10 and 12." + +@pytest.mark.parametrize("test_input", [1,2,3,4]) +def test_quarterIsValid_returns_true_if_valid(test_input): + """Test `quarterIsValid` gives a valid result for values 1-4""" + + validator = validators.quarterIsValid() + is_valid, error = validator(test_input) + + assert is_valid is True + assert error is None + + +@pytest.mark.parametrize("test_input", [" ", 0, 5, "A"]) +def test_quarterIsValid_returns_false_if_invalid(test_input): + """Test `quarterIsValid` gives an invalid result for values not 1-4""" + + validator = validators.quarterIsValid() + is_valid, error = validator(test_input) + + assert is_valid is False + assert error == f"{test_input} is not a valid quarter." + @pytest.mark.usefixtures('db') class TestCat3ValidatorsBase: """A base test class for tests that evaluate category three validators.""" From 929e94f2ffdb1eec1f11890af0514d3f48463819 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 18 Mar 2024 10:21:05 -0400 Subject: [PATCH 03/20] fixing tests --- tdrs-backend/None | 3 +++ tdrs-backend/tdpservice/parsers/parse.py | 17 +++++++------- .../tdpservice/parsers/test/test_parse.py | 4 +--- .../parsers/test/test_validators.py | 22 +++++++++++++------ 4 files changed, 28 insertions(+), 18 deletions(-) create mode 100644 tdrs-backend/None diff --git a/tdrs-backend/None b/tdrs-backend/None new file mode 100644 index 000000000..8e88b1fb0 --- /dev/null +++ b/tdrs-backend/None @@ -0,0 +1,3 @@ +HEADER20204G06 TAN1 N +T620204000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884 +TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index a448ba0d4..1c71dc2c7 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -239,8 +239,6 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): try: records = manager_parse_line(line, schema_manager, generate_error, is_encrypted) - # this bubbles up when uyploading a file on port 3000 in the history (XLS) - # get the errors in this structure record_number = 0 for i in range(len(records)): r = records[i] @@ -258,18 +256,21 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): record.datafile = datafile unsaved_records.setdefault(s.document, []).append(record) - except ValueError as ex: - err = generate_error( + except ValueError as ex: + err_obj = generate_error( schema=None, error_category=ParserErrorCategoryChoices.PRE_CHECK, error_message=ex, record=None, - field="Record_Type" + field=None ) - - errors.update({f"{line_number}_0": [err]}) - unsaved_parser_errors.update({f"{line_number}_0": [err]}) num_errors += 1 + parse_error = {line_number: [err_obj]} + unsaved_parser_errors.update(parse_error) + rollback_records(unsaved_records, datafile) + rollback_parser_errors(datafile) + bulk_create_errors(parse_error, num_errors, flush=True) + return errors all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count, datafile) unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 4f34b5015..0aaa98efa 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1497,7 +1497,7 @@ def tribal_section_4_bad_quarter(stt_user, stt): @pytest.mark.django_db() def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): - """Test parsing invalid quarter value""" + """Test handling invalid quarter value that raises a ValueError exception""" tribal_section_4_bad_quarter.year = 2020 tribal_section_4_bad_quarter.quarter = 'Q1' dfs.datafile = tribal_section_4_bad_quarter @@ -1508,7 +1508,5 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): assert parser_errors.count() == 1 error = parser_errors.first() - print(error.error_message) - assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert error.error_message == "Invalid quarter value." diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 238f5edb7..27d071e6f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -101,17 +101,25 @@ def test_validate__FAM_AFF__SSN(): result = validators.validate__FAM_AFF__SSN()(instance) assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS', 'SSN']) - -def test_quarterIsValid(): +@pytest.mark.parametrize( + "value, valid", + [ + ("20201", True), + ("20202", True), + ("20203", True), + ("20204", True), + ("20200", False), + ("20205", False), + ("2020 ", False), + ("2020A", False) + ]) +def test_quarterIsValid(value, valid): """Test `quarterIsValid`.""" - value = "20204" val = validators.quarterIsValid() result = val(value) - assert result == (True, None) - value = "20205" - result = val(value) - assert result == (False, "5 is not a valid quarter.") + errorText = None if valid else f"{value[-1:]} is not a valid quarter." + assert result == (valid, errorText) def test_validateSSN(): """Test `validateSSN`.""" From 4880610b71fe64aaeb88fcd543142eef2b19769c Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 18 Mar 2024 11:10:14 -0400 Subject: [PATCH 04/20] fixing tests --- .../parsers/test/data/ADS.E2J.FTP3.TS142 | 4 ++-- .../tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 | 4 ++-- .../tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 | 4 ++-- tdrs-backend/tdpservice/parsers/test/test_parse.py | 14 +++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 index 6e5798b8a..7ebd8bbcc 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 @@ -1,3 +1,3 @@ -HEADER20194G00142TAN1EU -T620194 5 1 1 5 1 1 0 0 0 21097 22915 21441 40 41 40 5 6 4 8 9 11 27 26 25 85 89 85 18 21 19 67 68 66 0 0 0 0 0 0 0 0 0 3 3 0 +HEADER20204G00142TAN1EU +T620204 5 1 1 5 1 1 0 0 0 21097 22915 21441 40 41 40 5 6 4 8 9 11 27 26 25 85 89 85 18 21 19 67 68 66 0 0 0 0 0 0 0 0 0 3 3 0 TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 index bbe8ee4f0..6ea30625f 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 @@ -1,3 +1,3 @@ -HEADER20184G24 SSP1 N -M620184000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229 +HEADER20204G24 SSP1 N +M620204000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229 TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 index 1393729b8..fbf049ce9 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 @@ -1,3 +1,3 @@ -HEADER20184S24 SSP1 N -M720184101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229 +HEADER20204S24 SSP1 N +M720204101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229 TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 0aaa98efa..66352930d 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1059,7 +1059,7 @@ def ssp_section4_file(stt_user, stt): @pytest.mark.django_db() def test_parse_ssp_section4_file(ssp_section4_file, dfs): """Test parsing SSP Section 4 submission.""" - ssp_section4_file.year = 2019 + ssp_section4_file.year = 2021 ssp_section4_file.quarter = 'Q1' dfs.datafile = ssp_section4_file @@ -1079,7 +1079,7 @@ def test_parse_ssp_section4_file(ssp_section4_file, dfs): assert m7_objs.count() == 12 first = m7_objs.first() - assert first.RPT_MONTH_YEAR == 201810 + assert first.RPT_MONTH_YEAR == 202010 assert first.FAMILIES_MONTH == 748 @pytest.fixture @@ -1152,7 +1152,7 @@ def ssp_section3_file(stt_user, stt): @pytest.mark.django_db() def test_parse_ssp_section3_file(ssp_section3_file, dfs): """Test parsing TANF Section 3 submission.""" - ssp_section3_file.year = 2019 + ssp_section3_file.year = 2021 ssp_section3_file.quarter = 'Q1' dfs.datafile = ssp_section3_file @@ -1179,9 +1179,9 @@ def test_parse_ssp_section3_file(ssp_section3_file, dfs): second = m6_objs[1] third = m6_objs[2] - assert first.RPT_MONTH_YEAR == 201810 - assert second.RPT_MONTH_YEAR == 201811 - assert third.RPT_MONTH_YEAR == 201812 + assert first.RPT_MONTH_YEAR == 202010 + assert second.RPT_MONTH_YEAR == 202011 + assert third.RPT_MONTH_YEAR == 202012 assert first.SSPMOE_FAMILIES == 15869 assert second.SSPMOE_FAMILIES == 16008 @@ -1330,7 +1330,7 @@ def tribal_section_3_file(stt_user, stt): @pytest.mark.django_db() def test_parse_tribal_section_3_file(tribal_section_3_file, dfs): """Test parsing Tribal TANF Section 3 submission.""" - tribal_section_3_file.year = 2020 + tribal_section_3_file.year = 2021 tribal_section_3_file.quarter = 'Q1' dfs.datafile = tribal_section_3_file From f897a98272bdf7e5c00da398441739bc0aea6aac Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 20 Mar 2024 10:30:28 -0400 Subject: [PATCH 05/20] remove generated file --- tdrs-backend/None | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tdrs-backend/None diff --git a/tdrs-backend/None b/tdrs-backend/None deleted file mode 100644 index 8e88b1fb0..000000000 --- a/tdrs-backend/None +++ /dev/null @@ -1,3 +0,0 @@ -HEADER20204G06 TAN1 N -T620204000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884 -TRAILER0000001 From 4d87f183ae21524f013a71712b6afa61c76dec5b Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 20 Mar 2024 11:08:46 -0400 Subject: [PATCH 06/20] requested change --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py | 9 ++++++--- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py | 5 ++++- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py | 4 +++- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py | 4 +++- .../tdpservice/parsers/schema_defs/tribal_tanf/t6.py | 4 +++- .../tdpservice/parsers/schema_defs/tribal_tanf/t7.py | 4 +++- 6 files changed, 22 insertions(+), 8 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 94d77673f..21d11d79a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -6,6 +6,9 @@ from ...row_schema import RowSchema, SchemaManager from ... import validators from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument +import datetime + +minYear = datetime.date.today().year - 5; s1 = RowSchema( document=SSP_M6DataSubmissionDocument(), @@ -47,7 +50,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(minYear), validators.quarterIsValid() ] ), @@ -208,7 +211,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(minYear), validators.quarterIsValid() ] ), @@ -369,7 +372,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(minYear), validators.quarterIsValid() ] ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 806b4251a..5ca6134ea 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -5,6 +5,9 @@ from ...transforms import calendar_quarter_to_rpt_month_year from ... import validators from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument +import datetime + +minYear = datetime.date.today().year - 5; schemas = [] @@ -47,7 +50,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(minYear), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index f0e5f5621..4dff7dbfe 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -6,7 +6,9 @@ from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T6DataSubmissionDocument +import datetime +minYear = datetime.date.today().year - 5; s1 = RowSchema( document=TANF_T6DataSubmissionDocument(), @@ -54,7 +56,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(minYear), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index 81e0f70e5..bfc41da44 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -5,7 +5,9 @@ from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T7DataSubmissionDocument +import datetime +minYear = datetime.date.today().year - 5; schemas = [] validator_index = 7 @@ -47,7 +49,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(minYear), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index 41980bb9b..a226c15fe 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -6,7 +6,9 @@ from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T6DataSubmissionDocument +import datetime +minYear = datetime.date.today().year - 5; s1 = RowSchema( document=Tribal_TANF_T6DataSubmissionDocument(), @@ -42,7 +44,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(minYear), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index a17820514..8bd815e83 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -5,7 +5,9 @@ from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T7DataSubmissionDocument +import datetime +minYear = datetime.date.today().year - 5; schemas = [] validator_index = 7 @@ -47,7 +49,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(minYear), validators.quarterIsValid(), ], ), From 4545374ee3c820ca8044e4d4a7f8762e1d68f5b4 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 20 Mar 2024 11:14:24 -0400 Subject: [PATCH 07/20] fixing lint --- tdrs-backend/tdpservice/parsers/fields.py | 2 +- tdrs-backend/tdpservice/parsers/parse.py | 2 +- .../tdpservice/parsers/schema_defs/ssp/m6.py | 2 +- .../tdpservice/parsers/schema_defs/ssp/m7.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t6.py | 2 +- .../tdpservice/parsers/schema_defs/tanf/t7.py | 2 +- .../parsers/schema_defs/tribal_tanf/t6.py | 2 +- .../parsers/schema_defs/tribal_tanf/t7.py | 2 +- tdrs-backend/tdpservice/parsers/test/test_parse.py | 4 ++-- .../tdpservice/parsers/test/test_validators.py | 14 ++++++-------- 10 files changed, 16 insertions(+), 18 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index b283f13d3..6aff02f26 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -81,4 +81,4 @@ def parse_value(self, line): """Parse and transform the value for a field given a line, startIndex, endIndex, and field type.""" value = super().parse_value(line) # wrap in try catch return none to avoid db write - return self.transform_func(value, **self.kwargs) + return self.transform_func(value, **self.kwargs) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 1c71dc2c7..eb32370b8 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -256,7 +256,7 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): record.datafile = datafile unsaved_records.setdefault(s.document, []).append(record) - except ValueError as ex: + except ValueError as ex: err_obj = generate_error( schema=None, error_category=ParserErrorCategoryChoices.PRE_CHECK, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 21d11d79a..0375f60de 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -8,7 +8,7 @@ from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument import datetime -minYear = datetime.date.today().year - 5; +minYear = datetime.date.today().year - 5 s1 = RowSchema( document=SSP_M6DataSubmissionDocument(), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 5ca6134ea..7ee2aa8f3 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -7,7 +7,7 @@ from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument import datetime -minYear = datetime.date.today().year - 5; +minYear = datetime.date.today().year - 5 schemas = [] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index 4dff7dbfe..06c552258 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -8,7 +8,7 @@ from tdpservice.search_indexes.documents.tanf import TANF_T6DataSubmissionDocument import datetime -minYear = datetime.date.today().year - 5; +minYear = datetime.date.today().year - 5 s1 = RowSchema( document=TANF_T6DataSubmissionDocument(), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index bfc41da44..83e834720 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -7,7 +7,7 @@ from tdpservice.search_indexes.documents.tanf import TANF_T7DataSubmissionDocument import datetime -minYear = datetime.date.today().year - 5; +minYear = datetime.date.today().year - 5 schemas = [] validator_index = 7 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index a226c15fe..57d38cc49 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -8,7 +8,7 @@ from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T6DataSubmissionDocument import datetime -minYear = datetime.date.today().year - 5; +minYear = datetime.date.today().year - 5 s1 = RowSchema( document=Tribal_TANF_T6DataSubmissionDocument(), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index 8bd815e83..d2c1473cf 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -7,7 +7,7 @@ from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T7DataSubmissionDocument import datetime -minYear = datetime.date.today().year - 5; +minYear = datetime.date.today().year - 5 schemas = [] validator_index = 7 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 66352930d..16ce42e0f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1497,14 +1497,14 @@ def tribal_section_4_bad_quarter(stt_user, stt): @pytest.mark.django_db() def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): - """Test handling invalid quarter value that raises a ValueError exception""" + """Test handling invalid quarter value that raises a ValueError exception.""" tribal_section_4_bad_quarter.year = 2020 tribal_section_4_bad_quarter.quarter = 'Q1' dfs.datafile = tribal_section_4_bad_quarter parse.parse_datafile(tribal_section_4_bad_quarter) parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter) - + assert parser_errors.count() == 1 error = parser_errors.first() diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 27d071e6f..9f32ac78c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -102,7 +102,7 @@ def test_validate__FAM_AFF__SSN(): assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS', 'SSN']) @pytest.mark.parametrize( - "value, valid", + "value, valid", [ ("20201", True), ("20202", True), @@ -458,24 +458,22 @@ def test_notEmpty_returns_nonexistent_substring(): assert error == "111 333 contains blanks between positions 10 and 12." -@pytest.mark.parametrize("test_input", [1,2,3,4]) +@pytest.mark.parametrize("test_input", [1, 2, 3, 4]) def test_quarterIsValid_returns_true_if_valid(test_input): - """Test `quarterIsValid` gives a valid result for values 1-4""" - + """Test `quarterIsValid` gives a valid result for values 1-4.""" validator = validators.quarterIsValid() is_valid, error = validator(test_input) - + assert is_valid is True assert error is None @pytest.mark.parametrize("test_input", [" ", 0, 5, "A"]) def test_quarterIsValid_returns_false_if_invalid(test_input): - """Test `quarterIsValid` gives an invalid result for values not 1-4""" - + """Test `quarterIsValid` gives an invalid result for values not 1-4.""" validator = validators.quarterIsValid() is_valid, error = validator(test_input) - + assert is_valid is False assert error == f"{test_input} is not a valid quarter." From dffbe66d76f3efe011da689eb6992ca8c7185f39 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 20 Mar 2024 16:20:00 -0400 Subject: [PATCH 08/20] reworked try/catch to a more appropriate area --- tdrs-backend/tdpservice/parsers/parse.py | 53 +++++++------------ .../tdpservice/parsers/test/test_parse.py | 14 +++-- tdrs-backend/tdpservice/parsers/transforms.py | 5 +- 3 files changed, 32 insertions(+), 40 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index eb32370b8..9e7af9ce9 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -236,41 +236,24 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): schema_manager = get_schema_manager(line, section, program_type) - try: - records = manager_parse_line(line, schema_manager, generate_error, is_encrypted) - - record_number = 0 - for i in range(len(records)): - r = records[i] - record_number += 1 - record, record_is_valid, record_errors = r - if not record_is_valid: - logger.debug(f"Record #{i} from line {line_number} is invalid.") - line_errors = errors.get(f"{line_number}_{i}", {}) - line_errors.update({record_number: record_errors}) - errors.update({f"{line_number}_{i}": record_errors}) - unsaved_parser_errors.update({f"{line_number}_{i}": record_errors}) - num_errors += len(record_errors) - if record: - s = schema_manager.schemas[i] - record.datafile = datafile - unsaved_records.setdefault(s.document, []).append(record) - - except ValueError as ex: - err_obj = generate_error( - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message=ex, - record=None, - field=None - ) - num_errors += 1 - parse_error = {line_number: [err_obj]} - unsaved_parser_errors.update(parse_error) - rollback_records(unsaved_records, datafile) - rollback_parser_errors(datafile) - bulk_create_errors(parse_error, num_errors, flush=True) - return errors + records = manager_parse_line(line, schema_manager, generate_error, is_encrypted) + + record_number = 0 + for i in range(len(records)): + r = records[i] + record_number += 1 + record, record_is_valid, record_errors = r + if not record_is_valid: + logger.debug(f"Record #{i} from line {line_number} is invalid.") + line_errors = errors.get(f"{line_number}_{i}", {}) + line_errors.update({record_number: record_errors}) + errors.update({f"{line_number}_{i}": record_errors}) + unsaved_parser_errors.update({f"{line_number}_{i}": record_errors}) + num_errors += len(record_errors) + if record: + s = schema_manager.schemas[i] + record.datafile = datafile + unsaved_records.setdefault(s.document, []).append(record) all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count, datafile) unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 16ce42e0f..2117a0aa2 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1505,8 +1505,14 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): parse.parse_datafile(tribal_section_4_bad_quarter) parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter) - assert parser_errors.count() == 1 - error = parser_errors.first() + assert parser_errors.count() == 36 + + for error in parser_errors: + assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert error.error_message == "Invalid quarter value." + if (error.field_name == "RPT_MONTH_YEAR"): + assert error.error_message == "RPT_MONTH_YEAR is required but a value was not provided." + elif (error.field_name == "CALENDAR_QUARTER"): + assert error.error_message == "0 is not a valid quarter." + else: + assert False, f"Unexpected field name {error.field_name}" diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index 75e4b2fe6..c4fd85367 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -8,7 +8,10 @@ def transform(value, **kwargs): value = str(value) year = value[:4] quarter = f"Q{value[-1]}" - month = transform_to_months(quarter)[month_index] + try: + month = transform_to_months(quarter)[month_index] + except ValueError: + return None return f"{year}{month_to_int(month)}" return transform From 9c02d6ad072347e9dd8a47456f3441f1c5f692e8 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 20 Mar 2024 16:23:46 -0400 Subject: [PATCH 09/20] remove comment --- tdrs-backend/tdpservice/parsers/fields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py index 6aff02f26..076743096 100644 --- a/tdrs-backend/tdpservice/parsers/fields.py +++ b/tdrs-backend/tdpservice/parsers/fields.py @@ -80,5 +80,4 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex, def parse_value(self, line): """Parse and transform the value for a field given a line, startIndex, endIndex, and field type.""" value = super().parse_value(line) - # wrap in try catch return none to avoid db write return self.transform_func(value, **self.kwargs) From a11c653bc6d8e9d43915f08dce565753c5fa2ef3 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 27 Mar 2024 10:10:21 -0400 Subject: [PATCH 10/20] remove relative year calc --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 0375f60de..94d77673f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -6,9 +6,6 @@ from ...row_schema import RowSchema, SchemaManager from ... import validators from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument -import datetime - -minYear = datetime.date.today().year - 5 s1 = RowSchema( document=SSP_M6DataSubmissionDocument(), @@ -50,7 +47,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(minYear), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), @@ -211,7 +208,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(minYear), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), @@ -372,7 +369,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(minYear), + validators.dateYearIsLargerThan(2019), validators.quarterIsValid() ] ), From 40f2ecc1430768af020c7d32a5109bc9630cb44c Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 3 Apr 2024 10:54:25 -0400 Subject: [PATCH 11/20] lint --- .../tdpservice/parsers/test/test_parse.py | 17 +++++++++-------- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 54752e31f..39c841693 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1673,12 +1673,13 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): assert parser_errors.count() == 2 - for error in parser_errors: - assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK + error1 = parser_errors[0] + error2 = parser_errors[1] - if (error.field_name == None): - assert error.error_message == "No records created." - elif (error.field_name == "Record_Type"): - assert error.error_message == "Reporting month year None does not match file reporting year:2020, quarter:Q1." - else: - assert False, f"Unexpected field name {error.field_name}" + assert error1.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert error1.field_name is None + assert error1.error_message == "No records created." + + assert error2.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert error2.field_name == "Record_Type" + assert error2.error_message == "Reporting month year None does not match file reporting year:2020, quarter:Q1." diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 1254ef06d..1c7449898 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -178,7 +178,7 @@ def validate_reporting_month_year_fields_header(line, row_schema_instance): field_month_year = row_schema_instance.get_field_values_by_names(line, ['RPT_MONTH_YEAR']).get('RPT_MONTH_YEAR') df_quarter = row_schema_instance.datafile.quarter df_year = row_schema_instance.datafile.year - + # get reporting month year from header field_year, field_quarter = year_month_to_year_quarter(f"{field_month_year}") file_calendar_year, file_calendar_qtr = fiscal_to_calendar(df_year, f"{df_quarter}") From 03d8f9f48b9fe4f45e60aa3eca500d3289573123 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 3 Apr 2024 11:06:16 -0400 Subject: [PATCH 12/20] lint --- .../tdpservice/parsers/test/test_parse.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 39c841693..434e69a3c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1673,13 +1673,13 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): assert parser_errors.count() == 2 - error1 = parser_errors[0] - error2 = parser_errors[1] - - assert error1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert error1.field_name is None - assert error1.error_message == "No records created." + msg = "Reporting month year None does not match file reporting year:2020, quarter:Q1." + for error in parser_errors: + assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert error2.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert error2.field_name == "Record_Type" - assert error2.error_message == "Reporting month year None does not match file reporting year:2020, quarter:Q1." + if (error.field_name is None): + assert error.error_message == "No records created." + elif (error.field_name == "Record_Type"): + assert error.error_message == msg + else: + assert False, f"Unexpected field name {error.field_name}" From a1d5cf57fb39bab0c6e0f42a89ff3a8480b66cc3 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 3 Apr 2024 14:24:49 -0400 Subject: [PATCH 13/20] update 2019 -> 2020 and fix datafiles and tests --- .../tdpservice/parsers/schema_defs/ssp/m6.py | 6 +-- .../tdpservice/parsers/schema_defs/ssp/m7.py | 5 +-- .../tdpservice/parsers/schema_defs/tanf/t6.py | 5 +-- .../tdpservice/parsers/schema_defs/tanf/t7.py | 4 +- .../parsers/schema_defs/tribal_tanf/t6.py | 5 +-- .../parsers/schema_defs/tribal_tanf/t7.py | 4 +- .../parsers/test/data/ADS.E2J.FTP3.TS06 | 4 +- .../parsers/test/data/ADS.E2J.FTP3.TS142 | 4 +- .../parsers/test/data/ADS.E2J.FTP4.TS06 | 4 +- .../parsers/test/data/ADS.E2J.NDM3.MS24 | 4 +- .../parsers/test/data/ADS.E2J.NDM4.MS24 | 4 +- .../test/data/tanf_section4_with_errors.txt | 4 +- .../test/data/tribal_section_4_fake.txt | 4 +- .../tdpservice/parsers/test/test_parse.py | 41 ++++++++++--------- tdrs-backend/tdpservice/parsers/validators.py | 1 - 15 files changed, 44 insertions(+), 55 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 722105c86..57ae9262f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -48,7 +48,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(2020), validators.quarterIsValid() ] ), @@ -211,7 +211,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(2020), validators.quarterIsValid() ] ), @@ -374,7 +374,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(2019), + validators.dateYearIsLargerThan(2020), validators.quarterIsValid() ] ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index d4d28aed9..9ddd4a212 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -5,9 +5,6 @@ from ...transforms import calendar_quarter_to_rpt_month_year from ... import validators from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument -import datetime - -minYear = datetime.date.today().year - 5 schemas = [] @@ -51,7 +48,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(minYear), + validators.dateYearIsLargerThan(2020), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index d6a96a948..c7e020ae3 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -6,9 +6,6 @@ from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T6DataSubmissionDocument -import datetime - -minYear = datetime.date.today().year - 5 s1 = RowSchema( document=TANF_T6DataSubmissionDocument(), @@ -57,7 +54,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(minYear), + validators.dateYearIsLargerThan(2020), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index f45439f29..714587acc 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -5,9 +5,7 @@ from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T7DataSubmissionDocument -import datetime -minYear = datetime.date.today().year - 5 schemas = [] validator_index = 7 @@ -50,7 +48,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(minYear), + validators.dateYearIsLargerThan(2020), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index dca8a9158..ddd0af8d0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -6,9 +6,6 @@ from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T6DataSubmissionDocument -import datetime - -minYear = datetime.date.today().year - 5 s1 = RowSchema( document=Tribal_TANF_T6DataSubmissionDocument(), @@ -45,7 +42,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(minYear), + validators.dateYearIsLargerThan(2020), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index 2575cb26c..97b1b8e5a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -5,9 +5,7 @@ from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T7DataSubmissionDocument -import datetime -minYear = datetime.date.today().year - 5 schemas = [] validator_index = 7 @@ -50,7 +48,7 @@ endIndex=7, required=True, validators=[ - validators.dateYearIsLargerThan(minYear), + validators.dateYearIsLargerThan(2020), validators.quarterIsValid(), ], ), diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06 index 8e88b1fb0..1fe4531ab 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06 @@ -1,3 +1,3 @@ -HEADER20204G06 TAN1 N -T620204000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884 +HEADER20214G06 TAN1 N +T620214000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884 TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 index 7ebd8bbcc..00ec788ff 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 @@ -1,3 +1,3 @@ -HEADER20204G00142TAN1EU -T620204 5 1 1 5 1 1 0 0 0 21097 22915 21441 40 41 40 5 6 4 8 9 11 27 26 25 85 89 85 18 21 19 67 68 66 0 0 0 0 0 0 0 0 0 3 3 0 +HEADER20214G00142TAN1EU +T620214 5 1 1 5 1 1 0 0 0 21097 22915 21441 40 41 40 5 6 4 8 9 11 27 26 25 85 89 85 18 21 19 67 68 66 0 0 0 0 0 0 0 0 0 3 3 0 TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 index 5c344cf42..b38ed125a 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 @@ -1,3 +1,3 @@ -HEADER20204S06 TAN1 N -T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +HEADER20214S06 TAN1 N +T720214101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 index 6ea30625f..9bd4925f6 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 @@ -1,3 +1,3 @@ -HEADER20204G24 SSP1 N -M620204000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229 +HEADER20214G24 SSP1 N +M620214000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229 TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 index fbf049ce9..74a14b2d0 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 @@ -1,3 +1,3 @@ -HEADER20204S24 SSP1 N -M720204101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229 +HEADER20214S24 SSP1 N +M720214101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229 TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt b/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt index 53590eccf..afe9ef1bf 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt @@ -1,3 +1,3 @@ -HEADER20204S06 TAN1 N -T720204700006853700680540068454103000312400000000003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +HEADER20214S06 TAN1 N +T720214700006853700680540068454103000312400000000003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt index 9ec1943d3..41fab8910 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt @@ -1,3 +1,3 @@ -HEADER20204S00142TAN1EU -T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 +HEADER20214S00142TAN1EU +T720214101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 TRAILER0000001 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 434e69a3c..a92b849fd 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1020,7 +1020,7 @@ def tanf_section3_file(stt_user, stt): @pytest.mark.django_db() def test_parse_tanf_section3_file(tanf_section3_file, dfs): """Test parsing TANF Section 3 submission.""" - tanf_section3_file.year = 2021 + tanf_section3_file.year = 2022 tanf_section3_file.quarter = 'Q1' dfs.datafile = tanf_section3_file @@ -1049,9 +1049,9 @@ def test_parse_tanf_section3_file(tanf_section3_file, dfs): second = t6_objs[1] third = t6_objs[2] - assert first.RPT_MONTH_YEAR == 202012 - assert second.RPT_MONTH_YEAR == 202011 - assert third.RPT_MONTH_YEAR == 202010 + assert first.RPT_MONTH_YEAR == 202112 + assert second.RPT_MONTH_YEAR == 202111 + assert third.RPT_MONTH_YEAR == 202110 assert first.NUM_APPROVED == 3924 assert second.NUM_APPROVED == 3977 @@ -1102,7 +1102,7 @@ def tanf_section4_file(stt_user, stt): @pytest.mark.django_db() def test_parse_tanf_section4_file(tanf_section4_file, dfs): """Test parsing TANF Section 4 submission.""" - tanf_section4_file.year = 2021 + tanf_section4_file.year = 2022 tanf_section4_file.quarter = 'Q1' dfs.datafile = tanf_section4_file @@ -1130,8 +1130,8 @@ def test_parse_tanf_section4_file(tanf_section4_file, dfs): first = t7_objs.first() sixth = t7_objs[5] - assert first.RPT_MONTH_YEAR == 202011 - assert sixth.RPT_MONTH_YEAR == 202012 + assert first.RPT_MONTH_YEAR == 202111 + assert sixth.RPT_MONTH_YEAR == 202112 assert first.TDRS_SECTION_IND == '2' assert sixth.TDRS_SECTION_IND == '2' @@ -1189,7 +1189,7 @@ def ssp_section4_file(stt_user, stt): @pytest.mark.django_db() def test_parse_ssp_section4_file(ssp_section4_file, dfs): """Test parsing SSP Section 4 submission.""" - ssp_section4_file.year = 2021 + ssp_section4_file.year = 2022 ssp_section4_file.quarter = 'Q1' dfs.datafile = ssp_section4_file @@ -1210,7 +1210,7 @@ def test_parse_ssp_section4_file(ssp_section4_file, dfs): assert m7_objs.count() == 12 first = m7_objs.first() - assert first.RPT_MONTH_YEAR == 202010 + assert first.RPT_MONTH_YEAR == 202110 assert first.FAMILIES_MONTH == 748 @pytest.fixture @@ -1277,7 +1277,7 @@ def ssp_section3_file(stt_user, stt): @pytest.mark.django_db() def test_parse_ssp_section3_file(ssp_section3_file, dfs): """Test parsing TANF Section 3 submission.""" - ssp_section3_file.year = 2021 + ssp_section3_file.year = 2022 ssp_section3_file.quarter = 'Q1' dfs.datafile = ssp_section3_file @@ -1305,9 +1305,9 @@ def test_parse_ssp_section3_file(ssp_section3_file, dfs): second = m6_objs[1] third = m6_objs[2] - assert first.RPT_MONTH_YEAR == 202010 - assert second.RPT_MONTH_YEAR == 202011 - assert third.RPT_MONTH_YEAR == 202012 + assert first.RPT_MONTH_YEAR == 202110 + assert second.RPT_MONTH_YEAR == 202111 + assert third.RPT_MONTH_YEAR == 202112 assert first.SSPMOE_FAMILIES == 15869 assert second.SSPMOE_FAMILIES == 16008 @@ -1460,7 +1460,7 @@ def tribal_section_3_file(stt_user, stt): @pytest.mark.django_db() def test_parse_tribal_section_3_file(tribal_section_3_file, dfs): """Test parsing Tribal TANF Section 3 submission.""" - tribal_section_3_file.year = 2021 + tribal_section_3_file.year = 2022 tribal_section_3_file.quarter = 'Q1' dfs.datafile = tribal_section_3_file @@ -1496,7 +1496,7 @@ def tribal_section_4_file(stt_user, stt): @pytest.mark.django_db() def test_parse_tribal_section_4_file(tribal_section_4_file, dfs): """Test parsing Tribal TANF Section 4 submission.""" - tribal_section_4_file.year = 2021 + tribal_section_4_file.year = 2022 tribal_section_4_file.quarter = 'Q1' dfs.datafile = tribal_section_4_file @@ -1519,8 +1519,8 @@ def test_parse_tribal_section_4_file(tribal_section_4_file, dfs): first = t7_objs.first() sixth = t7_objs[5] - assert first.RPT_MONTH_YEAR == 202011 - assert sixth.RPT_MONTH_YEAR == 202012 + assert first.RPT_MONTH_YEAR == 202111 + assert sixth.RPT_MONTH_YEAR == 202112 assert first.TDRS_SECTION_IND == '2' assert sixth.TDRS_SECTION_IND == '2' @@ -1593,6 +1593,8 @@ def tanf_section_4_file_with_errors(stt_user, stt): @pytest.mark.django_db() def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, dfs): """Test parsing TANF Section 4 submission.""" + tanf_section_4_file_with_errors.year = 2022 + tanf_section_4_file_with_errors.quarter = 'Q1' dfs.datafile = tanf_section_4_file_with_errors parse.parse_datafile(tanf_section_4_file_with_errors, dfs) @@ -1611,6 +1613,7 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d assert TANF_T7.objects.all().count() == 18 parser_errors = ParserError.objects.filter(file=tanf_section_4_file_with_errors) + assert parser_errors.count() == 6 t7_objs = TANF_T7.objects.all().order_by('FAMILIES_MONTH') @@ -1618,8 +1621,8 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d first = t7_objs.first() sixth = t7_objs[5] - assert first.RPT_MONTH_YEAR == 202011 - assert sixth.RPT_MONTH_YEAR == 202010 + assert first.RPT_MONTH_YEAR == 202111 + assert sixth.RPT_MONTH_YEAR == 202110 assert first.TDRS_SECTION_IND == '1' assert sixth.TDRS_SECTION_IND == '1' diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 1c7449898..e90cf7762 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -182,7 +182,6 @@ def validate_reporting_month_year_fields_header(line, row_schema_instance): # get reporting month year from header field_year, field_quarter = year_month_to_year_quarter(f"{field_month_year}") file_calendar_year, file_calendar_qtr = fiscal_to_calendar(df_year, f"{df_quarter}") - return (True, None) if str(file_calendar_year) == str(field_year) and file_calendar_qtr == field_quarter else ( False, f"Reporting month year {field_month_year} " + f"does not match file reporting year:{df_year}, quarter:{df_quarter}.", From c3368ee3e53e5895c65e158a69090c57d4327321 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 8 Apr 2024 11:34:12 -0600 Subject: [PATCH 14/20] - Update tests --- .../tdpservice/parsers/test/test_validators.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index ce25840d9..ecc8e97e8 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -123,9 +123,9 @@ def test_validate__FAM_AFF__SSN(): def test_quarterIsValid(value, valid): """Test `quarterIsValid`.""" val = validators.quarterIsValid() - result = val(value) + result = val(value, RowSchema(), "friendly_name", "item_no") - errorText = None if valid else f"{value[-1:]} is not a valid quarter." + errorText = None if valid else f"T1: {value[-1:]} is not a valid quarter." assert result == (valid, errorText) def test_validateSSN(): @@ -472,7 +472,7 @@ def test_notEmpty_returns_nonexistent_substring(): def test_quarterIsValid_returns_true_if_valid(test_input): """Test `quarterIsValid` gives a valid result for values 1-4.""" validator = validators.quarterIsValid() - is_valid, error = validator(test_input) + is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") assert is_valid is True assert error is None @@ -482,10 +482,10 @@ def test_quarterIsValid_returns_true_if_valid(test_input): def test_quarterIsValid_returns_false_if_invalid(test_input): """Test `quarterIsValid` gives an invalid result for values not 1-4.""" validator = validators.quarterIsValid() - is_valid, error = validator(test_input) + is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error == f"{test_input} is not a valid quarter." + assert error == f"T1: {test_input} is not a valid quarter." @pytest.mark.usefixtures('db') class TestCat3ValidatorsBase: From 83ab2f17b25f97474751a8eb0669ebb12c84f97c Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 8 Apr 2024 11:38:44 -0600 Subject: [PATCH 15/20] - Fix parse test --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 50d80cb63..b849dd52c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1675,12 +1675,9 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): parse.parse_datafile(tribal_section_4_bad_quarter, dfs) parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter) - for error in parser_errors: - print(f"{error}") - assert parser_errors.count() == 2 - msg = "Reporting month year None does not match file reporting year:2020, quarter:Q1." + msg = "T7: Reporting month year None does not match file reporting year:2020, quarter:Q1." for error in parser_errors: assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK From 6a09786c29e692797ec855a074fe73b2f31e8fe6 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 9 Apr 2024 10:52:03 -0600 Subject: [PATCH 16/20] - Add cat1 validator for calendar quarter - Update test --- .../tdpservice/parsers/schema_defs/ssp/m6.py | 3 +++ .../tdpservice/parsers/schema_defs/ssp/m7.py | 1 + .../tdpservice/parsers/schema_defs/tanf/t6.py | 3 +++ .../tdpservice/parsers/schema_defs/tanf/t7.py | 1 + .../parsers/schema_defs/tribal_tanf/t6.py | 3 +++ .../parsers/schema_defs/tribal_tanf/t7.py | 1 + .../tribal_section_4_fake_bad_quarter.txt | 4 ++-- .../tdpservice/parsers/test/test_parse.py | 17 +++++------------ .../parsers/test/test_validators.py | 19 +++++++++++++++++++ tdrs-backend/tdpservice/parsers/validators.py | 12 ++++++++++++ 10 files changed, 50 insertions(+), 14 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index 979ca091e..e85bc8fb1 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -13,6 +13,7 @@ preparsing_validators=[ validators.recordHasLength(259), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual( @@ -177,6 +178,7 @@ preparsing_validators=[ validators.recordHasLength(259), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual( @@ -341,6 +343,7 @@ preparsing_validators=[ validators.recordHasLength(259), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 8d6dc85fb..8d6664a43 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -27,6 +27,7 @@ validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[], fields=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py index 6d94e5255..e2e3b1a7f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py @@ -13,6 +13,7 @@ preparsing_validators=[ validators.recordHasLength(379), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual( @@ -233,6 +234,7 @@ preparsing_validators=[ validators.recordHasLength(379), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual( @@ -447,6 +449,7 @@ preparsing_validators=[ validators.recordHasLength(379), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py index 42fe3ec2a..581cd3883 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py @@ -27,6 +27,7 @@ validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[], fields=[ diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py index 50b7bb123..d32ed3693 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py @@ -13,6 +13,7 @@ preparsing_validators=[ validators.recordHasLength(379), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), @@ -221,6 +222,7 @@ preparsing_validators=[ validators.recordHasLength(379), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), @@ -423,6 +425,7 @@ preparsing_validators=[ validators.recordHasLength(379), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[ validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py index 164fbf322..dfbebb3bf 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py @@ -27,6 +27,7 @@ validators.notEmpty(0, 7), validators.notEmpty(validator_index, validator_index + 24), validators.field_year_month_with_header_year_quarter(), + validators.calendarQuarterIsValid(2, 7), ], postparsing_validators=[], fields=[ diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt index c4b6851d7..e8ad6c8f5 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt @@ -1,3 +1,3 @@ -HEADER20194S00142TAN1EU +HEADER20204S00142TAN1EU T72020 101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499 -TRAILER0000001 \ No newline at end of file +TRAILER0000001 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index b849dd52c..2e98be41a 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1668,22 +1668,15 @@ def tribal_section_4_bad_quarter(stt_user, stt): @pytest.mark.django_db() def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): """Test handling invalid quarter value that raises a ValueError exception.""" - tribal_section_4_bad_quarter.year = 2020 + tribal_section_4_bad_quarter.year = 2021 tribal_section_4_bad_quarter.quarter = 'Q1' dfs.datafile = tribal_section_4_bad_quarter parse.parse_datafile(tribal_section_4_bad_quarter, dfs) - parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter) + parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter).order_by('id') - assert parser_errors.count() == 2 + assert parser_errors.count() == 3 - msg = "T7: Reporting month year None does not match file reporting year:2020, quarter:Q1." - for error in parser_errors: - assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK + parser_errors.first().error_message == "T7: 2020 is an invalid value for the CALENDAR_QUARTER field." - if (error.field_name is None): - assert error.error_message == "No records created." - elif (error.field_name == "Record_Type"): - assert error.error_message == msg - else: - assert False, f"Unexpected field name {error.field_name}" + Tribal_TANF_T7.objects.count() == 0 diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index ecc8e97e8..350cde40f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -487,6 +487,25 @@ def test_quarterIsValid_returns_false_if_invalid(test_input): assert is_valid is False assert error == f"T1: {test_input} is not a valid quarter." +@pytest.mark.parametrize("value", ["T72020 ", "T720194", "T720200", "T720207", "T72020$"]) +def test_calendarQuarterIsValid_returns_invalid(value): + """Test `calendarQuarterIsValid` returns false on invalid input.""" + val = validators.calendarQuarterIsValid(2, 7) + is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") + + assert is_valid is False + assert error_msg == f"T1: {value[2:7]} is an invalid value for the CALENDAR_QUARTER field." + + +@pytest.mark.parametrize("value", ["T720201", "T720202", "T720203", "T720204"]) +def test_calendarQuarterIsValid_returns_valid(value): + """Test `calendarQuarterIsValid` returns false on invalid input.""" + val = validators.calendarQuarterIsValid(2, 7) + is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") + + assert is_valid is True + assert error_msg is None + @pytest.mark.usefixtures('db') class TestCat3ValidatorsBase: """A base test class for tests that evaluate category three validators.""" diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 7fa36c581..2b1ce0ff5 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -370,6 +370,18 @@ def caseNumberNotEmpty(start=0, end=None): ) +def calendarQuarterIsValid(start=0, end=None): + """Validate that the calendar quarter value is valid.""" + return make_validator( + lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 \ + and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type}: {value[start:end]} is an invalid value for the CALENDAR_QUARTER field.", + ) + + def isEmpty(start=0, end=None): """Validate that string value is only blanks.""" return make_validator( From b82621c9bc5347ebb9201b34fd5a903514f6a3ba Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 9 Apr 2024 10:54:54 -0600 Subject: [PATCH 17/20] - fix lint --- tdrs-backend/tdpservice/parsers/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 2b1ce0ff5..463895e63 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -373,8 +373,8 @@ def caseNumberNotEmpty(start=0, end=None): def calendarQuarterIsValid(start=0, end=None): """Validate that the calendar quarter value is valid.""" return make_validator( - lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 \ - and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, + lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 + and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5, lambda value, row_schema, friendly_name, From b77392eff5d878b7f8e8596a7cb7e28c70d64b57 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 11 Apr 2024 14:06:57 -0600 Subject: [PATCH 18/20] - debugging --- .circleci/build-and-test/jobs.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml index c143ea8d7..8fbefebdb 100644 --- a/.circleci/build-and-test/jobs.yml +++ b/.circleci/build-and-test/jobs.yml @@ -53,12 +53,12 @@ - disable-npm-audit - install-nodejs-packages: app-dir: tdrs-frontend - - run: - name: Wait for backend to become available - command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \ - "PATH=$PATH:/home/zap/.local/bin && - pip install wait-for-it && - wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\"" + # - run: + # name: Wait for backend to become available + # command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \ + # "PATH=$PATH:/home/zap/.local/bin && + # pip install wait-for-it && + # wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\"" - run: name: apply the migrations command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate" From 05e979cbba8750f7ee4be5a14ced83df8de914bb Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 11 Apr 2024 14:09:34 -0600 Subject: [PATCH 19/20] Revert "- debugging" This reverts commit b77392eff5d878b7f8e8596a7cb7e28c70d64b57. --- .circleci/build-and-test/jobs.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml index 8fbefebdb..c143ea8d7 100644 --- a/.circleci/build-and-test/jobs.yml +++ b/.circleci/build-and-test/jobs.yml @@ -53,12 +53,12 @@ - disable-npm-audit - install-nodejs-packages: app-dir: tdrs-frontend - # - run: - # name: Wait for backend to become available - # command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \ - # "PATH=$PATH:/home/zap/.local/bin && - # pip install wait-for-it && - # wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\"" + - run: + name: Wait for backend to become available + command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \ + "PATH=$PATH:/home/zap/.local/bin && + pip install wait-for-it && + wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\"" - run: name: apply the migrations command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate" From 0c783987e78bb1a5ee43aaa6cd07db19bee378db Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 15 Apr 2024 08:06:19 -0600 Subject: [PATCH 20/20] - Update based on ux feedback --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 3 ++- tdrs-backend/tdpservice/parsers/test/test_validators.py | 3 ++- tdrs-backend/tdpservice/parsers/validators.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 240e74419..f0593abe6 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1697,6 +1697,7 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): assert parser_errors.count() == 3 - parser_errors.first().error_message == "T7: 2020 is an invalid value for the CALENDAR_QUARTER field." + parser_errors.first().error_message == "T7: 2020 is invalid. Calendar Quarter must be a numeric" + \ + "representing the Calendar Year and Quarter formatted as YYYYQ" Tribal_TANF_T7.objects.count() == 0 diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 350cde40f..e80d35f65 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -494,7 +494,8 @@ def test_calendarQuarterIsValid_returns_invalid(value): is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no") assert is_valid is False - assert error_msg == f"T1: {value[2:7]} is an invalid value for the CALENDAR_QUARTER field." + assert error_msg == f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + \ + "representing the Calendar Year and Quarter formatted as YYYYQ" @pytest.mark.parametrize("value", ["T720201", "T720202", "T720203", "T720204"]) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 463895e63..99c8c9db4 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -378,7 +378,8 @@ def calendarQuarterIsValid(start=0, end=None): lambda value, row_schema, friendly_name, - item_num: f"{row_schema.record_type}: {value[start:end]} is an invalid value for the CALENDAR_QUARTER field.", + item_num: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric " + "representing the Calendar Year and Quarter formatted as YYYYQ", )