From 96289b05cbd6136e319f25564b4a42e15cbd4eb4 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 15 Apr 2024 11:35:54 -0400 Subject: [PATCH 01/26] T3:Cat2 CITIZENSHIP_STATUS --- .../tdpservice/parsers/schema_defs/tanf/t3.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index c5554a182..04b4f4f18 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -293,7 +293,7 @@ startIndex=51, endIndex=52, required=False, - validators=[validators.oneOf([0, 1, 2, 9])], + validators=[validators.oneOf([1, 2, 9])], ), Field( item="77A", diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 62cbfab88..f839599d3 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1676,3 +1676,38 @@ def test_parse_no_records_file(no_records_file, dfs): assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK assert error.content_type is None assert error.object_id is None + + +@pytest.fixture +def t3_cat2_invalid_citizenship_file(): + """Fixture for T3 file with an invalid CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t2_invalid_dob_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20204A06 TAN1EN\n' + b'T320201011111111112420190127WTTTT90W022212222204398000000000 ' + b' \n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.mark.django_db() +# @pytest.mark.parametrize("") +def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs): + """Test parsing a TANF T3 record with an invalid CITIZENSHIP_STATUS.""" + dfs.datafile = t3_cat2_invalid_citizenship_file + t3_cat2_invalid_citizenship_file.year = 2021 + t3_cat2_invalid_citizenship_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(t3_cat2_invalid_citizenship_file, dfs) + + parser_errors = ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file).order_by("pk") + + assert parser_errors.count() == 1 + + citizenship_status_error = parser_errors.first() + + assert citizenship_status_error.error_message == "T3: 0 is not in [1, 2, 9]." From 23b6e28b7538dd5599de2f5de52088b0007faa6f Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 17 Apr 2024 11:02:31 -0400 Subject: [PATCH 02/26] SSP M2 validation changes --- .../tdpservice/parsers/schema_defs/ssp/m2.py | 9 ++-- .../tdpservice/parsers/test/test_parse.py | 41 +++++++++++++++++-- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 0f72a48e1..802b9067c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -370,14 +370,15 @@ item="37", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=55, endIndex=57, required=False, validators=[ validators.or_validators( validators.isInLimits(0, 16), validators.isInLimits(98, 99) - ) + ), + validators.notMatches('00') ] ), Field( @@ -388,7 +389,7 @@ startIndex=57, endIndex=58, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="39", @@ -398,7 +399,7 @@ startIndex=58, endIndex=59, required=False, - validators=[validators.oneOf([0, 1, 2, 9])] + validators=[validators.oneOf([1, 2, 9])] ), Field( item="40", diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index f839599d3..48bcd7663 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1684,9 +1684,9 @@ def t3_cat2_invalid_citizenship_file(): parsing_file = ParsingFileFactory( year=2021, quarter='Q1', - file__name='t2_invalid_dob_file.txt', + file__name='t3_invalid_citizenship_file.txt', file__section='Active Case Data', - file__data=(b'HEADER20204A06 TAN1EN\n' + file__data=(b'HEADER20204A06 TAN1ED\n' b'T320201011111111112420190127WTTTT90W022212222204398000000000 ' b' \n' b'TRAILER0000001 ') @@ -1694,7 +1694,6 @@ def t3_cat2_invalid_citizenship_file(): return parsing_file @pytest.mark.django_db() -# @pytest.mark.parametrize("") def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs): """Test parsing a TANF T3 record with an invalid CITIZENSHIP_STATUS.""" dfs.datafile = t3_cat2_invalid_citizenship_file @@ -1711,3 +1710,39 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs citizenship_status_error = parser_errors.first() assert citizenship_status_error.error_message == "T3: 0 is not in [1, 2, 9]." + +@pytest.fixture +def m2_cat2_invalid_37_38_39_file(): + """Fixture for M2 file with an invaliud EDUCATION_LEVEL, CITIZENSHIP_STATUS, and COOPERATION_CHILD_SUPPORT.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m2_cat2_invalid_37_38_39_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M2202310111111111275219811103WTTT#PW@W2221222222225012200001011935000000000000000000000000000000000000000000000000000000000000225300000000000000000000\n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.mark.django_db() +def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs): + """Test parsing an SSP M2 file with an invaliud EDUCATION_LEVEL, CITIZENSHIP_STATUS, and COOPERATION_CHILD_SUPPORT.""" + dfs.datafile = m2_cat2_invalid_37_38_39_file + m2_cat2_invalid_37_38_39_file.year = 2024 + m2_cat2_invalid_37_38_39_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m2_cat2_invalid_37_38_39_file, dfs) + + parser_errors = ParserError.objects.filter(file=m2_cat2_invalid_37_38_39_file).order_by("pk") + + assert parser_errors.count() == 3 + + education_level_error = parser_errors[0]; + citizenship_status_error = parser_errors[1]; + cooperation_child_support_error = parser_errors[2]; + + assert education_level_error.error_message == "M2: 00 matches 00." + assert citizenship_status_error.error_message == "M2: 0 is not in [1, 2, 3, 9]." + assert cooperation_child_support_error.error_message == "M2: 0 is not in [1, 2, 9]." diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 7fa36c581..509127d27 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -426,7 +426,7 @@ def isSmallerThanOrEqualTo(UpperBound): def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( - lambda value: value >= LowerBound and value <= UpperBound, + lambda value: int(value) >= LowerBound and int(value) <= UpperBound, lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger or equal to {LowerBound} and " f"smaller or equal to {UpperBound}." From 33078624775643d9ed552e8ce056a0394ac1cd78 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 17 Apr 2024 11:45:20 -0400 Subject: [PATCH 03/26] M3 validation changes --- .../tdpservice/parsers/schema_defs/ssp/m3.py | 11 +++-- .../tdpservice/parsers/test/test_parse.py | 47 +++++++++++++++++-- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index e4b173b2e..ee68998c6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -276,7 +276,7 @@ item="68", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=49, endIndex=51, required=True, @@ -284,7 +284,8 @@ validators.or_validators( validators.isInStringRange(0, 16), validators.isInStringRange(98, 99) - ) + ), + validators.notMatches('00') ] ), Field( @@ -295,7 +296,7 @@ startIndex=51, endIndex=52, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="70A", @@ -590,7 +591,7 @@ item="68", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=90, endIndex=92, required=True, @@ -609,7 +610,7 @@ startIndex=92, endIndex=93, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="70A", diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 48bcd7663..59bbd7875 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1713,21 +1713,22 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs @pytest.fixture def m2_cat2_invalid_37_38_39_file(): - """Fixture for M2 file with an invaliud EDUCATION_LEVEL, CITIZENSHIP_STATUS, and COOPERATION_CHILD_SUPPORT.""" + """Fixture for M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" parsing_file = ParsingFileFactory( year=2024, quarter='Q1', file__name='m2_cat2_invalid_37_38_39_file.txt', section='SSP Active Case Data', file__data=(b'HEADER20234A24 SSP1ED\n' - b'M2202310111111111275219811103WTTT#PW@W2221222222225012200001011935000000000000000000000000000000000000000000000000000000000000225300000000000000000000\n' + b'M2202310111111111275219811103WTTT#PW@W22212222222250122000010119350000000000000000000000000000000' + b'00000000000000000000000000000225300000000000000000000\n' b'TRAILER0000001 ') ) return parsing_file @pytest.mark.django_db() def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs): - """Test parsing an SSP M2 file with an invaliud EDUCATION_LEVEL, CITIZENSHIP_STATUS, and COOPERATION_CHILD_SUPPORT.""" + """Test parsing an SSP M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" dfs.datafile = m2_cat2_invalid_37_38_39_file m2_cat2_invalid_37_38_39_file.year = 2024 m2_cat2_invalid_37_38_39_file.quarter = 'Q1' @@ -1746,3 +1747,43 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert education_level_error.error_message == "M2: 00 matches 00." assert citizenship_status_error.error_message == "M2: 0 is not in [1, 2, 3, 9]." assert cooperation_child_support_error.error_message == "M2: 0 is not in [1, 2, 9]." + +@pytest.fixture +def m3_cat2_invalid_68_69_file(): + """Fixture for M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m3_cat2_invalid_68_69_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M320231011111111127420110615WTTTP99B#22212222204300000000000\n' + b'M320231011111111127120110615WTTTP99B#2221222220430110000000042010010133333333300000001100000099998888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.mark.django_db() +def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): + """Test parsing an SSP M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m3_cat2_invalid_68_69_file + m3_cat2_invalid_68_69_file.year = 2024 + m3_cat2_invalid_68_69_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m3_cat2_invalid_68_69_file, dfs) + + parser_errors = ParserError.objects.filter(file=m3_cat2_invalid_68_69_file).order_by("pk") + + for error in parser_errors: + print(f"{error}") + + assert parser_errors.count() == 3 + + education_level_error = parser_errors[0]; + citizenship_status_1_error = parser_errors[1]; + citizenship_status_2_error = parser_errors[2]; + + assert education_level_error.error_message == "M3: 00 matches 00." + assert citizenship_status_1_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." + assert citizenship_status_2_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." \ No newline at end of file From 356b2097123205bb9cc707b29e23ba565e22d40b Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 17 Apr 2024 11:57:34 -0400 Subject: [PATCH 04/26] T3 update validation on part 2 --- .../tdpservice/parsers/schema_defs/tanf/t3.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 04b4f4f18..dfd8f3d6e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -604,7 +604,7 @@ startIndex=92, endIndex=93, required=False, - validators=[validators.oneOf([0, 1, 2, 9])], + validators=[validators.oneOf([1, 2, 9])], ), Field( item="77A", diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 59bbd7875..20a814e00 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1687,8 +1687,8 @@ def t3_cat2_invalid_citizenship_file(): file__name='t3_invalid_citizenship_file.txt', file__section='Active Case Data', file__data=(b'HEADER20204A06 TAN1ED\n' - b'T320201011111111112420190127WTTTT90W022212222204398000000000 ' - b' \n' + b'T320201011111111112420190127WTTTT90W022212222204398000000000\n' + b'T320201011111111112420190127WTTTT90W02221222220439810000000042010010133333333300000001100000099998888\n' b'TRAILER0000001 ') ) return parsing_file @@ -1705,11 +1705,13 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs parser_errors = ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file).order_by("pk") - assert parser_errors.count() == 1 + assert parser_errors.count() == 2 - citizenship_status_error = parser_errors.first() + citizenship_status_1_error = parser_errors[0] + citizenship_status_2_error = parser_errors[1] - assert citizenship_status_error.error_message == "T3: 0 is not in [1, 2, 9]." + assert citizenship_status_1_error.error_message == "T3: 0 is not in [1, 2, 9]." + assert citizenship_status_2_error.error_message == "T3: 0 is not in [1, 2, 9]." @pytest.fixture def m2_cat2_invalid_37_38_39_file(): @@ -1758,7 +1760,8 @@ def m3_cat2_invalid_68_69_file(): section='SSP Active Case Data', file__data=(b'HEADER20234A24 SSP1ED\n' b'M320231011111111127420110615WTTTP99B#22212222204300000000000\n' - b'M320231011111111127120110615WTTTP99B#2221222220430110000000042010010133333333300000001100000099998888\n' + b'M320231011111111127120110615WTTTP99B#222122222043011000000004201001013333333330000000110000009999' + b'8888\n' b'TRAILER0000002 ') ) return parsing_file @@ -1775,9 +1778,6 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): parser_errors = ParserError.objects.filter(file=m3_cat2_invalid_68_69_file).order_by("pk") - for error in parser_errors: - print(f"{error}") - assert parser_errors.count() == 3 education_level_error = parser_errors[0]; From 87feef434a8502277b17a0512d9f66298762a301 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 17 Apr 2024 13:38:10 -0400 Subject: [PATCH 05/26] M3 validation --- .../tdpservice/parsers/schema_defs/ssp/m5.py | 10 +++--- .../tdpservice/parsers/test/test_parse.py | 36 ++++++++++++++++++- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index 7ba6aa2c6..bcbe4d7ff 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -340,14 +340,15 @@ name="EDUCATION_LEVEL", friendly_name="education level", type="string", - startIndex=54, + startIndex=54, endIndex=56, required=False, validators=[ validators.or_validators( validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), - ) + ), + validators.notMatches("00") ], ), Field( @@ -359,10 +360,7 @@ endIndex=57, required=False, validators=[ - validators.or_validators( - validators.isInLimits(0, 3), - validators.matches(9) - ) + validators.oneOf([1, 2, 3, 9]), ], ), Field( diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 20a814e00..3c724e4bf 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1786,4 +1786,38 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert education_level_error.error_message == "M3: 00 matches 00." assert citizenship_status_1_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." - assert citizenship_status_2_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." \ No newline at end of file + assert citizenship_status_2_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." + +@pytest.fixture +def m5_cat2_invalid_23_24_file(): + """Fixture for M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m5_cat2_invalid_23_24_file.txt', + section='SSP Closed Case Data', + file__data=(b'HEADER20184C24 SSP1ED\n' + b'M520181011111111161519791106WTTTY0ZB922212222222210112000112970000\n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.mark.django_db() +def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): + """Test parsing an SSP M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m5_cat2_invalid_23_24_file + m5_cat2_invalid_23_24_file.year = 2019 + m5_cat2_invalid_23_24_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m5_cat2_invalid_23_24_file, dfs) + + parser_errors = ParserError.objects.filter(file=m5_cat2_invalid_23_24_file).order_by("pk") + + assert parser_errors.count() == 2 + + education_level_error = parser_errors[0]; + citizenship_status_error = parser_errors[1]; + + assert education_level_error.error_message == "M5: 00 matches 00." + assert citizenship_status_error.error_message == "M5: 0 is not in [1, 2, 3, 9]." From 118f624e81a1ee8df21305e19f01e9c11bd3d71e Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 17 Apr 2024 13:53:03 -0400 Subject: [PATCH 06/26] lint fixes --- .../tdpservice/parsers/schema_defs/ssp/m5.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index bcbe4d7ff..080716cb0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -340,7 +340,7 @@ name="EDUCATION_LEVEL", friendly_name="education level", type="string", - startIndex=54, + startIndex=54, endIndex=56, required=False, validators=[ diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index fd81b3fb2..1e55a27f5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1713,8 +1713,9 @@ def t3_cat2_invalid_citizenship_file(): file__section='Active Case Data', file__data=(b'HEADER20204A06 TAN1ED\n' b'T320201011111111112420190127WTTTT90W022212222204398000000000\n' - b'T320201011111111112420190127WTTTT90W02221222220439810000000042010010133333333300000001100000099998888\n' - b'TRAILER0000001 ') + b'T320201011111111112420190127WTTTT90W0222122222043981000000004201001013333333330000000' + b'1100000099998888\n' + b'TRAILER0000002 ') ) return parsing_file @@ -1767,9 +1768,9 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - education_level_error = parser_errors[0]; - citizenship_status_error = parser_errors[1]; - cooperation_child_support_error = parser_errors[2]; + education_level_error = parser_errors[0] + citizenship_status_error = parser_errors[1] + cooperation_child_support_error = parser_errors[2] assert education_level_error.error_message == "M2: 00 matches 00." assert citizenship_status_error.error_message == "M2: 0 is not in [1, 2, 3, 9]." @@ -1805,9 +1806,9 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert parser_errors.count() == 3 - education_level_error = parser_errors[0]; - citizenship_status_1_error = parser_errors[1]; - citizenship_status_2_error = parser_errors[2]; + education_level_error = parser_errors[0] + citizenship_status_1_error = parser_errors[1] + citizenship_status_2_error = parser_errors[2] assert education_level_error.error_message == "M3: 00 matches 00." assert citizenship_status_1_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." @@ -1838,11 +1839,11 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): parse.parse_datafile(m5_cat2_invalid_23_24_file, dfs) parser_errors = ParserError.objects.filter(file=m5_cat2_invalid_23_24_file).order_by("pk") - + assert parser_errors.count() == 2 - education_level_error = parser_errors[0]; - citizenship_status_error = parser_errors[1]; + education_level_error = parser_errors[0] + citizenship_status_error = parser_errors[1] assert education_level_error.error_message == "M5: 00 matches 00." assert citizenship_status_error.error_message == "M5: 0 is not in [1, 2, 3, 9]." From 99841ec7621eb04fdfe7c16d6ff94009fcc7894e Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 17 Apr 2024 16:33:23 -0400 Subject: [PATCH 07/26] rest of the validators --- .../tdpservice/parsers/schema_defs/ssp/m1.py | 17 ++- .../tdpservice/parsers/schema_defs/ssp/m4.py | 13 +- .../tdpservice/parsers/schema_defs/tanf/t4.py | 13 +- .../parsers/schema_defs/tribal_tanf/t1.py | 10 +- .../parsers/schema_defs/tribal_tanf/t4.py | 12 +- .../tdpservice/parsers/test/test_parse.py | 142 ++++++++++++++++++ tdrs-backend/tdpservice/parsers/transforms.py | 7 + 7 files changed, 194 insertions(+), 20 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index a125727cb..faffad487 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -1,7 +1,7 @@ """Schema for SSP M1 record type.""" - -from tdpservice.parsers.fields import Field +from ...transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M1DataSubmissionDocument @@ -133,15 +133,20 @@ required=True, validators=[validators.notEmpty()] ), - Field( + TransformField( + zero_pad(3), item="2", - name='COUNTY_FIPS_CODE', + name="COUNTY_FIPS_CODE", friendly_name="county fips code", - type='string', + type="string", startIndex=19, endIndex=22, required=True, - validators=[validators.isNumber(),] + validators=[ + validators.isInStringRange(1, 999), + validators.recordHasLength(3), + validators.isNumber() + ], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index e5159ad21..2b8d10b49 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -1,7 +1,7 @@ """Schema for SSP M1 record type.""" - -from tdpservice.parsers.fields import Field +from ...transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M4DataSubmissionDocument @@ -54,7 +54,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -62,7 +63,11 @@ startIndex=19, endIndex=22, required=True, - validators=[validators.isInStringRange(0, 999)], + validators=[ + validators.isInStringRange(1, 999), + validators.recordHasLength(3), + validators.isNumber() + ], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index ef68876c9..991bdd137 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -1,7 +1,7 @@ """Schema for HEADER row of all submission types.""" - -from tdpservice.parsers.fields import Field +from ...transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T4DataSubmissionDocument @@ -55,7 +55,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -63,7 +64,11 @@ startIndex=19, endIndex=22, required=True, - validators=[validators.isInStringRange(1, 999)], + validators=[ + validators.isInStringRange(1, 999), + validators.recordHasLength(3), + validators.isNumber() + ], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index 77fa5d58e..4b1f84bbc 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -1,6 +1,7 @@ """Schema for Tribal TANF T1 record types.""" -from ...fields import Field +from ...transforms import zero_pad +from ...fields import Field, TransformField from ...row_schema import RowSchema, SchemaManager from ... import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T1DataSubmissionDocument @@ -156,7 +157,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -165,7 +167,9 @@ endIndex=22, required=False, validators=[ - validators.isNumber(), + validators.isInStringRange(1, 999), + validators.recordHasLength(3), + validators.isNumber() ], ), Field( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index ecb0a1548..c5193fe30 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -1,6 +1,7 @@ """Schema for Tribal TANF T4 record types.""" -from ...fields import Field +from ...transforms import zero_pad +from ...fields import Field, TransformField from ...row_schema import RowSchema, SchemaManager from ... import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T4DataSubmissionDocument @@ -54,7 +55,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -62,7 +64,11 @@ startIndex=19, endIndex=22, required=False, - validators=[validators.matches("000")], + validators=[ + validators.isInStringRange(1, 999), + validators.recordHasLength(3), + validators.isNumber() + ], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 1e55a27f5..5a3815967 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1847,3 +1847,145 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): assert education_level_error.error_message == "M5: 00 matches 00." assert citizenship_status_error.error_message == "M5: 0 is not in [1, 2, 3, 9]." + +@pytest.fixture +def t4_cat2_country_fips_code_zero_pad_file(): + """Fixture for TANF T4 COUNTRY_FIPS_CODE without leading zeros.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + section='Closed Case Data', + file__name='t4_cat2_country_fips_code_zero_pad_file.txt', + file__data=(b'HEADER20204C06 TAN1ED\n' + b'T420201011111111158 101400141123113 \n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.mark.django_db() +def test_parse_t4_cat2_country_fips_code_zero_pad_file(t4_cat2_country_fips_code_zero_pad_file, dfs): + """Test parsing a TANF T4 COUNTRY_FIPS_CODE without leading zeros.""" + dfs.datafile = t4_cat2_country_fips_code_zero_pad_file + t4_cat2_country_fips_code_zero_pad_file.year = 2021 + t4_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(t4_cat2_country_fips_code_zero_pad_file, dfs) + + parser_errors = ParserError.objects.filter(file=t4_cat2_country_fips_code_zero_pad_file).order_by("pk") + + assert parser_errors.count() == 0 + +@pytest.fixture +def m1_cat2_country_fips_code_zero_pad_file(): + """Fixture for SSP M1 COUNTRY_FIPS_CODE without leading zeros.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + section='SSP Active Case Data', + file__name='m1_cat2_country_fips_code_zero_pad_file.txt', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M120231011111111127 1014003510213211002720000000000000001054000000000000000000000000000000000022' + b'2222000000002229 \n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.mark.django_db() +def test_parse_m1_cat2_country_fips_code_zero_pad_file(m1_cat2_country_fips_code_zero_pad_file, dfs): + """Test parsing a SSP M1 COUNTRY_FIPS_CODE without leading zeros.""" + dfs.datafile = m1_cat2_country_fips_code_zero_pad_file + m1_cat2_country_fips_code_zero_pad_file.year = 2024 + m1_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m1_cat2_country_fips_code_zero_pad_file, dfs) + + parser_errors = ParserError.objects.filter(file=m1_cat2_country_fips_code_zero_pad_file).order_by("pk") + + assert parser_errors.count() == 0 + +@pytest.fixture +def m4_cat2_country_fips_code_zero_pad_file(): + """Fixture for SSP M4 COUNTRY_FIPS_CODE without leading zeros.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + section='SSP Closed Case Data', + file__name='m4_cat2_country_fips_code_zero_pad_file.txt', + file__data=(b'HEADER20184C24 SSP1ED\n' + b'M420181011111111161 100406911161112 \n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.mark.django_db() +def test_parse_m4_cat2_country_fips_code_zero_pad_file(m4_cat2_country_fips_code_zero_pad_file, dfs): + """Test parsing a SSP M4 COUNTRY_FIPS_CODE without leading zeros.""" + dfs.datafile = m4_cat2_country_fips_code_zero_pad_file + m4_cat2_country_fips_code_zero_pad_file.year = 2019 + m4_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m4_cat2_country_fips_code_zero_pad_file, dfs) + + parser_errors = ParserError.objects.filter(file=m4_cat2_country_fips_code_zero_pad_file).order_by("pk") + + assert parser_errors.count() == 0 + +@pytest.fixture +def tribal_t1_cat2_country_fips_code_zero_pad_file(): + """Fixture for Tribal T1 COUNTRY_FIPS_CODE without leading zeros.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + section='Tribal Active Case Data', + file__name='tribal_t1_cat2_country_fips_code_zero_pad_file.txt', + file__data=(b'HEADER20214A00142TAN1ED\n' + b'T120211011111111119255 140245112 23322 03 0 0 0 502 9 0 0 0 0 0 0 0 0 3 02' + b'22222 0 0222 12 \n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.mark.django_db() +def test_parse_tribal_t1_cat2_country_fips_code_zero_pad_file(tribal_t1_cat2_country_fips_code_zero_pad_file, dfs): + """Test parsing a Tribal T1 COUNTRY_FIPS_CODE without leading zeros.""" + dfs.datafile = tribal_t1_cat2_country_fips_code_zero_pad_file + tribal_t1_cat2_country_fips_code_zero_pad_file.year = 2022 + tribal_t1_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(tribal_t1_cat2_country_fips_code_zero_pad_file, dfs) + + parser_errors = ParserError.objects.filter(file=tribal_t1_cat2_country_fips_code_zero_pad_file).order_by("pk") + + assert parser_errors.count() == 0 + +@pytest.fixture +def tribal_t4_cat2_country_fips_code_zero_pad_file(): + """Fixture for Tribal T4 COUNTRY_FIPS_CODE without leading zeros.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + section='Tribal Closed Case Data', + file__name='tribal_t4_cat2_country_fips_code_zero_pad_file.txt', + file__data=(b'HEADER20194C00142TAN1ED\n' + b'T420191011111111762 1 0402451153123 \n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.mark.django_db() +def test_parse_tribal_t4_cat2_country_fips_code_zero_pad_file(tribal_t4_cat2_country_fips_code_zero_pad_file, dfs): + """Test parsing a Tribal T4 COUNTRY_FIPS_CODE without leading zeros.""" + dfs.datafile = tribal_t4_cat2_country_fips_code_zero_pad_file + tribal_t4_cat2_country_fips_code_zero_pad_file.year = 2020 + tribal_t4_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(tribal_t4_cat2_country_fips_code_zero_pad_file, dfs) + + parser_errors = ParserError.objects.filter(file=tribal_t4_cat2_country_fips_code_zero_pad_file).order_by("pk") + + assert parser_errors.count() == 0 diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index 75e4b2fe6..2a6046f4f 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -29,3 +29,10 @@ def ssp_ssn_decryption_func(value, **kwargs): decryption_table = str.maketrans(decryption_dict) return value.translate(decryption_table) return value + +def zero_pad(digits): + """Zero pad a string""" + def transform(value, **kwargs): + return value.lstrip().zfill(digits) + return transform + From d6d9432a306bdef77e6488303bfb96a8706b8d8a Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 22 Apr 2024 10:08:44 -0400 Subject: [PATCH 08/26] Fixing data file so test still fails, used to fail because != 000 --- .../parsers/test/data/ADS.E2J.FTP2.TS142.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt index 7ef4b46b0..4078f53ee 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt @@ -1,21 +1,21 @@ HEADER20194C00142TAN1ED -T420191011111111762255 0402451153123 +T420191011111111762 1 0402451153123 T520191011111111762120160102WTTTTT@YB2122222222221 822981 0 03 0 0 T520191011111111762120170526WTTTTTZPW2122221222221 822981 0 03 0 0 T520191011111111762319880112WTTTTTTY#2222212222222 122161 1591 0 0 T520191011111111762319610502WTTTTTT##2222212222222 222161 0601 0 0 -T420191011111112343255 0402451 91113 +T420191011111112343 1 0402451 91113 T520191011111112343119860308WTTTTTTTY2122222222221 122111 44162 0 0 -T420191011111112970255 0403561 91112 +T420191011111112970 1 0403561 91112 T520191011111112970119940807WTTTTT@#Z2122221222221 122121 10501 0 0 -T420191111111111339255 0403561 83113 +T420191111111111339 1 0403561 83113 T520191111111111339119880402WTTTTTZ#B2122221222223 221121 3571 0 0 T520191111111111339119970502WTTTTTTYB2122222222221 111111 8522 0 0 -T420191111111112073255 0403561151123 +T420191111111112073 1 0403561151123 T520191111111112073319900312WTTTTTT0@2122222222222 122121 0601 0 0 T520191111111112073319920507WTTTTT@B02122221222222 222121 0601 0 0 T520191111111112073120090514WTTTTT@@@2122222222221 822 11 0 03 0 0 -T420191111111112472255 0403561183113 +T420191111111112472 1 0403561183113 T520191111111112472120140814WTTTTTZZ02222212222221 422981 0 03 0 0 T520191111111112472119840305WTTTTT90W2122221222221 122101 40202 0 0 TRAILER 19 From 5ecfea167f5fbb865af933b3fbf02913627a51cc Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 22 Apr 2024 10:09:45 -0400 Subject: [PATCH 09/26] lint --- tdrs-backend/tdpservice/parsers/transforms.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index 2a6046f4f..4a43922be 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -31,8 +31,7 @@ def ssp_ssn_decryption_func(value, **kwargs): return value def zero_pad(digits): - """Zero pad a string""" + """Zero pad a string.""" def transform(value, **kwargs): return value.lstrip().zfill(digits) return transform - From 915b9a46933b309926c6b165e314be1266b58be7 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 29 Apr 2024 10:41:48 -0400 Subject: [PATCH 10/26] search index type changes --- .../search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py | 4 ++-- tdrs-backend/tdpservice/search_indexes/models/ssp.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py b/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py index 413859b9c..d8962a8b9 100644 --- a/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py @@ -87,7 +87,7 @@ class Migration(migrations.Migration): ('RELATIONSHIP_HOH', models.IntegerField(null=True)), ('PARENT_MINOR_CHILD', models.IntegerField(null=True)), ('NEEDS_PREGNANT_WOMAN', models.IntegerField(null=True)), - ('EDUCATION_LEVEL', models.IntegerField(null=True)), + ('EDUCATION_LEVEL', models.CharField(null=True)), ('CITIZENSHIP_STATUS', models.IntegerField(null=True)), ('COOPERATION_CHILD_SUPPORT', models.IntegerField(null=True)), ('EMPLOYMENT_STATUS', models.IntegerField(null=True)), @@ -154,7 +154,7 @@ class Migration(migrations.Migration): ('RECEIVE_SSI', models.IntegerField(null=True)), ('RELATIONSHIP_HOH', models.IntegerField(null=True)), ('PARENT_MINOR_CHILD', models.IntegerField(null=True)), - ('EDUCATION_LEVEL', models.IntegerField(null=True)), + ('EDUCATION_LEVEL', models.CharField(null=True)), ('CITIZENSHIP_STATUS', models.IntegerField(null=True)), ('UNEARNED_SSI', models.IntegerField(null=True)), ('OTHER_UNEARNED_INCOME', models.IntegerField(null=True)), diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index b11e6fff5..3f3b928f0 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -113,7 +113,7 @@ class SSP_M2(models.Model): RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) NEEDS_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) COOPERATION_CHILD_SUPPORT = models.IntegerField(null=True, blank=False) EMPLOYMENT_STATUS = models.IntegerField(null=True, blank=False) @@ -194,7 +194,7 @@ class SSP_M3(models.Model): RECEIVE_SSI = models.IntegerField(null=True, blank=False) RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) UNEARNED_SSI = models.IntegerField(null=True, blank=False) OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False) From 889716a16a71ab5b6fc1a6adb1d2b2831c19c2f4 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 29 Apr 2024 14:17:24 -0400 Subject: [PATCH 11/26] missed elastic schema change + autogen migration file --- .../migrations/0028_auto_20240429_1805.py | 23 +++++++++++++++++++ .../tdpservice/search_indexes/models/ssp.py | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0028_auto_20240429_1805.py diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0028_auto_20240429_1805.py b/tdrs-backend/tdpservice/search_indexes/migrations/0028_auto_20240429_1805.py new file mode 100644 index 000000000..4a7cf36b4 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0028_auto_20240429_1805.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.15 on 2024-04-29 18:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0027_tribal_ssp_tanf_dob_to_string'), + ] + + operations = [ + migrations.AlterField( + model_name='ssp_m2', + name='EDUCATION_LEVEL', + field=models.CharField(max_length=2, null=True), + ), + migrations.AlterField( + model_name='ssp_m3', + name='EDUCATION_LEVEL', + field=models.CharField(max_length=2, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index 3f3b928f0..bb5840323 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -113,7 +113,7 @@ class SSP_M2(models.Model): RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) NEEDS_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.CharField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) COOPERATION_CHILD_SUPPORT = models.IntegerField(null=True, blank=False) EMPLOYMENT_STATUS = models.IntegerField(null=True, blank=False) @@ -194,7 +194,7 @@ class SSP_M3(models.Model): RECEIVE_SSI = models.IntegerField(null=True, blank=False) RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.CharField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) UNEARNED_SSI = models.IntegerField(null=True, blank=False) OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False) From bbd8bb843fe8ff55167e0a7af8a0326c2304e0a4 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 29 Apr 2024 15:11:01 -0400 Subject: [PATCH 12/26] test transform + remove redundant integration tests --- .../tdpservice/parsers/test/test_parse.py | 142 ------------------ .../parsers/test/test_transforms.py | 21 +++ 2 files changed, 21 insertions(+), 142 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/test/test_transforms.py diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index b00c9bd11..09d79d8f0 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1872,145 +1872,3 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): assert education_level_error.error_message == "M5: 00 matches 00." assert citizenship_status_error.error_message == "M5: 0 is not in [1, 2, 3, 9]." - -@pytest.fixture -def t4_cat2_country_fips_code_zero_pad_file(): - """Fixture for TANF T4 COUNTRY_FIPS_CODE without leading zeros.""" - parsing_file = ParsingFileFactory( - year=2024, - quarter='Q1', - section='Closed Case Data', - file__name='t4_cat2_country_fips_code_zero_pad_file.txt', - file__data=(b'HEADER20204C06 TAN1ED\n' - b'T420201011111111158 101400141123113 \n' - b'TRAILER0000001 ') - ) - return parsing_file - -@pytest.mark.django_db() -def test_parse_t4_cat2_country_fips_code_zero_pad_file(t4_cat2_country_fips_code_zero_pad_file, dfs): - """Test parsing a TANF T4 COUNTRY_FIPS_CODE without leading zeros.""" - dfs.datafile = t4_cat2_country_fips_code_zero_pad_file - t4_cat2_country_fips_code_zero_pad_file.year = 2021 - t4_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' - dfs.save() - - parse.parse_datafile(t4_cat2_country_fips_code_zero_pad_file, dfs) - - parser_errors = ParserError.objects.filter(file=t4_cat2_country_fips_code_zero_pad_file).order_by("pk") - - assert parser_errors.count() == 0 - -@pytest.fixture -def m1_cat2_country_fips_code_zero_pad_file(): - """Fixture for SSP M1 COUNTRY_FIPS_CODE without leading zeros.""" - parsing_file = ParsingFileFactory( - year=2024, - quarter='Q1', - section='SSP Active Case Data', - file__name='m1_cat2_country_fips_code_zero_pad_file.txt', - file__data=(b'HEADER20234A24 SSP1ED\n' - b'M120231011111111127 1014003510213211002720000000000000001054000000000000000000000000000000000022' - b'2222000000002229 \n' - b'TRAILER0000001 ') - ) - return parsing_file - -@pytest.mark.django_db() -def test_parse_m1_cat2_country_fips_code_zero_pad_file(m1_cat2_country_fips_code_zero_pad_file, dfs): - """Test parsing a SSP M1 COUNTRY_FIPS_CODE without leading zeros.""" - dfs.datafile = m1_cat2_country_fips_code_zero_pad_file - m1_cat2_country_fips_code_zero_pad_file.year = 2024 - m1_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' - dfs.save() - - parse.parse_datafile(m1_cat2_country_fips_code_zero_pad_file, dfs) - - parser_errors = ParserError.objects.filter(file=m1_cat2_country_fips_code_zero_pad_file).order_by("pk") - - assert parser_errors.count() == 0 - -@pytest.fixture -def m4_cat2_country_fips_code_zero_pad_file(): - """Fixture for SSP M4 COUNTRY_FIPS_CODE without leading zeros.""" - parsing_file = ParsingFileFactory( - year=2024, - quarter='Q1', - section='SSP Closed Case Data', - file__name='m4_cat2_country_fips_code_zero_pad_file.txt', - file__data=(b'HEADER20184C24 SSP1ED\n' - b'M420181011111111161 100406911161112 \n' - b'TRAILER0000001 ') - ) - return parsing_file - -@pytest.mark.django_db() -def test_parse_m4_cat2_country_fips_code_zero_pad_file(m4_cat2_country_fips_code_zero_pad_file, dfs): - """Test parsing a SSP M4 COUNTRY_FIPS_CODE without leading zeros.""" - dfs.datafile = m4_cat2_country_fips_code_zero_pad_file - m4_cat2_country_fips_code_zero_pad_file.year = 2019 - m4_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' - dfs.save() - - parse.parse_datafile(m4_cat2_country_fips_code_zero_pad_file, dfs) - - parser_errors = ParserError.objects.filter(file=m4_cat2_country_fips_code_zero_pad_file).order_by("pk") - - assert parser_errors.count() == 0 - -@pytest.fixture -def tribal_t1_cat2_country_fips_code_zero_pad_file(): - """Fixture for Tribal T1 COUNTRY_FIPS_CODE without leading zeros.""" - parsing_file = ParsingFileFactory( - year=2024, - quarter='Q1', - section='Tribal Active Case Data', - file__name='tribal_t1_cat2_country_fips_code_zero_pad_file.txt', - file__data=(b'HEADER20214A00142TAN1ED\n' - b'T120211011111111119255 140245112 23322 03 0 0 0 502 9 0 0 0 0 0 0 0 0 3 02' - b'22222 0 0222 12 \n' - b'TRAILER0000001 ') - ) - return parsing_file - -@pytest.mark.django_db() -def test_parse_tribal_t1_cat2_country_fips_code_zero_pad_file(tribal_t1_cat2_country_fips_code_zero_pad_file, dfs): - """Test parsing a Tribal T1 COUNTRY_FIPS_CODE without leading zeros.""" - dfs.datafile = tribal_t1_cat2_country_fips_code_zero_pad_file - tribal_t1_cat2_country_fips_code_zero_pad_file.year = 2022 - tribal_t1_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' - dfs.save() - - parse.parse_datafile(tribal_t1_cat2_country_fips_code_zero_pad_file, dfs) - - parser_errors = ParserError.objects.filter(file=tribal_t1_cat2_country_fips_code_zero_pad_file).order_by("pk") - - assert parser_errors.count() == 0 - -@pytest.fixture -def tribal_t4_cat2_country_fips_code_zero_pad_file(): - """Fixture for Tribal T4 COUNTRY_FIPS_CODE without leading zeros.""" - parsing_file = ParsingFileFactory( - year=2024, - quarter='Q1', - section='Tribal Closed Case Data', - file__name='tribal_t4_cat2_country_fips_code_zero_pad_file.txt', - file__data=(b'HEADER20194C00142TAN1ED\n' - b'T420191011111111762 1 0402451153123 \n' - b'TRAILER0000001 ') - ) - return parsing_file - -@pytest.mark.django_db() -def test_parse_tribal_t4_cat2_country_fips_code_zero_pad_file(tribal_t4_cat2_country_fips_code_zero_pad_file, dfs): - """Test parsing a Tribal T4 COUNTRY_FIPS_CODE without leading zeros.""" - dfs.datafile = tribal_t4_cat2_country_fips_code_zero_pad_file - tribal_t4_cat2_country_fips_code_zero_pad_file.year = 2020 - tribal_t4_cat2_country_fips_code_zero_pad_file.quarter = 'Q1' - dfs.save() - - parse.parse_datafile(tribal_t4_cat2_country_fips_code_zero_pad_file, dfs) - - parser_errors = ParserError.objects.filter(file=tribal_t4_cat2_country_fips_code_zero_pad_file).order_by("pk") - - assert parser_errors.count() == 0 diff --git a/tdrs-backend/tdpservice/parsers/test/test_transforms.py b/tdrs-backend/tdpservice/parsers/test/test_transforms.py new file mode 100644 index 000000000..b88131d41 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/test_transforms.py @@ -0,0 +1,21 @@ +"""Test for Transforms.""" + +import pytest +from .. import transforms + +@pytest.mark.parametrize("value,digits,expected", [ + ("1", 3, "001"), + ("10", 3, "010"), + ("100", 3, "100"), + ("1000", 3, "1000"), + ("1 ", 3, "01 "), + ("1 ", 3, "1 "), + ("1", 0, "1"), + ("1", -1, "1") +]) +def test_zero_pad(value, digits, expected): + """Test zero_pad returns valid value.""" + transform = transforms.zero_pad(digits) + result = transform(value) + + assert result == expected From 3bb928712f1c6638d7354468979e184b48cfbd26 Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Mon, 29 Apr 2024 15:44:30 -0400 Subject: [PATCH 13/26] fixed autogenerated file --- .../search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py b/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py index d8962a8b9..8bae725de 100644 --- a/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py @@ -87,7 +87,7 @@ class Migration(migrations.Migration): ('RELATIONSHIP_HOH', models.IntegerField(null=True)), ('PARENT_MINOR_CHILD', models.IntegerField(null=True)), ('NEEDS_PREGNANT_WOMAN', models.IntegerField(null=True)), - ('EDUCATION_LEVEL', models.CharField(null=True)), + ('EDUCATION_LEVEL', models.CharField(max_length=2, null=True)), ('CITIZENSHIP_STATUS', models.IntegerField(null=True)), ('COOPERATION_CHILD_SUPPORT', models.IntegerField(null=True)), ('EMPLOYMENT_STATUS', models.IntegerField(null=True)), @@ -154,7 +154,7 @@ class Migration(migrations.Migration): ('RECEIVE_SSI', models.IntegerField(null=True)), ('RELATIONSHIP_HOH', models.IntegerField(null=True)), ('PARENT_MINOR_CHILD', models.IntegerField(null=True)), - ('EDUCATION_LEVEL', models.CharField(null=True)), + ('EDUCATION_LEVEL', models.CharField(max_length=2, null=True)), ('CITIZENSHIP_STATUS', models.IntegerField(null=True)), ('UNEARNED_SSI', models.IntegerField(null=True)), ('OTHER_UNEARNED_INCOME', models.IntegerField(null=True)), From 0e1a013fbe03982d3d3821aabea06527e81ef64d Mon Sep 17 00:00:00 2001 From: Andrew Trimpe Date: Wed, 1 May 2024 10:39:05 -0400 Subject: [PATCH 14/26] move test fixtures --- .../tdpservice/parsers/test/conftest.py | 64 +++++++++++++++++++ .../tdpservice/parsers/test/test_parse.py | 62 ------------------ 2 files changed, 64 insertions(+), 62 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/test/conftest.py diff --git a/tdrs-backend/tdpservice/parsers/test/conftest.py b/tdrs-backend/tdpservice/parsers/test/conftest.py new file mode 100644 index 000000000..1754e66e3 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/conftest.py @@ -0,0 +1,64 @@ +"""Fixtures for parsing integration tests.""" +import pytest +from .factories import ParsingFileFactory + +@pytest.fixture +def t3_cat2_invalid_citizenship_file(): + """Fixture for T3 file with an invalid CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t3_invalid_citizenship_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20204A06 TAN1ED\n' + b'T320201011111111112420190127WTTTT90W022212222204398000000000\n' + b'T320201011111111112420190127WTTTT90W0222122222043981000000004201001013333333330000000' + b'1100000099998888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.fixture +def m2_cat2_invalid_37_38_39_file(): + """Fixture for M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m2_cat2_invalid_37_38_39_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M2202310111111111275219811103WTTT#PW@W22212222222250122000010119350000000000000000000000000000000' + b'00000000000000000000000000000225300000000000000000000\n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.fixture +def m3_cat2_invalid_68_69_file(): + """Fixture for M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m3_cat2_invalid_68_69_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M320231011111111127420110615WTTTP99B#22212222204300000000000\n' + b'M320231011111111127120110615WTTTP99B#222122222043011000000004201001013333333330000000110000009999' + b'8888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.fixture +def m5_cat2_invalid_23_24_file(): + """Fixture for M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m5_cat2_invalid_23_24_file.txt', + section='SSP Closed Case Data', + file__data=(b'HEADER20184C24 SSP1ED\n' + b'M520181011111111161519791106WTTTY0ZB922212222222210112000112970000\n' + b'TRAILER0000001 ') + ) + return parsing_file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 09d79d8f0..857ad7ea9 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1727,23 +1727,6 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): Tribal_TANF_T7.objects.count() == 0 - -@pytest.fixture -def t3_cat2_invalid_citizenship_file(): - """Fixture for T3 file with an invalid CITIZENSHIP_STATUS.""" - parsing_file = ParsingFileFactory( - year=2021, - quarter='Q1', - file__name='t3_invalid_citizenship_file.txt', - file__section='Active Case Data', - file__data=(b'HEADER20204A06 TAN1ED\n' - b'T320201011111111112420190127WTTTT90W022212222204398000000000\n' - b'T320201011111111112420190127WTTTT90W0222122222043981000000004201001013333333330000000' - b'1100000099998888\n' - b'TRAILER0000002 ') - ) - return parsing_file - @pytest.mark.django_db() def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs): """Test parsing a TANF T3 record with an invalid CITIZENSHIP_STATUS.""" @@ -1764,21 +1747,6 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs assert citizenship_status_1_error.error_message == "T3: 0 is not in [1, 2, 9]." assert citizenship_status_2_error.error_message == "T3: 0 is not in [1, 2, 9]." -@pytest.fixture -def m2_cat2_invalid_37_38_39_file(): - """Fixture for M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" - parsing_file = ParsingFileFactory( - year=2024, - quarter='Q1', - file__name='m2_cat2_invalid_37_38_39_file.txt', - section='SSP Active Case Data', - file__data=(b'HEADER20234A24 SSP1ED\n' - b'M2202310111111111275219811103WTTT#PW@W22212222222250122000010119350000000000000000000000000000000' - b'00000000000000000000000000000225300000000000000000000\n' - b'TRAILER0000001 ') - ) - return parsing_file - @pytest.mark.django_db() def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs): """Test parsing an SSP M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" @@ -1801,22 +1769,6 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert citizenship_status_error.error_message == "M2: 0 is not in [1, 2, 3, 9]." assert cooperation_child_support_error.error_message == "M2: 0 is not in [1, 2, 9]." -@pytest.fixture -def m3_cat2_invalid_68_69_file(): - """Fixture for M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" - parsing_file = ParsingFileFactory( - year=2024, - quarter='Q1', - file__name='m3_cat2_invalid_68_69_file.txt', - section='SSP Active Case Data', - file__data=(b'HEADER20234A24 SSP1ED\n' - b'M320231011111111127420110615WTTTP99B#22212222204300000000000\n' - b'M320231011111111127120110615WTTTP99B#222122222043011000000004201001013333333330000000110000009999' - b'8888\n' - b'TRAILER0000002 ') - ) - return parsing_file - @pytest.mark.django_db() def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): """Test parsing an SSP M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" @@ -1839,20 +1791,6 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): assert citizenship_status_1_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." assert citizenship_status_2_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." -@pytest.fixture -def m5_cat2_invalid_23_24_file(): - """Fixture for M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" - parsing_file = ParsingFileFactory( - year=2024, - quarter='Q1', - file__name='m5_cat2_invalid_23_24_file.txt', - section='SSP Closed Case Data', - file__data=(b'HEADER20184C24 SSP1ED\n' - b'M520181011111111161519791106WTTTY0ZB922212222222210112000112970000\n' - b'TRAILER0000001 ') - ) - return parsing_file - @pytest.mark.django_db() def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): """Test parsing an SSP M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" From 3943595d3f7854e4dc2417076cfafef91dc73c41 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 15 May 2024 14:13:15 -0400 Subject: [PATCH 15/26] - Use absolute paths --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py | 2 +- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py | 2 +- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py | 2 +- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py | 2 +- tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index faffad487..e4562c234 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -1,6 +1,6 @@ """Schema for SSP M1 record type.""" -from ...transforms import zero_pad +from tdpservice.parsers.transforms import zero_pad from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 2b8d10b49..7bee38374 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -1,6 +1,6 @@ """Schema for SSP M1 record type.""" -from ...transforms import zero_pad +from tdpservice.parsers.transforms import zero_pad from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index 991bdd137..c1dd40c5e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -1,6 +1,6 @@ """Schema for HEADER row of all submission types.""" -from ...transforms import zero_pad +from tdpservice.parsers.transforms import zero_pad from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index 4b1f84bbc..fff76464f 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -1,6 +1,6 @@ """Schema for Tribal TANF T1 record types.""" -from ...transforms import zero_pad +from tdpservice.parsers.transforms import zero_pad from ...fields import Field, TransformField from ...row_schema import RowSchema, SchemaManager from ... import validators diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index c5193fe30..27f3228c8 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -1,6 +1,6 @@ """Schema for Tribal TANF T4 record types.""" -from ...transforms import zero_pad +from tdpservice.parsers.transforms import zero_pad from ...fields import Field, TransformField from ...row_schema import RowSchema, SchemaManager from ... import validators From 921a7a089a483a9ddf5314f4788c4e5a7686cc6e Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 15 May 2024 14:41:25 -0400 Subject: [PATCH 16/26] - Update string fields to string validators - added field length validator --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py | 2 +- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py | 3 +-- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py | 5 ++--- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py | 2 +- tdrs-backend/tdpservice/parsers/validators.py | 11 +++++++++++ 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index e4562c234..a87882ff5 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -144,7 +144,7 @@ required=True, validators=[ validators.isInStringRange(1, 999), - validators.recordHasLength(3), + validators.fieldHasLength(3), validators.isNumber() ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 802b9067c..c44bfc76a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -376,9 +376,8 @@ required=False, validators=[ validators.or_validators( - validators.isInLimits(0, 16), validators.isInLimits(98, 99) + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) ), - validators.notMatches('00') ] ), Field( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index ee68998c6..b255d7131 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -282,10 +282,9 @@ required=True, validators=[ validators.or_validators( - validators.isInStringRange(0, 16), + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) ), - validators.notMatches('00') ] ), Field( @@ -597,7 +596,7 @@ required=True, validators=[ validators.or_validators( - validators.isInStringRange(0, 16), + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) ) ] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 7bee38374..f9905f0c1 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -65,7 +65,7 @@ required=True, validators=[ validators.isInStringRange(1, 999), - validators.recordHasLength(3), + validators.fieldHasLength(3), validators.isNumber() ], ), diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index d41241d62..c10ea974e 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -274,6 +274,17 @@ def recordHasLength(length): ) +def fieldHasLength(length): + """Validate that the field value (string or array) has a length matching length param.""" + return make_validator( + lambda value: len(value) == length, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", + ) + + def intHasLength(num_digits): """Validate the number of digits in an integer.""" return make_validator( From d6464834f2def46d8ba7239d939f03152ce41de4 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 15 May 2024 14:46:06 -0400 Subject: [PATCH 17/26] - use absolute paths - update Tanf t1 to use transform field --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py | 8 ++++---- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py | 8 ++++---- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 10 +++++++--- .../tdpservice/parsers/schema_defs/tribal_tanf/t1.py | 6 +++--- .../tdpservice/parsers/schema_defs/tribal_tanf/t2.py | 8 ++++---- .../tdpservice/parsers/schema_defs/tribal_tanf/t3.py | 8 ++++---- .../tdpservice/parsers/schema_defs/tribal_tanf/t4.py | 6 +++--- .../tdpservice/parsers/schema_defs/tribal_tanf/t5.py | 8 ++++---- 8 files changed, 33 insertions(+), 29 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index e85bc8fb1..69d1bda7a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -1,10 +1,10 @@ """Schema for HEADER row of all submission types.""" -from ...transforms import calendar_quarter_to_rpt_month_year -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument s1 = RowSchema( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 8d6664a43..39ecf8f84 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -1,9 +1,9 @@ """Schema for TANF T7 Row.""" -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ...transforms import calendar_quarter_to_rpt_month_year -from ... import validators +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument schemas = [] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 368365f3c..728e0d8ae 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -1,6 +1,7 @@ """Schema for t1 record types.""" -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T1DataSubmissionDocument @@ -155,7 +156,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -164,7 +166,9 @@ endIndex=22, required=True, validators=[ - validators.isNumber(), + validators.isInStringRange(1, 999), + validators.fieldHasLength(3), + validators.isNumber() ], ), Field( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index fff76464f..5404e4775 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -1,9 +1,9 @@ """Schema for Tribal TANF T1 record types.""" from tdpservice.parsers.transforms import zero_pad -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T1DataSubmissionDocument t1 = SchemaManager( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 396418cba..1d7086d70 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T2 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T2DataSubmissionDocument diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py index 2252407d4..3b96046c7 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T3 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T3DataSubmissionDocument diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index 27f3228c8..53c299f75 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -1,9 +1,9 @@ """Schema for Tribal TANF T4 record types.""" from tdpservice.parsers.transforms import zero_pad -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T4DataSubmissionDocument diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index ef6769ac5..34ab152ad 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T5 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T5DataSubmissionDocument From d3756a812b1729eaac9af1a67c1823f2b9fccb28 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 15 May 2024 14:48:34 -0400 Subject: [PATCH 18/26] - rename migration --- ...28_auto_20240429_1805.py => 0028_education_level_to_string.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tdrs-backend/tdpservice/search_indexes/migrations/{0028_auto_20240429_1805.py => 0028_education_level_to_string.py} (100%) diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0028_auto_20240429_1805.py b/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py similarity index 100% rename from tdrs-backend/tdpservice/search_indexes/migrations/0028_auto_20240429_1805.py rename to tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py From 56dd94165072574031163778159aeae935493c3b Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 15 May 2024 15:59:04 -0400 Subject: [PATCH 19/26] - fixed comments surrounding tests --- .../tdpservice/parsers/test/test_parse.py | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 857ad7ea9..9caed2689 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1741,11 +1741,9 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs assert parser_errors.count() == 2 - citizenship_status_1_error = parser_errors[0] - citizenship_status_2_error = parser_errors[1] + for e in parser_errors: + assert e.error_message == "T3: 0 is not in [1, 2, 9]." - assert citizenship_status_1_error.error_message == "T3: 0 is not in [1, 2, 9]." - assert citizenship_status_2_error.error_message == "T3: 0 is not in [1, 2, 9]." @pytest.mark.django_db() def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs): @@ -1761,13 +1759,11 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) assert parser_errors.count() == 3 - education_level_error = parser_errors[0] - citizenship_status_error = parser_errors[1] - cooperation_child_support_error = parser_errors[2] - - assert education_level_error.error_message == "M2: 00 matches 00." - assert citizenship_status_error.error_message == "M2: 0 is not in [1, 2, 3, 9]." - assert cooperation_child_support_error.error_message == "M2: 0 is not in [1, 2, 9]." + error_msgs = {"M2: 00 is not in range [1, 16]. or M2: 00 is not in range [98, 99].", + "M2: 0 is not in [1, 2, 3, 9].", + "M2: 0 is not in [1, 2, 9]."} + for e in parser_errors: + assert e.error_message in error_msgs @pytest.mark.django_db() def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): @@ -1781,15 +1777,13 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): parser_errors = ParserError.objects.filter(file=m3_cat2_invalid_68_69_file).order_by("pk") - assert parser_errors.count() == 3 + assert parser_errors.count() == 4 - education_level_error = parser_errors[0] - citizenship_status_1_error = parser_errors[1] - citizenship_status_2_error = parser_errors[2] + error_msgs = {"M3: 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", + "M3: 0 is not in [1, 2, 3, 9]."} - assert education_level_error.error_message == "M3: 00 matches 00." - assert citizenship_status_1_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." - assert citizenship_status_2_error.error_message == "M3: 0 is not in [1, 2, 3, 9]." + for e in parser_errors: + assert e.error_message in error_msgs @pytest.mark.django_db() def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): @@ -1805,8 +1799,9 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): assert parser_errors.count() == 2 - education_level_error = parser_errors[0] - citizenship_status_error = parser_errors[1] + error_msgs = {"M5: 00 matches 00.", + "M5: 0 is not in [1, 2, 3, 9]."} + + for e in parser_errors: + assert e.error_message in error_msgs - assert education_level_error.error_message == "M5: 00 matches 00." - assert citizenship_status_error.error_message == "M5: 0 is not in [1, 2, 3, 9]." From 548d19cab411959f99782354dd8bd517e6d40742 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 15 May 2024 15:59:54 -0400 Subject: [PATCH 20/26] - fix lint --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 9caed2689..06d47e546 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1804,4 +1804,3 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): for e in parser_errors: assert e.error_message in error_msgs - From b7d413cadf5bcc0296f29ee82db90e173409cb74 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 15 May 2024 17:34:04 -0400 Subject: [PATCH 21/26] - excluding unnecessary cat4 errors --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 499604fa3..f5fecf8de 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1881,7 +1881,8 @@ def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs parse.parse_datafile(t3_cat2_invalid_citizenship_file, dfs) - parser_errors = ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file).order_by("pk") + parser_errors = ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") assert parser_errors.count() == 2 @@ -1899,7 +1900,8 @@ def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs) parse.parse_datafile(m2_cat2_invalid_37_38_39_file, dfs) - parser_errors = ParserError.objects.filter(file=m2_cat2_invalid_37_38_39_file).order_by("pk") + parser_errors = ParserError.objects.filter(file=m2_cat2_invalid_37_38_39_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") assert parser_errors.count() == 3 @@ -1919,7 +1921,8 @@ def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): parse.parse_datafile(m3_cat2_invalid_68_69_file, dfs) - parser_errors = ParserError.objects.filter(file=m3_cat2_invalid_68_69_file).order_by("pk") + parser_errors = ParserError.objects.filter(file=m3_cat2_invalid_68_69_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") assert parser_errors.count() == 4 @@ -1939,7 +1942,8 @@ def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): parse.parse_datafile(m5_cat2_invalid_23_24_file, dfs) - parser_errors = ParserError.objects.filter(file=m5_cat2_invalid_23_24_file).order_by("pk") + parser_errors = ParserError.objects.filter(file=m5_cat2_invalid_23_24_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") assert parser_errors.count() == 2 From 339ac3bbe3ccb630ee9c21430891dbdac0320798 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 17 May 2024 11:21:03 -0400 Subject: [PATCH 22/26] Revert "fixed autogenerated file" This reverts commit 3bb928712f1c6638d7354468979e184b48cfbd26. --- .../search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py b/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py index 8bae725de..d8962a8b9 100644 --- a/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py @@ -87,7 +87,7 @@ class Migration(migrations.Migration): ('RELATIONSHIP_HOH', models.IntegerField(null=True)), ('PARENT_MINOR_CHILD', models.IntegerField(null=True)), ('NEEDS_PREGNANT_WOMAN', models.IntegerField(null=True)), - ('EDUCATION_LEVEL', models.CharField(max_length=2, null=True)), + ('EDUCATION_LEVEL', models.CharField(null=True)), ('CITIZENSHIP_STATUS', models.IntegerField(null=True)), ('COOPERATION_CHILD_SUPPORT', models.IntegerField(null=True)), ('EMPLOYMENT_STATUS', models.IntegerField(null=True)), @@ -154,7 +154,7 @@ class Migration(migrations.Migration): ('RECEIVE_SSI', models.IntegerField(null=True)), ('RELATIONSHIP_HOH', models.IntegerField(null=True)), ('PARENT_MINOR_CHILD', models.IntegerField(null=True)), - ('EDUCATION_LEVEL', models.CharField(max_length=2, null=True)), + ('EDUCATION_LEVEL', models.CharField(null=True)), ('CITIZENSHIP_STATUS', models.IntegerField(null=True)), ('UNEARNED_SSI', models.IntegerField(null=True)), ('OTHER_UNEARNED_INCOME', models.IntegerField(null=True)), From 51103015742bdb2ae88d64742bc2e77ed3b33dff Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 17 May 2024 11:22:42 -0400 Subject: [PATCH 23/26] Revert "search index type changes" This reverts commit 915b9a46933b309926c6b165e314be1266b58be7. --- .../search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py b/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py index d8962a8b9..413859b9c 100644 --- a/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0007_ssp_m1_ssp_m2_ssp_m3.py @@ -87,7 +87,7 @@ class Migration(migrations.Migration): ('RELATIONSHIP_HOH', models.IntegerField(null=True)), ('PARENT_MINOR_CHILD', models.IntegerField(null=True)), ('NEEDS_PREGNANT_WOMAN', models.IntegerField(null=True)), - ('EDUCATION_LEVEL', models.CharField(null=True)), + ('EDUCATION_LEVEL', models.IntegerField(null=True)), ('CITIZENSHIP_STATUS', models.IntegerField(null=True)), ('COOPERATION_CHILD_SUPPORT', models.IntegerField(null=True)), ('EMPLOYMENT_STATUS', models.IntegerField(null=True)), @@ -154,7 +154,7 @@ class Migration(migrations.Migration): ('RECEIVE_SSI', models.IntegerField(null=True)), ('RELATIONSHIP_HOH', models.IntegerField(null=True)), ('PARENT_MINOR_CHILD', models.IntegerField(null=True)), - ('EDUCATION_LEVEL', models.CharField(null=True)), + ('EDUCATION_LEVEL', models.IntegerField(null=True)), ('CITIZENSHIP_STATUS', models.IntegerField(null=True)), ('UNEARNED_SSI', models.IntegerField(null=True)), ('OTHER_UNEARNED_INCOME', models.IntegerField(null=True)), From b9505168f4b124558814364294197a138ac3c08f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 17 May 2024 11:42:12 -0400 Subject: [PATCH 24/26] - remove length checks --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py | 1 - tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py | 1 - tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 1 - tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py | 1 - tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py | 1 - tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py | 1 - 6 files changed, 6 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index a87882ff5..ca2594e18 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -144,7 +144,6 @@ required=True, validators=[ validators.isInStringRange(1, 999), - validators.fieldHasLength(3), validators.isNumber() ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index f9905f0c1..9edc4d464 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -65,7 +65,6 @@ required=True, validators=[ validators.isInStringRange(1, 999), - validators.fieldHasLength(3), validators.isNumber() ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 728e0d8ae..7997a8cc0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -167,7 +167,6 @@ required=True, validators=[ validators.isInStringRange(1, 999), - validators.fieldHasLength(3), validators.isNumber() ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index c4a29f728..c49778ac7 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -66,7 +66,6 @@ required=True, validators=[ validators.isInStringRange(1, 999), - validators.recordHasLength(3), validators.isNumber() ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index 5404e4775..8db8545fe 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -168,7 +168,6 @@ required=False, validators=[ validators.isInStringRange(1, 999), - validators.recordHasLength(3), validators.isNumber() ], ), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index 271556989..0febdb46d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -66,7 +66,6 @@ required=False, validators=[ validators.isInStringRange(1, 999), - validators.recordHasLength(3), validators.isNumber() ], ), From 0fb77feabfad65f5660f59743c6eedc69c8757d3 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 20 May 2024 09:42:41 -0400 Subject: [PATCH 25/26] - remove unnecessary validators --- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py | 5 +---- tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py | 5 +---- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py | 5 +---- tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py | 5 +---- .../tdpservice/parsers/schema_defs/tribal_tanf/t1.py | 5 +---- .../tdpservice/parsers/schema_defs/tribal_tanf/t4.py | 5 +---- 6 files changed, 6 insertions(+), 24 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index ca2594e18..1c89435f2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -142,10 +142,7 @@ startIndex=19, endIndex=22, required=True, - validators=[ - validators.isInStringRange(1, 999), - validators.isNumber() - ], + validators=[validators.isNumber()], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 9edc4d464..31218b670 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -63,10 +63,7 @@ startIndex=19, endIndex=22, required=True, - validators=[ - validators.isInStringRange(1, 999), - validators.isNumber() - ], + validators=[validators.isNumber()], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 7997a8cc0..47b67e985 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -165,10 +165,7 @@ startIndex=19, endIndex=22, required=True, - validators=[ - validators.isInStringRange(1, 999), - validators.isNumber() - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index c49778ac7..fd96f938b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -64,10 +64,7 @@ startIndex=19, endIndex=22, required=True, - validators=[ - validators.isInStringRange(1, 999), - validators.isNumber() - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index 8db8545fe..eff60c425 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -166,10 +166,7 @@ startIndex=19, endIndex=22, required=False, - validators=[ - validators.isInStringRange(1, 999), - validators.isNumber() - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index 0febdb46d..d759493da 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -64,10 +64,7 @@ startIndex=19, endIndex=22, required=False, - validators=[ - validators.isInStringRange(1, 999), - validators.isNumber() - ], + validators=[validators.isNumber()], ), Field( item="5", From 01c4117f4db604927994c319f1c3465dac38b1bf Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 30 May 2024 09:54:20 -0400 Subject: [PATCH 26/26] - Updated ADD_WORK_ACTIVITIES to zero fill --- .../tdpservice/parsers/schema_defs/tribal_tanf/t2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 1d7086d70..cbf67c62d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -1,7 +1,7 @@ """Schema for Tribal TANF T2 row of all submission types.""" -from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.transforms import tanf_ssn_decryption_func, zero_pad from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators @@ -622,7 +622,8 @@ validators.isInStringRange(0, 99), ], ), - Field( + TransformField( + zero_pad(2), item="61", name="ADD_WORK_ACTIVITIES", friendly_name="additional work activities",