Merge branch '2883-kibana-dashboard' of https://github.com/raft-tech/…

…TANF-app into 2883-kibana-dashboard
raft-tech · May 30, 2024 · e195916 · e195916
2 parents 5f51ce5 + aa11679
commit e195916
Show file tree

Hide file tree

Showing 23 changed files with 291 additions and 81 deletions.
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py
@@ -1,7 +1,7 @@
 """Schema for SSP M1 record type."""
 
-
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M1DataSubmissionDocument
@@ -133,15 +133,16 @@
                     required=True,
                     validators=[validators.notEmpty()]
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
-                    name='COUNTY_FIPS_CODE',
+                    name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
-                    type='string',
+                    type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[validators.isNumber(),]
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="4",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py
@@ -370,14 +370,14 @@
                     item="37",
                     name='EDUCATION_LEVEL',
                     friendly_name="education level",
-                    type='number',
+                    type='string',
                     startIndex=55,
                     endIndex=57,
                     required=False,
                     validators=[
                         validators.or_validators(
-                            validators.isInLimits(0, 16), validators.isInLimits(98, 99)
-                        )
+                            validators.isInStringRange(1, 16), validators.isInStringRange(98, 99)
+                        ),
                     ]
                 ),
                 Field(
@@ -388,7 +388,7 @@
                     startIndex=57,
                     endIndex=58,
                     required=False,
-                    validators=[validators.oneOf([0, 1, 2, 3, 9])]
+                    validators=[validators.oneOf([1, 2, 3, 9])]
                 ),
                 Field(
                     item="39",
@@ -398,7 +398,7 @@
                     startIndex=58,
                     endIndex=59,
                     required=False,
-                    validators=[validators.oneOf([0, 1, 2, 9])]
+                    validators=[validators.oneOf([1, 2, 9])]
                 ),
                 Field(
                     item="40",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py
@@ -279,15 +279,15 @@
             item="68",
             name='EDUCATION_LEVEL',
             friendly_name="education level",
-            type='number',
+            type='string',
             startIndex=49,
             endIndex=51,
             required=True,
             validators=[
                 validators.or_validators(
-                    validators.isInStringRange(0, 16),
+                    validators.isInStringRange(1, 16),
                     validators.isInStringRange(98, 99)
-                )
+                ),
             ]
         ),
         Field(
@@ -298,7 +298,7 @@
             startIndex=51,
             endIndex=52,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 3, 9])]
+            validators=[validators.oneOf([1, 2, 3, 9])]
         ),
         Field(
             item="70A",
@@ -593,13 +593,13 @@
             item="68",
             name='EDUCATION_LEVEL',
             friendly_name="education level",
-            type='number',
+            type='string',
             startIndex=90,
             endIndex=92,
             required=True,
             validators=[
                 validators.or_validators(
-                    validators.isInStringRange(0, 16),
+                    validators.isInStringRange(1, 16),
                     validators.isInStringRange(98, 99)
                 )
             ]
@@ -612,7 +612,7 @@
             startIndex=92,
             endIndex=93,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 3, 9])]
+            validators=[validators.oneOf([1, 2, 3, 9])]
         ),
         Field(
             item="70A",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py
@@ -1,7 +1,7 @@
 """Schema for SSP M1 record type."""
 
-
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M4DataSubmissionDocument
@@ -54,15 +54,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[validators.isInStringRange(0, 999)],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="4",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py
@@ -347,7 +347,8 @@
                         validators.or_validators(
                             validators.isInStringRange(0, 16),
                             validators.isInStringRange(98, 99),
-                        )
+                        ),
+                        validators.notMatches("00")
                     ],
                 ),
                 Field(
@@ -359,10 +360,7 @@
                     endIndex=57,
                     required=False,
                     validators=[
-                        validators.or_validators(
-                            validators.isInLimits(0, 3),
-                            validators.matches(9)
-                        )
+                        validators.oneOf([1, 2, 3, 9]),
                     ],
                 ),
                 Field(

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -1,10 +1,10 @@
 """Schema for HEADER row of all submission types."""
 
 
-from ...transforms import calendar_quarter_to_rpt_month_year
-from ...fields import Field, TransformField
-from ...row_schema import RowSchema, SchemaManager
-from ... import validators
+from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument
 
 s1 = RowSchema(

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
@@ -1,9 +1,9 @@
 """Schema for TANF T7 Row."""
 
-from ...fields import Field, TransformField
-from ...row_schema import RowSchema, SchemaManager
-from ...transforms import calendar_quarter_to_rpt_month_year
-from ... import validators
+from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument
 
 schemas = []

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py
@@ -1,6 +1,7 @@
 """Schema for t1 record types."""
 
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T1DataSubmissionDocument
@@ -155,17 +156,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[
-                        validators.isNumber(),
-                    ],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="5",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py
@@ -295,7 +295,7 @@
             startIndex=51,
             endIndex=52,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 9])],
+            validators=[validators.oneOf([1, 2, 9])],
         ),
         Field(
             item="77A",
@@ -608,7 +608,7 @@
             startIndex=92,
             endIndex=93,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 9])],
+            validators=[validators.oneOf([1, 2, 9])],
         ),
         Field(
             item="77A",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py
@@ -1,7 +1,7 @@
 """Schema for HEADER row of all submission types."""
 
-
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T4DataSubmissionDocument
@@ -55,15 +55,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[validators.isInStringRange(1, 999)],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="5",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py
@@ -1,8 +1,9 @@
 """Schema for Tribal TANF T1 record types."""
 
-from ...fields import Field
-from ...row_schema import RowSchema, SchemaManager
-from ... import validators
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T1DataSubmissionDocument
 
 t1 = SchemaManager(
@@ -156,17 +157,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=False,
-                    validators=[
-                        validators.isNumber(),
-                    ],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="5",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py
@@ -1,10 +1,10 @@
 """Schema for Tribal TANF T2 row of all submission types."""
 
 
-from ...transforms import tanf_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema, SchemaManager
-from ... import validators
+from tdpservice.parsers.transforms import tanf_ssn_decryption_func, zero_pad
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T2DataSubmissionDocument
 
 
@@ -622,7 +622,8 @@
                         validators.isInStringRange(0, 99),
                     ],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(2),
                     item="61",
                     name="ADD_WORK_ACTIVITIES",
                     friendly_name="additional work activities",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py
@@ -1,10 +1,10 @@
 """Schema for Tribal TANF T3 row of all submission types."""
 
 
-from ...transforms import tanf_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema, SchemaManager
-from ... import validators
+from tdpservice.parsers.transforms import tanf_ssn_decryption_func
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T3DataSubmissionDocument
 
 FIRST_CHILD = 1

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py
@@ -1,8 +1,9 @@
 """Schema for Tribal TANF T4 record types."""
 
-from ...fields import Field
-from ...row_schema import RowSchema, SchemaManager
-from ... import validators
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T4DataSubmissionDocument
 
 
@@ -54,15 +55,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=False,
-                    validators=[validators.matches("000")],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="5",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py
@@ -1,10 +1,10 @@
 """Schema for Tribal TANF T5 row of all submission types."""
 
 
-from ...transforms import tanf_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema, SchemaManager
-from ... import validators
+from tdpservice.parsers.transforms import tanf_ssn_decryption_func
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T5DataSubmissionDocument