Merge branch 'develop' into 2950-update-search-indexes-stt-filter

raft-tech · Jun 3, 2024 · 7f6369f · 7f6369f
2 parents e1fb547 + 52aa35f
commit 7f6369f
Show file tree

Hide file tree

Showing 36 changed files with 625 additions and 111 deletions.
diff --git a/tdrs-backend/docs/api/authentication.md b/tdrs-backend/docs/api/authentication.md
@@ -4,3 +4,14 @@ For clients to authenticate, they have to authenticate with Login.gov via the ba
  This will allow the backend to identify the browser which requested access and authorize them based on the cookie they provide in their API calls. 
 
  The secured portion of this authorization is due to the httpOnly cookie being inaccessible to the client's local browser. 
+
+# Generating API token
+
+In order to use APIs, an activated `OFA Sys Admin` user has to generate a new token and use it in the API request following these steps:
+1. User has to first login using frontend and going through the normal login process
+2. After user is logged in, user can grab a token at `/v1/security/get-token`
+3. The token then can be used in authorization header. As an example:
+
+```curl --location 'http://{host}/v1/users/' --header 'Authorization: Token {token}'```
+
+Note: the authentication token is available for 24 hours by default but this can be overridden using the `TOKEN_EXPIRATION_HOURS` environment variable.  
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py
@@ -1,7 +1,7 @@
 """Schema for SSP M1 record type."""
 
-
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M1DataSubmissionDocument
@@ -133,15 +133,16 @@
                     required=True,
                     validators=[validators.notEmpty()]
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
-                    name='COUNTY_FIPS_CODE',
+                    name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
-                    type='string',
+                    type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[validators.isNumber(),]
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="4",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py
@@ -370,14 +370,14 @@
                     item="37",
                     name='EDUCATION_LEVEL',
                     friendly_name="education level",
-                    type='number',
+                    type='string',
                     startIndex=55,
                     endIndex=57,
                     required=False,
                     validators=[
                         validators.or_validators(
-                            validators.isInLimits(0, 16), validators.isInLimits(98, 99)
-                        )
+                            validators.isInStringRange(1, 16), validators.isInStringRange(98, 99)
+                        ),
                     ]
                 ),
                 Field(
@@ -388,7 +388,7 @@
                     startIndex=57,
                     endIndex=58,
                     required=False,
-                    validators=[validators.oneOf([0, 1, 2, 3, 9])]
+                    validators=[validators.oneOf([1, 2, 3, 9])]
                 ),
                 Field(
                     item="39",
@@ -398,7 +398,7 @@
                     startIndex=58,
                     endIndex=59,
                     required=False,
-                    validators=[validators.oneOf([0, 1, 2, 9])]
+                    validators=[validators.oneOf([1, 2, 9])]
                 ),
                 Field(
                     item="40",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py
@@ -279,15 +279,15 @@
             item="68",
             name='EDUCATION_LEVEL',
             friendly_name="education level",
-            type='number',
+            type='string',
             startIndex=49,
             endIndex=51,
             required=True,
             validators=[
                 validators.or_validators(
-                    validators.isInStringRange(0, 16),
+                    validators.isInStringRange(1, 16),
                     validators.isInStringRange(98, 99)
-                )
+                ),
             ]
         ),
         Field(
@@ -298,7 +298,7 @@
             startIndex=51,
             endIndex=52,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 3, 9])]
+            validators=[validators.oneOf([1, 2, 3, 9])]
         ),
         Field(
             item="70A",
@@ -593,13 +593,13 @@
             item="68",
             name='EDUCATION_LEVEL',
             friendly_name="education level",
-            type='number',
+            type='string',
             startIndex=90,
             endIndex=92,
             required=True,
             validators=[
                 validators.or_validators(
-                    validators.isInStringRange(0, 16),
+                    validators.isInStringRange(1, 16),
                     validators.isInStringRange(98, 99)
                 )
             ]
@@ -612,7 +612,7 @@
             startIndex=92,
             endIndex=93,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 3, 9])]
+            validators=[validators.oneOf([1, 2, 3, 9])]
         ),
         Field(
             item="70A",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py
@@ -1,7 +1,7 @@
 """Schema for SSP M1 record type."""
 
-
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M4DataSubmissionDocument
@@ -54,15 +54,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[validators.isInStringRange(0, 999)],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="4",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py
@@ -347,7 +347,8 @@
                         validators.or_validators(
                             validators.isInStringRange(0, 16),
                             validators.isInStringRange(98, 99),
-                        )
+                        ),
+                        validators.notMatches("00")
                     ],
                 ),
                 Field(
@@ -359,10 +360,7 @@
                     endIndex=57,
                     required=False,
                     validators=[
-                        validators.or_validators(
-                            validators.isInLimits(0, 3),
-                            validators.matches(9)
-                        )
+                        validators.oneOf([1, 2, 3, 9]),
                     ],
                 ),
                 Field(

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -1,10 +1,10 @@
 """Schema for HEADER row of all submission types."""
 
 
-from ...transforms import calendar_quarter_to_rpt_month_year
-from ...fields import Field, TransformField
-from ...row_schema import RowSchema, SchemaManager
-from ... import validators
+from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument
 
 s1 = RowSchema(

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
@@ -1,9 +1,9 @@
 """Schema for TANF T7 Row."""
 
-from ...fields import Field, TransformField
-from ...row_schema import RowSchema, SchemaManager
-from ...transforms import calendar_quarter_to_rpt_month_year
-from ... import validators
+from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument
 
 schemas = []

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py
@@ -1,6 +1,7 @@
 """Schema for t1 record types."""
 
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T1DataSubmissionDocument
@@ -155,17 +156,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[
-                        validators.isNumber(),
-                    ],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="5",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py
@@ -295,7 +295,7 @@
             startIndex=51,
             endIndex=52,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 9])],
+            validators=[validators.oneOf([1, 2, 9])],
         ),
         Field(
             item="77A",
@@ -608,7 +608,7 @@
             startIndex=92,
             endIndex=93,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 9])],
+            validators=[validators.oneOf([1, 2, 9])],
         ),
         Field(
             item="77A",

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py
@@ -1,7 +1,7 @@
 """Schema for HEADER row of all submission types."""
 
-
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T4DataSubmissionDocument
@@ -55,15 +55,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="county fips code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[validators.isInStringRange(1, 999)],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="5",
@@ -72,7 +73,7 @@
                     type="string",
                     startIndex=22,
                     endIndex=24,
-                    required=True,
+                    required=False,
                     validators=[validators.isInStringRange(0, 99)],
                 ),
                 Field(
@@ -118,7 +119,7 @@
                     startIndex=32,
                     endIndex=33,
                     required=True,
-                    validators=[validators.isInLimits(1, 3)],
+                    validators=[validators.isInLimits(1, 2)],
                 ),
                 Field(
                     item="11",