From 06c291622af5cf3937ca0f39650729f0518af19e Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Fri, 15 Mar 2024 09:50:24 -0400
Subject: [PATCH 001/127] date update + catching invalid quarter error

---
 tdrs-backend/tdpservice/parsers/fields.py     |  3 +-
 tdrs-backend/tdpservice/parsers/parse.py      | 52 ++++++++++++-------
 .../tdpservice/parsers/schema_defs/ssp/m6.py  |  6 +--
 .../tdpservice/parsers/schema_defs/ssp/m7.py  |  2 +-
 .../tdpservice/parsers/schema_defs/tanf/t6.py |  2 +-
 .../tdpservice/parsers/schema_defs/tanf/t7.py |  2 +-
 .../parsers/schema_defs/tribal_tanf/t6.py     |  2 +-
 .../parsers/schema_defs/tribal_tanf/t7.py     |  2 +-
 .../tribal_section_4_fake_bad_quarter.txt     |  3 ++
 .../tdpservice/parsers/test/test_parse.py     | 23 ++++++++
 .../parsers/test/test_validators.py           | 22 ++++++++
 11 files changed, 92 insertions(+), 27 deletions(-)
 create mode 100644 tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt

diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py
index 076743096..b283f13d3 100644
--- a/tdrs-backend/tdpservice/parsers/fields.py
+++ b/tdrs-backend/tdpservice/parsers/fields.py
@@ -80,4 +80,5 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex,
     def parse_value(self, line):
         """Parse and transform the value for a field given a line, startIndex, endIndex, and field type."""
         value = super().parse_value(line)
-        return self.transform_func(value, **self.kwargs)
+        # wrap in try catch return none to avoid db write
+        return self.transform_func(value, **self.kwargs) 
diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py
index 9e7af9ce9..a448ba0d4 100644
--- a/tdrs-backend/tdpservice/parsers/parse.py
+++ b/tdrs-backend/tdpservice/parsers/parse.py
@@ -236,24 +236,40 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
 
         schema_manager = get_schema_manager(line, section, program_type)
 
-        records = manager_parse_line(line, schema_manager, generate_error, is_encrypted)
-
-        record_number = 0
-        for i in range(len(records)):
-            r = records[i]
-            record_number += 1
-            record, record_is_valid, record_errors = r
-            if not record_is_valid:
-                logger.debug(f"Record #{i} from line {line_number} is invalid.")
-                line_errors = errors.get(f"{line_number}_{i}", {})
-                line_errors.update({record_number: record_errors})
-                errors.update({f"{line_number}_{i}": record_errors})
-                unsaved_parser_errors.update({f"{line_number}_{i}": record_errors})
-                num_errors += len(record_errors)
-            if record:
-                s = schema_manager.schemas[i]
-                record.datafile = datafile
-                unsaved_records.setdefault(s.document, []).append(record)
+        try:
+            records = manager_parse_line(line, schema_manager, generate_error, is_encrypted)
+
+            # this bubbles up when uyploading a file on port 3000 in the history (XLS)
+            # get the errors in this structure
+            record_number = 0
+            for i in range(len(records)):
+                r = records[i]
+                record_number += 1
+                record, record_is_valid, record_errors = r
+                if not record_is_valid:
+                    logger.debug(f"Record #{i} from line {line_number} is invalid.")
+                    line_errors = errors.get(f"{line_number}_{i}", {})
+                    line_errors.update({record_number: record_errors})
+                    errors.update({f"{line_number}_{i}": record_errors})
+                    unsaved_parser_errors.update({f"{line_number}_{i}": record_errors})
+                    num_errors += len(record_errors)
+                if record:
+                    s = schema_manager.schemas[i]
+                    record.datafile = datafile
+                    unsaved_records.setdefault(s.document, []).append(record)
+
+        except ValueError as ex:
+            err = generate_error(
+                schema=None,
+                error_category=ParserErrorCategoryChoices.PRE_CHECK,
+                error_message=ex,
+                record=None,
+                field="Record_Type"
+            )
+
+            errors.update({f"{line_number}_0": [err]})
+            unsaved_parser_errors.update({f"{line_number}_0": [err]})
+            num_errors += 1
 
         all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count, datafile)
         unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors)
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
index 6f7f67d34..94d77673f 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -47,7 +47,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),
@@ -208,7 +208,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),
@@ -369,7 +369,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
index 24a680b24..806b4251a 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
@@ -47,7 +47,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2019),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
index e800aafc4..f0e5f5621 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
@@ -54,7 +54,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid(),
             ],
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
index 80ec21f20..81e0f70e5 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
@@ -47,7 +47,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2019),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
index 2d6d230b8..41980bb9b 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
@@ -42,7 +42,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid(),
             ],
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
index 1212f300d..a17820514 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
@@ -47,7 +47,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2019),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt
new file mode 100644
index 000000000..c4b6851d7
--- /dev/null
+++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt
@@ -0,0 +1,3 @@
+HEADER20194S00142TAN1EU
+T72020 101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+TRAILER0000001         
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 24e49f5a7..ab1d083ae 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1467,3 +1467,26 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d
 
     assert first.FAMILIES_MONTH == 0
     assert sixth.FAMILIES_MONTH == 446
+
+@pytest.fixture
+def tribal_section_4_bad_quarter(stt_user, stt):
+    """Fixture for tribal_section_4_bad_quarter."""
+    return util.create_test_datafile('tribal_section_4_fake_bad_quarter.txt', stt_user, stt, "Tribal Stratum Data")
+
+@pytest.mark.django_db()
+def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
+    """Test parsing invalid quarter value"""
+    tribal_section_4_bad_quarter.year = 2020
+    tribal_section_4_bad_quarter.quarter = 'Q1'
+    dfs.datafile = tribal_section_4_bad_quarter
+
+    parse.parse_datafile(tribal_section_4_bad_quarter)
+    parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter)
+    
+    assert parser_errors.count() == 1
+    error = parser_errors.first()
+
+    print(error.error_message)
+
+    assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK
+    assert error.error_message == "Invalid quarter value."
diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py
index bd3eb88ce..7f3488883 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_validators.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py
@@ -336,6 +336,28 @@ def test_notEmpty_returns_nonexistent_substring():
     assert is_valid is False
     assert error == "111  333 contains blanks between positions 10 and 12."
 
+
+@pytest.mark.parametrize("test_input", [1,2,3,4])
+def test_quarterIsValid_returns_true_if_valid(test_input):
+    """Test `quarterIsValid` gives a valid result for values 1-4"""
+
+    validator = validators.quarterIsValid()
+    is_valid, error = validator(test_input)
+    
+    assert is_valid is True
+    assert error is None
+
+
+@pytest.mark.parametrize("test_input", [" ", 0, 5, "A"])
+def test_quarterIsValid_returns_false_if_invalid(test_input):
+    """Test `quarterIsValid` gives an invalid result for values not 1-4"""
+
+    validator = validators.quarterIsValid()
+    is_valid, error = validator(test_input)
+    
+    assert is_valid is False
+    assert error == f"{test_input} is not a valid quarter."
+
 @pytest.mark.usefixtures('db')
 class TestCat3ValidatorsBase:
     """A base test class for tests that evaluate category three validators."""

From d20225cb94e0e8dfe80427c1c9538c3fc3b0f8f4 Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Fri, 15 Mar 2024 09:50:24 -0400
Subject: [PATCH 002/127] date update + catching invalid quarter error

---
 tdrs-backend/tdpservice/parsers/fields.py     |  3 +-
 tdrs-backend/tdpservice/parsers/parse.py      | 52 ++++++++++++-------
 .../tdpservice/parsers/schema_defs/ssp/m6.py  |  6 +--
 .../tdpservice/parsers/schema_defs/ssp/m7.py  |  2 +-
 .../tdpservice/parsers/schema_defs/tanf/t6.py |  2 +-
 .../tdpservice/parsers/schema_defs/tanf/t7.py |  2 +-
 .../parsers/schema_defs/tribal_tanf/t6.py     |  2 +-
 .../parsers/schema_defs/tribal_tanf/t7.py     |  2 +-
 .../tribal_section_4_fake_bad_quarter.txt     |  3 ++
 .../tdpservice/parsers/test/test_parse.py     | 23 ++++++++
 .../parsers/test/test_validators.py           | 22 ++++++++
 11 files changed, 92 insertions(+), 27 deletions(-)
 create mode 100644 tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt

diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py
index 076743096..b283f13d3 100644
--- a/tdrs-backend/tdpservice/parsers/fields.py
+++ b/tdrs-backend/tdpservice/parsers/fields.py
@@ -80,4 +80,5 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex,
     def parse_value(self, line):
         """Parse and transform the value for a field given a line, startIndex, endIndex, and field type."""
         value = super().parse_value(line)
-        return self.transform_func(value, **self.kwargs)
+        # wrap in try catch return none to avoid db write
+        return self.transform_func(value, **self.kwargs) 
diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py
index 9e7af9ce9..a448ba0d4 100644
--- a/tdrs-backend/tdpservice/parsers/parse.py
+++ b/tdrs-backend/tdpservice/parsers/parse.py
@@ -236,24 +236,40 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
 
         schema_manager = get_schema_manager(line, section, program_type)
 
-        records = manager_parse_line(line, schema_manager, generate_error, is_encrypted)
-
-        record_number = 0
-        for i in range(len(records)):
-            r = records[i]
-            record_number += 1
-            record, record_is_valid, record_errors = r
-            if not record_is_valid:
-                logger.debug(f"Record #{i} from line {line_number} is invalid.")
-                line_errors = errors.get(f"{line_number}_{i}", {})
-                line_errors.update({record_number: record_errors})
-                errors.update({f"{line_number}_{i}": record_errors})
-                unsaved_parser_errors.update({f"{line_number}_{i}": record_errors})
-                num_errors += len(record_errors)
-            if record:
-                s = schema_manager.schemas[i]
-                record.datafile = datafile
-                unsaved_records.setdefault(s.document, []).append(record)
+        try:
+            records = manager_parse_line(line, schema_manager, generate_error, is_encrypted)
+
+            # this bubbles up when uyploading a file on port 3000 in the history (XLS)
+            # get the errors in this structure
+            record_number = 0
+            for i in range(len(records)):
+                r = records[i]
+                record_number += 1
+                record, record_is_valid, record_errors = r
+                if not record_is_valid:
+                    logger.debug(f"Record #{i} from line {line_number} is invalid.")
+                    line_errors = errors.get(f"{line_number}_{i}", {})
+                    line_errors.update({record_number: record_errors})
+                    errors.update({f"{line_number}_{i}": record_errors})
+                    unsaved_parser_errors.update({f"{line_number}_{i}": record_errors})
+                    num_errors += len(record_errors)
+                if record:
+                    s = schema_manager.schemas[i]
+                    record.datafile = datafile
+                    unsaved_records.setdefault(s.document, []).append(record)
+
+        except ValueError as ex:
+            err = generate_error(
+                schema=None,
+                error_category=ParserErrorCategoryChoices.PRE_CHECK,
+                error_message=ex,
+                record=None,
+                field="Record_Type"
+            )
+
+            errors.update({f"{line_number}_0": [err]})
+            unsaved_parser_errors.update({f"{line_number}_0": [err]})
+            num_errors += 1
 
         all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count, datafile)
         unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors)
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
index 6f7f67d34..94d77673f 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -47,7 +47,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),
@@ -208,7 +208,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),
@@ -369,7 +369,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
index 24a680b24..806b4251a 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
@@ -47,7 +47,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2019),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
index e800aafc4..f0e5f5621 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
@@ -54,7 +54,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid(),
             ],
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
index 80ec21f20..81e0f70e5 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
@@ -47,7 +47,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2019),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
index 2d6d230b8..41980bb9b 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
@@ -42,7 +42,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(1998),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid(),
             ],
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
index 1212f300d..a17820514 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
@@ -47,7 +47,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(1998),
+                        validators.dateYearIsLargerThan(2019),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt
new file mode 100644
index 000000000..c4b6851d7
--- /dev/null
+++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt
@@ -0,0 +1,3 @@
+HEADER20194S00142TAN1EU
+T72020 101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+TRAILER0000001         
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 2aaabcc19..4f34b5015 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1489,3 +1489,26 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d
 
     assert first.FAMILIES_MONTH == 0
     assert sixth.FAMILIES_MONTH == 446
+
+@pytest.fixture
+def tribal_section_4_bad_quarter(stt_user, stt):
+    """Fixture for tribal_section_4_bad_quarter."""
+    return util.create_test_datafile('tribal_section_4_fake_bad_quarter.txt', stt_user, stt, "Tribal Stratum Data")
+
+@pytest.mark.django_db()
+def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
+    """Test parsing invalid quarter value"""
+    tribal_section_4_bad_quarter.year = 2020
+    tribal_section_4_bad_quarter.quarter = 'Q1'
+    dfs.datafile = tribal_section_4_bad_quarter
+
+    parse.parse_datafile(tribal_section_4_bad_quarter)
+    parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter)
+    
+    assert parser_errors.count() == 1
+    error = parser_errors.first()
+
+    print(error.error_message)
+
+    assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK
+    assert error.error_message == "Invalid quarter value."
diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py
index 7d6a6011a..238f5edb7 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_validators.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py
@@ -449,6 +449,28 @@ def test_notEmpty_returns_nonexistent_substring():
     assert is_valid is False
     assert error == "111  333 contains blanks between positions 10 and 12."
 
+
+@pytest.mark.parametrize("test_input", [1,2,3,4])
+def test_quarterIsValid_returns_true_if_valid(test_input):
+    """Test `quarterIsValid` gives a valid result for values 1-4"""
+
+    validator = validators.quarterIsValid()
+    is_valid, error = validator(test_input)
+    
+    assert is_valid is True
+    assert error is None
+
+
+@pytest.mark.parametrize("test_input", [" ", 0, 5, "A"])
+def test_quarterIsValid_returns_false_if_invalid(test_input):
+    """Test `quarterIsValid` gives an invalid result for values not 1-4"""
+
+    validator = validators.quarterIsValid()
+    is_valid, error = validator(test_input)
+    
+    assert is_valid is False
+    assert error == f"{test_input} is not a valid quarter."
+
 @pytest.mark.usefixtures('db')
 class TestCat3ValidatorsBase:
     """A base test class for tests that evaluate category three validators."""

From 929e94f2ffdb1eec1f11890af0514d3f48463819 Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Mon, 18 Mar 2024 10:21:05 -0400
Subject: [PATCH 003/127] fixing tests

---
 tdrs-backend/None                             |  3 +++
 tdrs-backend/tdpservice/parsers/parse.py      | 17 +++++++-------
 .../tdpservice/parsers/test/test_parse.py     |  4 +---
 .../parsers/test/test_validators.py           | 22 +++++++++++++------
 4 files changed, 28 insertions(+), 18 deletions(-)
 create mode 100644 tdrs-backend/None

diff --git a/tdrs-backend/None b/tdrs-backend/None
new file mode 100644
index 000000000..8e88b1fb0
--- /dev/null
+++ b/tdrs-backend/None
@@ -0,0 +1,3 @@
+HEADER20204G06   TAN1 N
+T620204000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884
+TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py
index a448ba0d4..1c71dc2c7 100644
--- a/tdrs-backend/tdpservice/parsers/parse.py
+++ b/tdrs-backend/tdpservice/parsers/parse.py
@@ -239,8 +239,6 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
         try:
             records = manager_parse_line(line, schema_manager, generate_error, is_encrypted)
 
-            # this bubbles up when uyploading a file on port 3000 in the history (XLS)
-            # get the errors in this structure
             record_number = 0
             for i in range(len(records)):
                 r = records[i]
@@ -258,18 +256,21 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
                     record.datafile = datafile
                     unsaved_records.setdefault(s.document, []).append(record)
 
-        except ValueError as ex:
-            err = generate_error(
+        except ValueError as ex:        
+            err_obj = generate_error(
                 schema=None,
                 error_category=ParserErrorCategoryChoices.PRE_CHECK,
                 error_message=ex,
                 record=None,
-                field="Record_Type"
+                field=None
             )
-
-            errors.update({f"{line_number}_0": [err]})
-            unsaved_parser_errors.update({f"{line_number}_0": [err]})
             num_errors += 1
+            parse_error = {line_number: [err_obj]}
+            unsaved_parser_errors.update(parse_error)
+            rollback_records(unsaved_records, datafile)
+            rollback_parser_errors(datafile)
+            bulk_create_errors(parse_error, num_errors, flush=True)
+            return errors
 
         all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count, datafile)
         unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors)
diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 4f34b5015..0aaa98efa 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1497,7 +1497,7 @@ def tribal_section_4_bad_quarter(stt_user, stt):
 
 @pytest.mark.django_db()
 def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
-    """Test parsing invalid quarter value"""
+    """Test handling invalid quarter value that raises a ValueError exception"""
     tribal_section_4_bad_quarter.year = 2020
     tribal_section_4_bad_quarter.quarter = 'Q1'
     dfs.datafile = tribal_section_4_bad_quarter
@@ -1508,7 +1508,5 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
     assert parser_errors.count() == 1
     error = parser_errors.first()
 
-    print(error.error_message)
-
     assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK
     assert error.error_message == "Invalid quarter value."
diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py
index 238f5edb7..27d071e6f 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_validators.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py
@@ -101,17 +101,25 @@ def test_validate__FAM_AFF__SSN():
     result = validators.validate__FAM_AFF__SSN()(instance)
     assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS', 'SSN'])
 
-
-def test_quarterIsValid():
+@pytest.mark.parametrize(
+        "value, valid", 
+        [
+            ("20201", True),
+            ("20202", True),
+            ("20203", True),
+            ("20204", True),
+            ("20200", False),
+            ("20205", False),
+            ("2020 ", False),
+            ("2020A", False)
+        ])
+def test_quarterIsValid(value, valid):
     """Test `quarterIsValid`."""
-    value = "20204"
     val = validators.quarterIsValid()
     result = val(value)
-    assert result == (True, None)
 
-    value = "20205"
-    result = val(value)
-    assert result == (False, "5 is not a valid quarter.")
+    errorText = None if valid else f"{value[-1:]} is not a valid quarter."
+    assert result == (valid, errorText)
 
 def test_validateSSN():
     """Test `validateSSN`."""

From 4880610b71fe64aaeb88fcd543142eef2b19769c Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Mon, 18 Mar 2024 11:10:14 -0400
Subject: [PATCH 004/127] fixing tests

---
 .../parsers/test/data/ADS.E2J.FTP3.TS142           |  4 ++--
 .../tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 |  4 ++--
 .../tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 |  4 ++--
 tdrs-backend/tdpservice/parsers/test/test_parse.py | 14 +++++++-------
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142
index 6e5798b8a..7ebd8bbcc 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142
+++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142
@@ -1,3 +1,3 @@
-HEADER20194G00142TAN1EU
-T620194       5       1       1       5       1       1       0       0       0       21097       22915       21441      40      41      40       5       6       4       8       9      11      27      26      25      85      89      85      18      21      19      67      68      66       0       0       0       0       0       0       0       0       0       3       3       0
+HEADER20204G00142TAN1EU
+T620204       5       1       1       5       1       1       0       0       0       21097       22915       21441      40      41      40       5       6       4       8       9      11      27      26      25      85      89      85      18      21      19      67      68      66       0       0       0       0       0       0       0       0       0       3       3       0
 TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24
index bbe8ee4f0..6ea30625f 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24
+++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24
@@ -1,3 +1,3 @@
-HEADER20184G24   SSP1 N
-M620184000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229
+HEADER20204G24   SSP1 N
+M620204000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229
 TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24
index 1393729b8..fbf049ce9 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24
+++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24
@@ -1,3 +1,3 @@
-HEADER20184S24   SSP1 N
-M720184101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229                                                                                                                                                
+HEADER20204S24   SSP1 N
+M720204101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229                                                                                                                                                
 TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 0aaa98efa..66352930d 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1059,7 +1059,7 @@ def ssp_section4_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_ssp_section4_file(ssp_section4_file, dfs):
     """Test parsing SSP Section 4 submission."""
-    ssp_section4_file.year = 2019
+    ssp_section4_file.year = 2021
     ssp_section4_file.quarter = 'Q1'
 
     dfs.datafile = ssp_section4_file
@@ -1079,7 +1079,7 @@ def test_parse_ssp_section4_file(ssp_section4_file, dfs):
     assert m7_objs.count() == 12
 
     first = m7_objs.first()
-    assert first.RPT_MONTH_YEAR == 201810
+    assert first.RPT_MONTH_YEAR == 202010
     assert first.FAMILIES_MONTH == 748
 
 @pytest.fixture
@@ -1152,7 +1152,7 @@ def ssp_section3_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_ssp_section3_file(ssp_section3_file, dfs):
     """Test parsing TANF Section 3 submission."""
-    ssp_section3_file.year = 2019
+    ssp_section3_file.year = 2021
     ssp_section3_file.quarter = 'Q1'
     dfs.datafile = ssp_section3_file
 
@@ -1179,9 +1179,9 @@ def test_parse_ssp_section3_file(ssp_section3_file, dfs):
     second = m6_objs[1]
     third = m6_objs[2]
 
-    assert first.RPT_MONTH_YEAR == 201810
-    assert second.RPT_MONTH_YEAR == 201811
-    assert third.RPT_MONTH_YEAR == 201812
+    assert first.RPT_MONTH_YEAR == 202010
+    assert second.RPT_MONTH_YEAR == 202011
+    assert third.RPT_MONTH_YEAR == 202012
 
     assert first.SSPMOE_FAMILIES == 15869
     assert second.SSPMOE_FAMILIES == 16008
@@ -1330,7 +1330,7 @@ def tribal_section_3_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tribal_section_3_file(tribal_section_3_file, dfs):
     """Test parsing Tribal TANF Section 3 submission."""
-    tribal_section_3_file.year = 2020
+    tribal_section_3_file.year = 2021
     tribal_section_3_file.quarter = 'Q1'
 
     dfs.datafile = tribal_section_3_file

From f897a98272bdf7e5c00da398441739bc0aea6aac Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 20 Mar 2024 10:30:28 -0400
Subject: [PATCH 005/127] remove generated file

---
 tdrs-backend/None | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 tdrs-backend/None

diff --git a/tdrs-backend/None b/tdrs-backend/None
deleted file mode 100644
index 8e88b1fb0..000000000
--- a/tdrs-backend/None
+++ /dev/null
@@ -1,3 +0,0 @@
-HEADER20204G06   TAN1 N
-T620204000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884
-TRAILER0000001         

From 4d87f183ae21524f013a71712b6afa61c76dec5b Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 20 Mar 2024 11:08:46 -0400
Subject: [PATCH 006/127] requested change

---
 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py    | 9 ++++++---
 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py    | 5 ++++-
 tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py   | 4 +++-
 tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py   | 4 +++-
 .../tdpservice/parsers/schema_defs/tribal_tanf/t6.py     | 4 +++-
 .../tdpservice/parsers/schema_defs/tribal_tanf/t7.py     | 4 +++-
 6 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
index 94d77673f..21d11d79a 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -6,6 +6,9 @@
 from ...row_schema import RowSchema, SchemaManager
 from ... import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument
+import datetime
+
+minYear = datetime.date.today().year - 5;
 
 s1 = RowSchema(
     document=SSP_M6DataSubmissionDocument(),
@@ -47,7 +50,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(2019),
+                validators.dateYearIsLargerThan(minYear),
                 validators.quarterIsValid()
             ]
         ),
@@ -208,7 +211,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(2019),
+                validators.dateYearIsLargerThan(minYear),
                 validators.quarterIsValid()
             ]
         ),
@@ -369,7 +372,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(2019),
+                validators.dateYearIsLargerThan(minYear),
                 validators.quarterIsValid()
             ]
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
index 806b4251a..5ca6134ea 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
@@ -5,6 +5,9 @@
 from ...transforms import calendar_quarter_to_rpt_month_year
 from ... import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument
+import datetime
+
+minYear = datetime.date.today().year - 5;
 
 schemas = []
 
@@ -47,7 +50,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(2019),
+                        validators.dateYearIsLargerThan(minYear),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
index f0e5f5621..4dff7dbfe 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
@@ -6,7 +6,9 @@
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T6DataSubmissionDocument
+import datetime
 
+minYear = datetime.date.today().year - 5;
 
 s1 = RowSchema(
     document=TANF_T6DataSubmissionDocument(),
@@ -54,7 +56,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(2019),
+                validators.dateYearIsLargerThan(minYear),
                 validators.quarterIsValid(),
             ],
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
index 81e0f70e5..bfc41da44 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
@@ -5,7 +5,9 @@
 from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T7DataSubmissionDocument
+import datetime
 
+minYear = datetime.date.today().year - 5;
 schemas = []
 
 validator_index = 7
@@ -47,7 +49,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(2019),
+                        validators.dateYearIsLargerThan(minYear),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
index 41980bb9b..a226c15fe 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
@@ -6,7 +6,9 @@
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T6DataSubmissionDocument
+import datetime
 
+minYear = datetime.date.today().year - 5;
 
 s1 = RowSchema(
     document=Tribal_TANF_T6DataSubmissionDocument(),
@@ -42,7 +44,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(2019),
+                validators.dateYearIsLargerThan(minYear),
                 validators.quarterIsValid(),
             ],
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
index a17820514..8bd815e83 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
@@ -5,7 +5,9 @@
 from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T7DataSubmissionDocument
+import datetime
 
+minYear = datetime.date.today().year - 5;
 schemas = []
 
 validator_index = 7
@@ -47,7 +49,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(2019),
+                        validators.dateYearIsLargerThan(minYear),
                         validators.quarterIsValid(),
                     ],
                 ),

From 4545374ee3c820ca8044e4d4a7f8762e1d68f5b4 Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 20 Mar 2024 11:14:24 -0400
Subject: [PATCH 007/127] fixing lint

---
 tdrs-backend/tdpservice/parsers/fields.py          |  2 +-
 tdrs-backend/tdpservice/parsers/parse.py           |  2 +-
 .../tdpservice/parsers/schema_defs/ssp/m6.py       |  2 +-
 .../tdpservice/parsers/schema_defs/ssp/m7.py       |  2 +-
 .../tdpservice/parsers/schema_defs/tanf/t6.py      |  2 +-
 .../tdpservice/parsers/schema_defs/tanf/t7.py      |  2 +-
 .../parsers/schema_defs/tribal_tanf/t6.py          |  2 +-
 .../parsers/schema_defs/tribal_tanf/t7.py          |  2 +-
 tdrs-backend/tdpservice/parsers/test/test_parse.py |  4 ++--
 .../tdpservice/parsers/test/test_validators.py     | 14 ++++++--------
 10 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py
index b283f13d3..6aff02f26 100644
--- a/tdrs-backend/tdpservice/parsers/fields.py
+++ b/tdrs-backend/tdpservice/parsers/fields.py
@@ -81,4 +81,4 @@ def parse_value(self, line):
         """Parse and transform the value for a field given a line, startIndex, endIndex, and field type."""
         value = super().parse_value(line)
         # wrap in try catch return none to avoid db write
-        return self.transform_func(value, **self.kwargs) 
+        return self.transform_func(value, **self.kwargs)
diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py
index 1c71dc2c7..eb32370b8 100644
--- a/tdrs-backend/tdpservice/parsers/parse.py
+++ b/tdrs-backend/tdpservice/parsers/parse.py
@@ -256,7 +256,7 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
                     record.datafile = datafile
                     unsaved_records.setdefault(s.document, []).append(record)
 
-        except ValueError as ex:        
+        except ValueError as ex:
             err_obj = generate_error(
                 schema=None,
                 error_category=ParserErrorCategoryChoices.PRE_CHECK,
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
index 21d11d79a..0375f60de 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -8,7 +8,7 @@
 from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument
 import datetime
 
-minYear = datetime.date.today().year - 5;
+minYear = datetime.date.today().year - 5
 
 s1 = RowSchema(
     document=SSP_M6DataSubmissionDocument(),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
index 5ca6134ea..7ee2aa8f3 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
@@ -7,7 +7,7 @@
 from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument
 import datetime
 
-minYear = datetime.date.today().year - 5;
+minYear = datetime.date.today().year - 5
 
 schemas = []
 
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
index 4dff7dbfe..06c552258 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
@@ -8,7 +8,7 @@
 from tdpservice.search_indexes.documents.tanf import TANF_T6DataSubmissionDocument
 import datetime
 
-minYear = datetime.date.today().year - 5;
+minYear = datetime.date.today().year - 5
 
 s1 = RowSchema(
     document=TANF_T6DataSubmissionDocument(),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
index bfc41da44..83e834720 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
@@ -7,7 +7,7 @@
 from tdpservice.search_indexes.documents.tanf import TANF_T7DataSubmissionDocument
 import datetime
 
-minYear = datetime.date.today().year - 5;
+minYear = datetime.date.today().year - 5
 schemas = []
 
 validator_index = 7
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
index a226c15fe..57d38cc49 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
@@ -8,7 +8,7 @@
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T6DataSubmissionDocument
 import datetime
 
-minYear = datetime.date.today().year - 5;
+minYear = datetime.date.today().year - 5
 
 s1 = RowSchema(
     document=Tribal_TANF_T6DataSubmissionDocument(),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
index 8bd815e83..d2c1473cf 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
@@ -7,7 +7,7 @@
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T7DataSubmissionDocument
 import datetime
 
-minYear = datetime.date.today().year - 5;
+minYear = datetime.date.today().year - 5
 schemas = []
 
 validator_index = 7
diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 66352930d..16ce42e0f 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1497,14 +1497,14 @@ def tribal_section_4_bad_quarter(stt_user, stt):
 
 @pytest.mark.django_db()
 def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
-    """Test handling invalid quarter value that raises a ValueError exception"""
+    """Test handling invalid quarter value that raises a ValueError exception."""
     tribal_section_4_bad_quarter.year = 2020
     tribal_section_4_bad_quarter.quarter = 'Q1'
     dfs.datafile = tribal_section_4_bad_quarter
 
     parse.parse_datafile(tribal_section_4_bad_quarter)
     parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter)
-    
+
     assert parser_errors.count() == 1
     error = parser_errors.first()
 
diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py
index 27d071e6f..9f32ac78c 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_validators.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py
@@ -102,7 +102,7 @@ def test_validate__FAM_AFF__SSN():
     assert result == (True, None, ['FAMILY_AFFILIATION', 'CITIZENSHIP_STATUS', 'SSN'])
 
 @pytest.mark.parametrize(
-        "value, valid", 
+        "value, valid",
         [
             ("20201", True),
             ("20202", True),
@@ -458,24 +458,22 @@ def test_notEmpty_returns_nonexistent_substring():
     assert error == "111  333 contains blanks between positions 10 and 12."
 
 
-@pytest.mark.parametrize("test_input", [1,2,3,4])
+@pytest.mark.parametrize("test_input", [1, 2, 3, 4])
 def test_quarterIsValid_returns_true_if_valid(test_input):
-    """Test `quarterIsValid` gives a valid result for values 1-4"""
-
+    """Test `quarterIsValid` gives a valid result for values 1-4."""
     validator = validators.quarterIsValid()
     is_valid, error = validator(test_input)
-    
+
     assert is_valid is True
     assert error is None
 
 
 @pytest.mark.parametrize("test_input", [" ", 0, 5, "A"])
 def test_quarterIsValid_returns_false_if_invalid(test_input):
-    """Test `quarterIsValid` gives an invalid result for values not 1-4"""
-
+    """Test `quarterIsValid` gives an invalid result for values not 1-4."""
     validator = validators.quarterIsValid()
     is_valid, error = validator(test_input)
-    
+
     assert is_valid is False
     assert error == f"{test_input} is not a valid quarter."
 

From dffbe66d76f3efe011da689eb6992ca8c7185f39 Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 20 Mar 2024 16:20:00 -0400
Subject: [PATCH 008/127] reworked try/catch to a more appropriate area

---
 tdrs-backend/tdpservice/parsers/parse.py      | 53 +++++++------------
 .../tdpservice/parsers/test/test_parse.py     | 14 +++--
 tdrs-backend/tdpservice/parsers/transforms.py |  5 +-
 3 files changed, 32 insertions(+), 40 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py
index eb32370b8..9e7af9ce9 100644
--- a/tdrs-backend/tdpservice/parsers/parse.py
+++ b/tdrs-backend/tdpservice/parsers/parse.py
@@ -236,41 +236,24 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
 
         schema_manager = get_schema_manager(line, section, program_type)
 
-        try:
-            records = manager_parse_line(line, schema_manager, generate_error, is_encrypted)
-
-            record_number = 0
-            for i in range(len(records)):
-                r = records[i]
-                record_number += 1
-                record, record_is_valid, record_errors = r
-                if not record_is_valid:
-                    logger.debug(f"Record #{i} from line {line_number} is invalid.")
-                    line_errors = errors.get(f"{line_number}_{i}", {})
-                    line_errors.update({record_number: record_errors})
-                    errors.update({f"{line_number}_{i}": record_errors})
-                    unsaved_parser_errors.update({f"{line_number}_{i}": record_errors})
-                    num_errors += len(record_errors)
-                if record:
-                    s = schema_manager.schemas[i]
-                    record.datafile = datafile
-                    unsaved_records.setdefault(s.document, []).append(record)
-
-        except ValueError as ex:
-            err_obj = generate_error(
-                schema=None,
-                error_category=ParserErrorCategoryChoices.PRE_CHECK,
-                error_message=ex,
-                record=None,
-                field=None
-            )
-            num_errors += 1
-            parse_error = {line_number: [err_obj]}
-            unsaved_parser_errors.update(parse_error)
-            rollback_records(unsaved_records, datafile)
-            rollback_parser_errors(datafile)
-            bulk_create_errors(parse_error, num_errors, flush=True)
-            return errors
+        records = manager_parse_line(line, schema_manager, generate_error, is_encrypted)
+
+        record_number = 0
+        for i in range(len(records)):
+            r = records[i]
+            record_number += 1
+            record, record_is_valid, record_errors = r
+            if not record_is_valid:
+                logger.debug(f"Record #{i} from line {line_number} is invalid.")
+                line_errors = errors.get(f"{line_number}_{i}", {})
+                line_errors.update({record_number: record_errors})
+                errors.update({f"{line_number}_{i}": record_errors})
+                unsaved_parser_errors.update({f"{line_number}_{i}": record_errors})
+                num_errors += len(record_errors)
+            if record:
+                s = schema_manager.schemas[i]
+                record.datafile = datafile
+                unsaved_records.setdefault(s.document, []).append(record)
 
         all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count, datafile)
         unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors)
diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 16ce42e0f..2117a0aa2 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1505,8 +1505,14 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
     parse.parse_datafile(tribal_section_4_bad_quarter)
     parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter)
 
-    assert parser_errors.count() == 1
-    error = parser_errors.first()
+    assert parser_errors.count() == 36
+
+    for error in parser_errors:
+        assert error.error_type == ParserErrorCategoryChoices.FIELD_VALUE
 
-    assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK
-    assert error.error_message == "Invalid quarter value."
+        if (error.field_name == "RPT_MONTH_YEAR"):
+            assert error.error_message == "RPT_MONTH_YEAR is required but a value was not provided."
+        elif (error.field_name == "CALENDAR_QUARTER"):
+            assert error.error_message == "0 is not a valid quarter."
+        else:
+            assert False, f"Unexpected field name {error.field_name}"
diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py
index 75e4b2fe6..c4fd85367 100644
--- a/tdrs-backend/tdpservice/parsers/transforms.py
+++ b/tdrs-backend/tdpservice/parsers/transforms.py
@@ -8,7 +8,10 @@ def transform(value, **kwargs):
         value = str(value)
         year = value[:4]
         quarter = f"Q{value[-1]}"
-        month = transform_to_months(quarter)[month_index]
+        try:
+            month = transform_to_months(quarter)[month_index]
+        except ValueError:
+            return None
         return f"{year}{month_to_int(month)}"
     return transform
 

From 9c02d6ad072347e9dd8a47456f3441f1c5f692e8 Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 20 Mar 2024 16:23:46 -0400
Subject: [PATCH 009/127] remove comment

---
 tdrs-backend/tdpservice/parsers/fields.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tdrs-backend/tdpservice/parsers/fields.py b/tdrs-backend/tdpservice/parsers/fields.py
index 6aff02f26..076743096 100644
--- a/tdrs-backend/tdpservice/parsers/fields.py
+++ b/tdrs-backend/tdpservice/parsers/fields.py
@@ -80,5 +80,4 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex,
     def parse_value(self, line):
         """Parse and transform the value for a field given a line, startIndex, endIndex, and field type."""
         value = super().parse_value(line)
-        # wrap in try catch return none to avoid db write
         return self.transform_func(value, **self.kwargs)

From d1e7665cec907bb84a06ce111d953c6dbec16b87 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 25 Mar 2024 07:52:57 -0600
Subject: [PATCH 010/127] - committing intermediate code

---
 tdrs-frontend/.env                              | 2 +-
 tdrs-frontend/nginx/local/default.conf.template | 5 ++++-
 tdrs-frontend/nginx/local/locations.conf        | 3 ++-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env
index 11902ac81..0a6401a63 100644
--- a/tdrs-frontend/.env
+++ b/tdrs-frontend/.env
@@ -37,7 +37,7 @@ REACT_APP_DEBOUNCE_TIME=30000
 REACT_APP_EVENT_THROTTLE_TIME=60000
 
 # Enable the Kibana tab for dev purposes.
-# REACT_APP_DEV_KIBANA=true
+REACT_APP_DEV_KIBANA=true
 
 # Setup SCSS:
 # The following makes it possible to import SASS modules
diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template
index 8cf7b79ab..ae1a57d26 100644
--- a/tdrs-frontend/nginx/local/default.conf.template
+++ b/tdrs-frontend/nginx/local/default.conf.template
@@ -8,6 +8,8 @@ events {
 http {
     include mime.types;
 
+    resolver 127.0.0.11 ipv6=off valid=10s;
+
     server {
         listen 80;
         client_max_body_size 100m;
@@ -99,7 +101,8 @@ http {
                                '"http_x_forwarded_for": "$http_x_forwarded_for",'
                                '"host": "$host",'
                                '"$http_referer" "$http_user_agent" "cookies=$http_cookie;"  "server=$server_name" "http_host=$http_host"'
-                               '"HTTP_ORIGIN = $http_origin"';
+                               '"HTTP_ORIGIN = $http_origin"'
+                               ' Proxy Pass: "$proxy_host" "$upstream_addr"';
 
     access_log /dev/stdout compression;
     #access_log stderr compression;
diff --git a/tdrs-frontend/nginx/local/locations.conf b/tdrs-frontend/nginx/local/locations.conf
index f19cac64b..9dbd56c2c 100644
--- a/tdrs-frontend/nginx/local/locations.conf
+++ b/tdrs-frontend/nginx/local/locations.conf
@@ -25,7 +25,8 @@ location /kibana/ {
     auth_request /kibana_auth_check;
     auth_request_set $auth_status $upstream_status;
 
-    proxy_pass http://${KIBANA}:5601/;
+    set $kibana http://${KIBANA}:5601/;
+    proxy_pass $kibana;
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";

From a0b8181a635cc1da542c19ddeb759d606c2f9e0b Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Tue, 26 Mar 2024 15:58:47 -0600
Subject: [PATCH 011/127] - Add support for proxy passing a dynamic kibana
 hostname

---
 tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf | 2 +-
 tdrs-frontend/nginx/cloud.gov/locations.conf       | 5 +++--
 tdrs-frontend/nginx/local/default.conf.template    | 2 +-
 tdrs-frontend/nginx/local/locations.conf           | 4 ++--
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf
index 514010873..c059f73dc 100644
--- a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf
+++ b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf
@@ -34,7 +34,7 @@ http {
     limit_req_zone $binary_remote_addr zone=limitreqsbyaddr:20m rate=1000r/s;
     limit_req_status 444;
 
-    resolver {{nameservers}} valid=10s;
+    resolver {{nameservers}} valid=1s;
 
     server {
         root public;
diff --git a/tdrs-frontend/nginx/cloud.gov/locations.conf b/tdrs-frontend/nginx/cloud.gov/locations.conf
index b7cd5517f..152535f50 100644
--- a/tdrs-frontend/nginx/cloud.gov/locations.conf
+++ b/tdrs-frontend/nginx/cloud.gov/locations.conf
@@ -22,11 +22,12 @@ location ~ ^/(v1|admin|static/admin|swagger|redocs) {
         add_header Access-Control-Allow-Origin 's3-us-gov-west-1.amazonaws.com';
 }
 
-location /kibana/ {
+location ~* ^/kibana/(.*)$ {
     auth_request /kibana_auth_check;
     auth_request_set $auth_status $upstream_status;
 
-    proxy_pass http://{{env "KIBANA_BASE_URL"}}:5601/;
+    set $kibana http://{{env "KIBANA_BASE_URL"}}:5601/;
+    proxy_pass $kibana$1$is_args$args;
     proxy_pass_header x-csrftoken;
     proxy_set_header   Host $host;
     proxy_set_header   X-Real-IP         $remote_addr;
diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template
index ae1a57d26..360968015 100644
--- a/tdrs-frontend/nginx/local/default.conf.template
+++ b/tdrs-frontend/nginx/local/default.conf.template
@@ -8,7 +8,7 @@ events {
 http {
     include mime.types;
 
-    resolver 127.0.0.11 ipv6=off valid=10s;
+    resolver 127.0.0.11 ipv6=off valid=5s;
 
     server {
         listen 80;
diff --git a/tdrs-frontend/nginx/local/locations.conf b/tdrs-frontend/nginx/local/locations.conf
index 9dbd56c2c..69843a419 100644
--- a/tdrs-frontend/nginx/local/locations.conf
+++ b/tdrs-frontend/nginx/local/locations.conf
@@ -21,12 +21,12 @@ location ~ ^/(v1|admin|static/admin|static/drf-yasg|swagger|redocs) {
         proxy_pass_header x-csrftoken;
 }
 
-location /kibana/ {
+location ~* ^/kibana/(.*)$ {
     auth_request /kibana_auth_check;
     auth_request_set $auth_status $upstream_status;
 
     set $kibana http://${KIBANA}:5601/;
-    proxy_pass $kibana;
+    proxy_pass $kibana$1$is_args$args;
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";

From 1a67b85079a4638dd59e888b8c12c498d123b04e Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Tue, 26 Mar 2024 16:44:45 -0600
Subject: [PATCH 012/127] - Update clamav nginx

---
 tdrs-backend/clamav-router/nginx.conf              | 11 ++++++-----
 tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf
index bec070813..6350ad4d9 100644
--- a/tdrs-backend/clamav-router/nginx.conf
+++ b/tdrs-backend/clamav-router/nginx.conf
@@ -3,13 +3,14 @@ events {
 }
 # This opens a route to clamav prod
 http{ 
-    resolver {{nameservers}} valid=10s;
+    resolver {{nameservers}} valid=5s;
+    set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan
     server {
         client_max_body_size 100m;
         listen {{port}};
         client_max_body_size 100m;
-        location /scan {
-            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+        location ~* ^/scan/(.*)$ {
+            proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
         }
     }
@@ -17,8 +18,8 @@ http{
         client_max_body_size 100m;
         listen 9000;
         client_max_body_size 100m;
-        location /scan {
-            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+        location ~* ^/scan/(.*)$ {
+            proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
         }
     }
diff --git a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf
index c059f73dc..5898a8a5c 100644
--- a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf
+++ b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf
@@ -34,7 +34,7 @@ http {
     limit_req_zone $binary_remote_addr zone=limitreqsbyaddr:20m rate=1000r/s;
     limit_req_status 444;
 
-    resolver {{nameservers}} valid=1s;
+    resolver {{nameservers}} valid=5s;
 
     server {
         root public;

From 524086c2ab6dd5f90dc7c9cabf27e29fa9b3889a Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Tue, 26 Mar 2024 16:45:23 -0600
Subject: [PATCH 013/127] - comment dev var

---
 tdrs-frontend/.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env
index 0a6401a63..11902ac81 100644
--- a/tdrs-frontend/.env
+++ b/tdrs-frontend/.env
@@ -37,7 +37,7 @@ REACT_APP_DEBOUNCE_TIME=30000
 REACT_APP_EVENT_THROTTLE_TIME=60000
 
 # Enable the Kibana tab for dev purposes.
-REACT_APP_DEV_KIBANA=true
+# REACT_APP_DEV_KIBANA=true
 
 # Setup SCSS:
 # The following makes it possible to import SASS modules

From a11c653bc6d8e9d43915f08dce565753c5fa2ef3 Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 27 Mar 2024 10:10:21 -0400
Subject: [PATCH 014/127] remove relative year calc

---
 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
index 0375f60de..94d77673f 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -6,9 +6,6 @@
 from ...row_schema import RowSchema, SchemaManager
 from ... import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument
-import datetime
-
-minYear = datetime.date.today().year - 5
 
 s1 = RowSchema(
     document=SSP_M6DataSubmissionDocument(),
@@ -50,7 +47,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(minYear),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),
@@ -211,7 +208,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(minYear),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),
@@ -372,7 +369,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(minYear),
+                validators.dateYearIsLargerThan(2019),
                 validators.quarterIsValid()
             ]
         ),

From bf7f97d240126a6244f4f41c3e411a9bf212aeca Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 08:59:04 -0600
Subject: [PATCH 015/127] - committing intermediate result to test kibana with
 oss elastic

---
 tdrs-backend/docker-compose.yml          |   5 +-
 tdrs-backend/elastic_setup/Dockerfile    |  10 -
 tdrs-backend/elastic_setup/entrypoint.sh | 110 -----------
 tdrs-backend/elastic_setup/util.sh       | 240 -----------------------
 tdrs-frontend/.env                       |   2 +-
 5 files changed, 3 insertions(+), 364 deletions(-)
 delete mode 100644 tdrs-backend/elastic_setup/Dockerfile
 delete mode 100644 tdrs-backend/elastic_setup/entrypoint.sh
 delete mode 100644 tdrs-backend/elastic_setup/util.sh

diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
index e6636e14d..e29c5b8dd 100644
--- a/tdrs-backend/docker-compose.yml
+++ b/tdrs-backend/docker-compose.yml
@@ -46,7 +46,7 @@ services:
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:7.4.2
+    image: kibana:7.10.2
     ports:
       - 5601:5601
     environment:
@@ -59,11 +59,10 @@ services:
       - elastic
 
   elastic:
-    image: elasticsearch:7.17.6
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
     environment:
       - discovery.type=single-node
       - logger.discovery.level=debug
-      - xpack.security.enabled=false
     ports:
       - 9200:9200
       - 9300:9300
diff --git a/tdrs-backend/elastic_setup/Dockerfile b/tdrs-backend/elastic_setup/Dockerfile
deleted file mode 100644
index 32e6429f6..000000000
--- a/tdrs-backend/elastic_setup/Dockerfile
+++ /dev/null
@@ -1,10 +0,0 @@
-ARG ELASTIC_VERSION
-
-FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
-
-COPY . /
-
-RUN ["chmod", "+x", "/entrypoint.sh"]
-RUN ["chmod", "+x", "/util.sh"]
-
-ENTRYPOINT ["/entrypoint.sh"]
diff --git a/tdrs-backend/elastic_setup/entrypoint.sh b/tdrs-backend/elastic_setup/entrypoint.sh
deleted file mode 100644
index 6073b0540..000000000
--- a/tdrs-backend/elastic_setup/entrypoint.sh
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-set -o pipefail
-
-source "${BASH_SOURCE[0]%/*}"/util.sh
-
-
-# --------------------------------------------------------
-# Users declarations
-
-declare -A users_passwords
-users_passwords=(
-	[kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}"
-	[ofa_admin]="${OFA_ADMIN_PASSWORD:-}"
-)
-
-declare -A users_roles
-users_roles=(
-	[kibana_system]='kibana_system'
-    [ofa_admin]='kibana_admin'
-)
-
-# --------------------------------------------------------
-# Roles declarations for custom roles
-
-declare -A roles_files
-roles_files=(
-
-)
-
-# --------------------------------------------------------
-
-
-log 'Waiting for availability of Elasticsearch. This can take several minutes.'
-
-declare -i exit_code=0
-wait_for_elasticsearch || exit_code=$?
-
-if ((exit_code)); then
-	case $exit_code in
-		6)
-			suberr 'Could not resolve host. Is Elasticsearch running?'
-			;;
-		7)
-			suberr 'Failed to connect to host. Is Elasticsearch healthy?'
-			;;
-		28)
-			suberr 'Timeout connecting to host. Is Elasticsearch healthy?'
-			;;
-		*)
-			suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}"
-			;;
-	esac
-
-	exit $exit_code
-fi
-
-sublog 'Elasticsearch is running'
-
-log 'Waiting for initialization of built-in users'
-
-wait_for_builtin_users || exit_code=$?
-
-if ((exit_code)); then
-	suberr 'Timed out waiting for condition'
-	exit $exit_code
-fi
-
-sublog 'Built-in users were initialized'
-
-for role in "${!roles_files[@]}"; do
-	log "Role '$role'"
-
-	declare body_file
-	body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}"
-	if [[ ! -f "${body_file:-}" ]]; then
-		sublog "No role body found at '${body_file}', skipping"
-		continue
-	fi
-
-	sublog 'Creating/updating'
-	ensure_role "$role" "$(<"${body_file}")"
-done
-
-for user in "${!users_passwords[@]}"; do
-	log "User '$user'"
-	if [[ -z "${users_passwords[$user]:-}" ]]; then
-		sublog 'No password defined, skipping'
-		continue
-	fi
-
-	declare -i user_exists=0
-	user_exists="$(check_user_exists "$user")"
-
-	if ((user_exists)); then
-		sublog 'User exists, setting password'
-		set_user_password "$user" "${users_passwords[$user]}"
-	else
-		if [[ -z "${users_roles[$user]:-}" ]]; then
-			suberr '  No role defined, skipping creation'
-			continue
-		fi
-
-		sublog 'User does not exist, creating'
-		create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}"
-	fi
-done
-
-log "Elastic setup completed. Exiting with code: $?"
diff --git a/tdrs-backend/elastic_setup/util.sh b/tdrs-backend/elastic_setup/util.sh
deleted file mode 100644
index 045110249..000000000
--- a/tdrs-backend/elastic_setup/util.sh
+++ /dev/null
@@ -1,240 +0,0 @@
-#!/usr/bin/env bash
-
-# Log a message.
-function log {
-	echo "[+] $1"
-}
-
-# Log a message at a sub-level.
-function sublog {
-	echo "   ⠿ $1"
-}
-
-# Log an error.
-function err {
-	echo "[x] $1" >&2
-}
-
-# Log an error at a sub-level.
-function suberr {
-	echo "   ⠍ $1" >&2
-}
-
-# Poll the 'elasticsearch' service until it responds with HTTP code 200.
-function wait_for_elasticsearch {
-	local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}"
-
-	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" )
-
-	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
-		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
-	fi
-
-	local -i result=1
-	local output
-
-	# retry for max 300s (60*5s)
-	for _ in $(seq 1 60); do
-		local -i exit_code=0
-		output="$(curl "${args[@]}")" || exit_code=$?
-
-		if ((exit_code)); then
-			result=$exit_code
-		fi
-
-		if [[ "${output: -3}" -eq 200 ]]; then
-			result=0
-			break
-		fi
-
-		sleep 5
-	done
-
-	if ((result)) && [[ "${output: -3}" -ne 000 ]]; then
-		echo -e "\n${output::-3}"
-	fi
-
-	return $result
-}
-
-# Poll the Elasticsearch users API until it returns users.
-function wait_for_builtin_users {
-	local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}"
-
-	local -a args=( '-s' '-D-' '-m15' "http://${elasticsearch_host}:9200/_security/user?pretty" )
-
-	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
-		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
-	fi
-
-	local -i result=1
-
-	local line
-	local -i exit_code
-	local -i num_users
-
-	# retry for max 30s (30*1s)
-	for _ in $(seq 1 30); do
-		num_users=0
-
-		# read exits with a non-zero code if the last read input doesn't end
-		# with a newline character. The printf without newline that follows the
-		# curl command ensures that the final input not only contains curl's
-		# exit code, but causes read to fail so we can capture the return value.
-		# Ref. https://unix.stackexchange.com/a/176703/152409
-		while IFS= read -r line || ! exit_code="$line"; do
-			if [[ "$line" =~ _reserved.+true ]]; then
-				(( num_users++ ))
-			fi
-		done < <(curl "${args[@]}"; printf '%s' "$?")
-
-		if ((exit_code)); then
-			result=$exit_code
-		fi
-
-		# we expect more than just the 'elastic' user in the result
-		if (( num_users > 1 )); then
-			result=0
-			break
-		fi
-
-		sleep 1
-	done
-
-	return $result
-}
-
-# Verify that the given Elasticsearch user exists.
-function check_user_exists {
-	local username=$1
-
-	local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}"
-
-	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
-		"http://${elasticsearch_host}:9200/_security/user/${username}"
-		)
-
-	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
-		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
-	fi
-
-	local -i result=1
-	local -i exists=0
-	local output
-
-	output="$(curl "${args[@]}")"
-	if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then
-		result=0
-	fi
-	if [[ "${output: -3}" -eq 200 ]]; then
-		exists=1
-	fi
-
-	if ((result)); then
-		echo -e "\n${output::-3}"
-	else
-		echo "$exists"
-	fi
-
-	return $result
-}
-
-# Set password of a given Elasticsearch user.
-function set_user_password {
-	local username=$1
-	local password=$2
-
-	local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}"
-
-	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
-		"http://${elasticsearch_host}:9200/_security/user/${username}/_password"
-		'-X' 'POST'
-		'-H' 'Content-Type: application/json'
-		'-d' "{\"password\" : \"${password}\"}"
-		)
-
-	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
-		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
-	fi
-
-	local -i result=1
-	local output
-
-	output="$(curl "${args[@]}")"
-	if [[ "${output: -3}" -eq 200 ]]; then
-		result=0
-	fi
-
-	if ((result)); then
-		echo -e "\n${output::-3}\n"
-	fi
-
-	return $result
-}
-
-# Create the given Elasticsearch user.
-function create_user {
-	local username=$1
-	local password=$2
-	local role=$3
-
-	local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}"
-
-	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
-		"http://${elasticsearch_host}:9200/_security/user/${username}"
-		'-X' 'POST'
-		'-H' 'Content-Type: application/json'
-		'-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}"
-		)
-
-	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
-		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
-	fi
-
-	local -i result=1
-	local output
-
-	output="$(curl "${args[@]}")"
-	if [[ "${output: -3}" -eq 200 ]]; then
-		result=0
-	fi
-
-	if ((result)); then
-		echo -e "\n${output::-3}\n"
-	fi
-
-	return $result
-}
-
-# Ensure that the given Elasticsearch role is up-to-date, create it if required.
-function ensure_role {
-	local name=$1
-	local body=$2
-
-	local elasticsearch_host="${ELASTICSEARCH_HOST:-elastic}"
-
-	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
-		"http://${elasticsearch_host}:9200/_security/role/${name}"
-		'-X' 'POST'
-		'-H' 'Content-Type: application/json'
-		'-d' "$body"
-		)
-
-	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
-		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
-	fi
-
-	local -i result=1
-	local output
-
-	output="$(curl "${args[@]}")"
-	if [[ "${output: -3}" -eq 200 ]]; then
-		result=0
-	fi
-
-	if ((result)); then
-		echo -e "\n${output::-3}\n"
-	fi
-
-	return $result
-}
\ No newline at end of file
diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env
index 11902ac81..0a6401a63 100644
--- a/tdrs-frontend/.env
+++ b/tdrs-frontend/.env
@@ -37,7 +37,7 @@ REACT_APP_DEBOUNCE_TIME=30000
 REACT_APP_EVENT_THROTTLE_TIME=60000
 
 # Enable the Kibana tab for dev purposes.
-# REACT_APP_DEV_KIBANA=true
+REACT_APP_DEV_KIBANA=true
 
 # Setup SCSS:
 # The following makes it possible to import SASS modules

From aa0b7ed29025ef40d2297a7b82d2a60b2ccc7763 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 09:23:10 -0600
Subject: [PATCH 016/127] - use correct container

---
 tdrs-backend/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
index e29c5b8dd..f2b58d85b 100644
--- a/tdrs-backend/docker-compose.yml
+++ b/tdrs-backend/docker-compose.yml
@@ -46,7 +46,7 @@ services:
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
   kibana:
-    image: kibana:7.10.2
+    image: kibana:7.10.1
     ports:
       - 5601:5601
     environment:

From 62568e0e89b53b995a7b971fbb2165c7cc46fe39 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 13:18:54 -0600
Subject: [PATCH 017/127] - fix conf errors

---
 tdrs-backend/clamav-router/nginx.conf | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf
index 6350ad4d9..10bf8cdfb 100644
--- a/tdrs-backend/clamav-router/nginx.conf
+++ b/tdrs-backend/clamav-router/nginx.conf
@@ -4,21 +4,22 @@ events {
 # This opens a route to clamav prod
 http{ 
     resolver {{nameservers}} valid=5s;
-    set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan
+
     server {
-        client_max_body_size 100m;
         listen {{port}};
         client_max_body_size 100m;
         location ~* ^/scan/(.*)$ {
+            set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
         }
     }
+
     server {
-        client_max_body_size 100m;
         listen 9000;
         client_max_body_size 100m;
         location ~* ^/scan/(.*)$ {
+            set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
         }

From b7f60aef0eb07f412a62902553b318499e155b17 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 13:36:44 -0600
Subject: [PATCH 018/127] - update scan url

---
 scripts/deploy-backend.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh
index 0dc5ba5b4..884f3eb63 100755
--- a/scripts/deploy-backend.sh
+++ b/scripts/deploy-backend.sh
@@ -131,10 +131,10 @@ update_backend()
     cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL"
     
     if [ "$CF_SPACE" = "tanf-prod" ]; then
-      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan"
+      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan/"
     else
       # Add environment varilables for clamav
-      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan"
+      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan/"
     fi
 
     if [ "$1" = "rolling" ] ; then

From a1aea3cb61cd2f2608ce2a7df267db7e8e370468 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 13:37:54 -0600
Subject: [PATCH 019/127] - Add logging for clamav proxy

---
 tdrs-backend/clamav-router/nginx.conf | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf
index 10bf8cdfb..6da18925b 100644
--- a/tdrs-backend/clamav-router/nginx.conf
+++ b/tdrs-backend/clamav-router/nginx.conf
@@ -5,6 +5,19 @@ events {
 http{ 
     resolver {{nameservers}} valid=5s;
 
+    log_format compression '$remote_addr - $remote_user [$time_local] '
+                            '"proxy_host and upstream_addr": $proxy_host $upstream_addr, '
+                           ' "request": $request, '
+                           '"body_bytes_sent" : $body_bytes_sent, '
+                           '"request_body": $request_body, '
+                           '"http_x_forwarded_for": $http_x_forwarded_for, '
+                            '"host": $host, '
+                            ' "status": $status, '
+                            '"proxy_add_x_forwarded_for": $proxy_add_x_forwarded_for, '
+                           '"http_referer": $http_referer, '
+                           '"http_user_agent": $http_user_agent, '
+                           '"cookies=$http_cookie;"  "server=$server_name" "http_host=$http_host"';
+
     server {
         listen {{port}};
         client_max_body_size 100m;

From f847a02c9b04594e7981fde6b27559837b3a6080 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 13:50:24 -0600
Subject: [PATCH 020/127] - add logging - update conf to allow logging

---
 docs/Technical-Documentation/clamav.md | 2 +-
 tdrs-backend/clamav-router/nginx.conf  | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/Technical-Documentation/clamav.md b/docs/Technical-Documentation/clamav.md
index 894853348..a8c37aaa8 100644
--- a/docs/Technical-Documentation/clamav.md
+++ b/docs/Technical-Documentation/clamav.md
@@ -23,7 +23,7 @@ The instance name then will be set as an environment variable to redirect each i
 First, set the environment variable __AV_SCAN_URL__ as follows:
 ```
 Environment variable name: AV_SCAN_URL
-Environment variable value: http://{nginx_instance}.apps.internal:9000/scan
+Environment variable value: http://{nginx_instance}.apps.internal:9000/scan/
 ```
 
 ### Add network policy from _{tdp-clamav-nginx}_ to clamav in prod
diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf
index 6da18925b..d0ce5ae0d 100644
--- a/tdrs-backend/clamav-router/nginx.conf
+++ b/tdrs-backend/clamav-router/nginx.conf
@@ -2,7 +2,11 @@ events {
     worker_connections 1024;
 }
 # This opens a route to clamav prod
-http{ 
+http{
+    charset utf-8;
+    log_format cloudfoundry 'NginxLog "$request" $status $body_bytes_sent';
+    access_log /dev/stdout cloudfoundry;
+
     resolver {{nameservers}} valid=5s;
 
     log_format compression '$remote_addr - $remote_user [$time_local] '

From 6fbbc2417b0201fb5b6808f853aac7ef2729d910 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 14:21:46 -0600
Subject: [PATCH 021/127] - use correct container

---
 tdrs-backend/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
index f2b58d85b..c6172a348 100644
--- a/tdrs-backend/docker-compose.yml
+++ b/tdrs-backend/docker-compose.yml
@@ -46,7 +46,7 @@ services:
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
   kibana:
-    image: kibana:7.10.1
+    image: docker.elastic.co/kibana/kibana-oss:7.10.2
     ports:
       - 5601:5601
     environment:

From c0657076fbc76f14268d4b7538f8b3a845b73896 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 14:41:01 -0600
Subject: [PATCH 022/127] - Add mixin class/file to export admin data as csv

---
 .../tdpservice/search_indexes/admin/mixins.py | 24 +++++++++++++++++++
 .../tdpservice/search_indexes/admin/tanf.py   |  5 +++-
 2 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 tdrs-backend/tdpservice/search_indexes/admin/mixins.py

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
new file mode 100644
index 000000000..df36300d8
--- /dev/null
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -0,0 +1,24 @@
+"""Mixin classes supproting custom functionality."""
+
+from django.http import HttpResponse
+import csv
+
+class ExportCsvMixin:
+    """Mixin class to support CSV exporting."""
+
+    def export_as_csv(self, request, queryset):
+        """Convert queryset to CSV."""
+        meta = self.model._meta
+        field_names = [field.name for field in meta.fields]
+
+        response = HttpResponse(content_type='text/csv')
+        response['Content-Disposition'] = 'attachment; filename={}.csv'.format(meta)
+        writer = csv.writer(response)
+
+        writer.writerow(field_names)
+        for obj in queryset:
+            writer.writerow([getattr(obj, field) for field in field_names])
+
+        return response
+
+    export_as_csv.short_description = "Export Selected as CSV"
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index af3429695..9a032909d 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,11 +1,14 @@
 """ModelAdmin classes for parsed TANF data files."""
 from django.contrib import admin
 from .filter import CreationDateFilter
+from .mixins import ExportCsvMixin
 
 
-class TANF_T1Admin(admin.ModelAdmin):
+class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin):
     """ModelAdmin class for parsed T1 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',

From 7f515684707b38d7f78a5baff1d49a6106dcf991 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 29 Mar 2024 15:42:48 -0600
Subject: [PATCH 023/127] - Add stt filter - Rename file to filters.py

---
 .../admin/{filter.py => filters.py}           | 37 +++++++++++++++++++
 .../tdpservice/search_indexes/admin/mixins.py |  8 ++++
 .../tdpservice/search_indexes/admin/ssp.py    |  2 +-
 .../tdpservice/search_indexes/admin/tanf.py   |  8 ++--
 .../tdpservice/search_indexes/admin/tribal.py |  2 +-
 5 files changed, 52 insertions(+), 5 deletions(-)
 rename tdrs-backend/tdpservice/search_indexes/admin/{filter.py => filters.py} (50%)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filter.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
similarity index 50%
rename from tdrs-backend/tdpservice/search_indexes/admin/filter.py
rename to tdrs-backend/tdpservice/search_indexes/admin/filters.py
index c3f46a6da..35e166c43 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filter.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -2,6 +2,8 @@
 from django.utils.translation import ugettext_lazy as _
 from django.contrib.admin import SimpleListFilter
 
+from tdpservice.stts.models import STT
+
 class CreationDateFilter(SimpleListFilter):
     """Simple filter class to show newest created datafile records."""
 
@@ -33,3 +35,38 @@ def queryset(self, request, queryset):
             datafile = queryset.order_by("-datafile__id").first().datafile
             return queryset.filter(datafile=datafile)
         return queryset
+
+
+class STTFilter(SimpleListFilter):
+    """Simple filter class to show records based on stt."""
+
+    title = _('STT Code')
+
+    parameter_name = 'stt_code'
+
+    def lookups(self, request, model_admin):
+        """Available options in dropdown."""
+        options = [(None, _('All'))]
+        for obj in STT.objects.all():
+            options.append((obj.stt_code, _(obj.name)))
+        return options
+
+    def choices(self, cl):
+        """Update query string based on selection."""
+        for lookup, title in self.lookup_choices:
+            yield {
+                'selected': self.value() == lookup,
+                'query_string': cl.get_query_string({
+                    self.parameter_name: lookup,
+                }, []),
+                'display': title,
+            }
+
+    def queryset(self, request, queryset):
+        """Sort queryset to show latest records."""
+        val = self.value()
+        if val is not None and queryset.exists():
+            if len(val) == 1:
+                val = f"0{val}"
+            queryset = queryset.filter(datafile__stt__stt_code=val)
+        return queryset
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index df36300d8..20d0448cc 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -22,3 +22,11 @@ def export_as_csv(self, request, queryset):
         return response
 
     export_as_csv.short_description = "Export Selected as CSV"
+
+
+class SttCodeMixin:
+    """Mixin class to display a record's associated stt code."""
+
+    def stt_code(self, obj):
+        """Return stt code."""
+        return obj.datafile.stt.stt_code
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index 8c590e14e..865d2e4ff 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -1,6 +1,6 @@
 """ModelAdmin classes for parsed SSP data files."""
 from django.contrib import admin
-from .filter import CreationDateFilter
+from .filters import CreationDateFilter
 
 
 class SSP_M1Admin(admin.ModelAdmin):
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 9a032909d..8bb0f176a 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,10 +1,10 @@
 """ModelAdmin classes for parsed TANF data files."""
 from django.contrib import admin
-from .filter import CreationDateFilter
-from .mixins import ExportCsvMixin
+from .filters import CreationDateFilter, STTFilter
+from .mixins import ExportCsvMixin, SttCodeMixin
 
 
-class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin):
+class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T1 data files."""
 
     actions = ["export_as_csv"]
@@ -17,9 +17,11 @@ class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin):
         'ZIP_CODE',
         'STRATUM',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
+        STTFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
         'ZIP_CODE',
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index 757620a24..d7d6b3241 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -1,6 +1,6 @@
 """ModelAdmin classes for parsed TRIBAL data files."""
 from django.contrib import admin
-from .filter import CreationDateFilter
+from .filters import CreationDateFilter
 
 
 class Tribal_TANF_T1Admin(admin.ModelAdmin):

From 94dd95a0e176bacefb84c3e2b9c17c7abfc7a694 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 1 Apr 2024 09:12:19 -0600
Subject: [PATCH 024/127] - Add fiscal year filter

---
 .../search_indexes/admin/filters.py           | 47 ++++++++++++++++++-
 .../tdpservice/search_indexes/admin/mixins.py |  2 +-
 .../tdpservice/search_indexes/admin/tanf.py   |  3 +-
 3 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index 35e166c43..3590ae3c2 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -1,8 +1,8 @@
 """Filter classes."""
 from django.utils.translation import ugettext_lazy as _
 from django.contrib.admin import SimpleListFilter
-
 from tdpservice.stts.models import STT
+import datetime
 
 class CreationDateFilter(SimpleListFilter):
     """Simple filter class to show newest created datafile records."""
@@ -63,10 +63,53 @@ def choices(self, cl):
             }
 
     def queryset(self, request, queryset):
-        """Sort queryset to show latest records."""
+        """Return queryset of records based on stt code."""
         val = self.value()
         if val is not None and queryset.exists():
             if len(val) == 1:
                 val = f"0{val}"
             queryset = queryset.filter(datafile__stt__stt_code=val)
         return queryset
+
+class FiscalYearFilter(SimpleListFilter):
+    """Simple filter class to filter records based on datafile fiscal year."""
+
+    title = _('Fiscal Year')
+
+    parameter_name = 'fiscal_year'
+
+    def lookups(self, request, model_admin):
+        """Available options in dropdown."""
+        current_year = datetime.date.today().year
+        quarters = [1, 2, 3, 4]
+        months = ["(Oct - Dec)", "(Jan - Mar)", "(Apr - Jun)", "(Jul - Sep)"]
+        years = [year for year in range(current_year - 5, current_year + 1)]
+        options = [(None, _('All'))]
+
+        for year in years:
+            for qtr, month in zip(quarters, months):
+                query = f"{year}Q{qtr}"
+                display = f"{year} - Q{qtr} {month}"
+                options.append((query, display))
+
+        return options
+
+    def choices(self, cl):
+        """Update query string based on selection."""
+        for lookup, title in self.lookup_choices:
+            yield {
+                'selected': self.value() == lookup,
+                'query_string': cl.get_query_string({
+                    self.parameter_name: lookup,
+                }, []),
+                'display': title,
+            }
+
+    def queryset(self, request, queryset):
+        """Filter queryset to show records matching selected fiscal year."""
+        if self.value() is not None and queryset.exists():
+            year = int(self.value()[0:4])
+            quarter = self.value()[4:6]
+            queryset = queryset.filter(datafile__quarter=quarter).filter(datafile__year=year)
+
+        return queryset
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index 20d0448cc..2dbfd9981 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -29,4 +29,4 @@ class SttCodeMixin:
 
     def stt_code(self, obj):
         """Return stt code."""
-        return obj.datafile.stt.stt_code
\ No newline at end of file
+        return obj.datafile.stt.stt_code
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 8bb0f176a..9cb066690 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,6 +1,6 @@
 """ModelAdmin classes for parsed TANF data files."""
 from django.contrib import admin
-from .filters import CreationDateFilter, STTFilter
+from .filters import CreationDateFilter, FiscalYearFilter, STTFilter
 from .mixins import ExportCsvMixin, SttCodeMixin
 
 
@@ -22,6 +22,7 @@ class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
 
     list_filter = [
         STTFilter,
+        FiscalYearFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
         'ZIP_CODE',

From 6c2c1aa7a9d42d0d508aa119e7bca2d5c222fa0a Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 1 Apr 2024 09:17:59 -0600
Subject: [PATCH 025/127] - Add filters to models - Update filter name

---
 .../tdpservice/search_indexes/admin/filters.py   |  6 +++---
 .../tdpservice/search_indexes/admin/ssp.py       | 16 +++++++++++++++-
 .../tdpservice/search_indexes/admin/tanf.py      | 16 ++++++++++++++--
 .../tdpservice/search_indexes/admin/tribal.py    | 16 +++++++++++++++-
 4 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index 3590ae3c2..c9f1762e4 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -71,12 +71,12 @@ def queryset(self, request, queryset):
             queryset = queryset.filter(datafile__stt__stt_code=val)
         return queryset
 
-class FiscalYearFilter(SimpleListFilter):
+class FiscalPeriodFilter(SimpleListFilter):
     """Simple filter class to filter records based on datafile fiscal year."""
 
-    title = _('Fiscal Year')
+    title = _('Fiscal Period')
 
-    parameter_name = 'fiscal_year'
+    parameter_name = 'fiscal_period'
 
     def lookups(self, request, model_admin):
         """Available options in dropdown."""
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index 865d2e4ff..dd5c44d1b 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -1,6 +1,6 @@
 """ModelAdmin classes for parsed SSP data files."""
 from django.contrib import admin
-from .filters import CreationDateFilter
+from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
 
 
 class SSP_M1Admin(admin.ModelAdmin):
@@ -17,6 +17,8 @@ class SSP_M1Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
         'ZIP_CODE',
@@ -35,6 +37,8 @@ class SSP_M2Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -51,6 +55,8 @@ class SSP_M3Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -66,6 +72,8 @@ class SSP_M4Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -81,6 +89,8 @@ class SSP_M5Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -97,6 +107,8 @@ class SSP_M6Admin(admin.ModelAdmin):
 
     list_filter = [
         'CALENDAR_QUARTER',
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR'
     ]
@@ -116,6 +128,8 @@ class SSP_M7Admin(admin.ModelAdmin):
 
     list_filter = [
         'CALENDAR_QUARTER',
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 9cb066690..5e2552e7e 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,6 +1,6 @@
 """ModelAdmin classes for parsed TANF data files."""
 from django.contrib import admin
-from .filters import CreationDateFilter, FiscalYearFilter, STTFilter
+from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
 from .mixins import ExportCsvMixin, SttCodeMixin
 
 
@@ -22,7 +22,7 @@ class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
 
     list_filter = [
         STTFilter,
-        FiscalYearFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
         'ZIP_CODE',
@@ -41,6 +41,8 @@ class TANF_T2Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -57,6 +59,8 @@ class TANF_T3Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -73,6 +77,8 @@ class TANF_T4Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -89,6 +95,8 @@ class TANF_T5Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -106,6 +114,8 @@ class TANF_T6Admin(admin.ModelAdmin):
 
     list_filter = [
         'CALENDAR_QUARTER',
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR'
     ]
@@ -126,6 +136,8 @@ class TANF_T7Admin(admin.ModelAdmin):
 
     list_filter = [
         'CALENDAR_QUARTER',
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index d7d6b3241..2e7f8865c 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -1,6 +1,6 @@
 """ModelAdmin classes for parsed TRIBAL data files."""
 from django.contrib import admin
-from .filters import CreationDateFilter
+from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
 
 
 class Tribal_TANF_T1Admin(admin.ModelAdmin):
@@ -17,6 +17,8 @@ class Tribal_TANF_T1Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
         'ZIP_CODE',
@@ -35,6 +37,8 @@ class Tribal_TANF_T2Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -51,6 +55,8 @@ class Tribal_TANF_T3Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -66,6 +72,8 @@ class Tribal_TANF_T4Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -80,6 +88,8 @@ class Tribal_TANF_T5Admin(admin.ModelAdmin):
     ]
 
     list_filter = [
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
@@ -96,6 +106,8 @@ class Tribal_TANF_T6Admin(admin.ModelAdmin):
 
     list_filter = [
         'CALENDAR_QUARTER',
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR'
     ]
@@ -115,6 +127,8 @@ class Tribal_TANF_T7Admin(admin.ModelAdmin):
 
     list_filter = [
         'CALENDAR_QUARTER',
+        STTFilter,
+        FiscalPeriodFilter,
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]

From 1092ced11fcae2bf8687601d48040e0edb358069 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 1 Apr 2024 11:36:53 -0600
Subject: [PATCH 026/127] - Convert to streaming interface to avoid timeouts

---
 .../tdpservice/search_indexes/admin/mixins.py | 24 +++++++++++++------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index 2dbfd9981..aeb15176b 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -1,8 +1,17 @@
 """Mixin classes supproting custom functionality."""
 
-from django.http import HttpResponse
+from django.http import StreamingHttpResponse
 import csv
 
+class Echo:
+    """An object that implements just the write method of the file-like
+    interface.
+    """
+
+    def write(self, value):
+        """Write the value by returning it, instead of storing in a buffer."""
+        return value
+
 class ExportCsvMixin:
     """Mixin class to support CSV exporting."""
 
@@ -11,15 +20,16 @@ def export_as_csv(self, request, queryset):
         meta = self.model._meta
         field_names = [field.name for field in meta.fields]
 
-        response = HttpResponse(content_type='text/csv')
-        response['Content-Disposition'] = 'attachment; filename={}.csv'.format(meta)
-        writer = csv.writer(response)
+        pseudo_buffer = Echo()
 
+        writer = csv.writer(pseudo_buffer)
         writer.writerow(field_names)
-        for obj in queryset:
-            writer.writerow([getattr(obj, field) for field in field_names])
 
-        return response
+        return StreamingHttpResponse(
+            (writer.writerow([getattr(obj, field) for field in field_names]) for obj in queryset),
+            content_type="text/csv",
+            headers={"Content-Disposition": f'attachment; filename="{meta}.csv"'},
+        )
 
     export_as_csv.short_description = "Export Selected as CSV"
 

From 7e9ec5b5bcf6bf3946fd537735f41596bb334a71 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 1 Apr 2024 11:48:19 -0600
Subject: [PATCH 027/127] - Move class to be internal

---
 .../tdpservice/search_indexes/admin/mixins.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index aeb15176b..bbcaf564f 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -3,24 +3,25 @@
 from django.http import StreamingHttpResponse
 import csv
 
-class Echo:
-    """An object that implements just the write method of the file-like
-    interface.
-    """
-
-    def write(self, value):
-        """Write the value by returning it, instead of storing in a buffer."""
-        return value
 
 class ExportCsvMixin:
     """Mixin class to support CSV exporting."""
 
+    class Echo:
+        """An object that implements just the write method of the file-like
+        interface.
+        """
+
+        def write(self, value):
+            """Write the value by returning it, instead of storing in a buffer."""
+            return value
+
     def export_as_csv(self, request, queryset):
         """Convert queryset to CSV."""
         meta = self.model._meta
         field_names = [field.name for field in meta.fields]
 
-        pseudo_buffer = Echo()
+        pseudo_buffer = ExportCsvMixin.Echo()
 
         writer = csv.writer(pseudo_buffer)
         writer.writerow(field_names)

From 90bfc6977c58a0837c11ca560fdf97cee52184be Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Tue, 2 Apr 2024 14:06:25 -0600
Subject: [PATCH 028/127] - Enable export for all record types - Add custom
 iterators to generate custom rows in the csv

---
 .../tdpservice/search_indexes/admin/mixins.py | 43 ++++++++++++++++---
 .../tdpservice/search_indexes/admin/ssp.py    | 36 +++++++++++++---
 .../tdpservice/search_indexes/admin/tanf.py   | 30 ++++++++++---
 .../tdpservice/search_indexes/admin/tribal.py | 36 +++++++++++++---
 4 files changed, 120 insertions(+), 25 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index bbcaf564f..d6cb42372 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -15,19 +15,52 @@ class Echo:
         def write(self, value):
             """Write the value by returning it, instead of storing in a buffer."""
             return value
+    
+    class RowIterator:
+        """Iterator class to support custom CSV row generation."""
+
+        def __init__(self, field_names, queryset):
+            self.field_names = field_names
+            self.queryset = queryset
+            self.writer = csv.writer(ExportCsvMixin.Echo())
+            self.is_header_row = True
+            self.header_row = self.__init_header_row(field_names)
+
+        def __init_header_row(self, field_names):
+            """Generate custom header row."""
+            header_row = []
+            for name in field_names:
+                header_row.append(name)
+                if name == "datafile":
+                    header_row.append("STT")
+            return header_row
+                    
+        
+        def __iter__(self):
+            """Yield the next row in the csv export.""" 
+            for obj in self.queryset:
+                row = []
+
+                if self.is_header_row:
+                    self.is_header_row = False
+                    yield self.writer.writerow(self.header_row)
+
+                for field_name in self.field_names:
+                    field = getattr(obj, field_name)
+                    row.append(field)
+                    if field_name == "datafile":
+                        row.append(field.stt.stt_code)
+                yield self.writer.writerow(row)
 
     def export_as_csv(self, request, queryset):
         """Convert queryset to CSV."""
         meta = self.model._meta
         field_names = [field.name for field in meta.fields]
 
-        pseudo_buffer = ExportCsvMixin.Echo()
-
-        writer = csv.writer(pseudo_buffer)
-        writer.writerow(field_names)
+        iterator = ExportCsvMixin.RowIterator(field_names, queryset)
 
         return StreamingHttpResponse(
-            (writer.writerow([getattr(obj, field) for field in field_names]) for obj in queryset),
+            iterator,
             content_type="text/csv",
             headers={"Content-Disposition": f'attachment; filename="{meta}.csv"'},
         )
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index dd5c44d1b..3685069b8 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -1,11 +1,14 @@
 """ModelAdmin classes for parsed SSP data files."""
 from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
+from .mixins import ExportCsvMixin, SttCodeMixin
 
 
-class SSP_M1Admin(admin.ModelAdmin):
+class SSP_M1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M1 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -14,6 +17,7 @@ class SSP_M1Admin(admin.ModelAdmin):
         'ZIP_CODE',
         'STRATUM',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -26,14 +30,17 @@ class SSP_M1Admin(admin.ModelAdmin):
     ]
 
 
-class SSP_M2Admin(admin.ModelAdmin):
+class SSP_M2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M2 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -44,14 +51,17 @@ class SSP_M2Admin(admin.ModelAdmin):
     ]
 
 
-class SSP_M3Admin(admin.ModelAdmin):
+class SSP_M3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -61,14 +71,17 @@ class SSP_M3Admin(admin.ModelAdmin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M4Admin(admin.ModelAdmin):
+class SSP_M4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -78,14 +91,17 @@ class SSP_M4Admin(admin.ModelAdmin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M5Admin(admin.ModelAdmin):
+class SSP_M5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -95,14 +111,17 @@ class SSP_M5Admin(admin.ModelAdmin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M6Admin(admin.ModelAdmin):
+class SSP_M6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M6 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
         'RPT_MONTH_YEAR',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -113,9 +132,11 @@ class SSP_M6Admin(admin.ModelAdmin):
         'RPT_MONTH_YEAR'
     ]
 
-class SSP_M7Admin(admin.ModelAdmin):
+class SSP_M7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M7 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
@@ -124,6 +145,7 @@ class SSP_M7Admin(admin.ModelAdmin):
         'STRATUM',
         'FAMILIES_MONTH',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 5e2552e7e..164a664fa 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -30,14 +30,17 @@ class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T2Admin(admin.ModelAdmin):
+class TANF_T2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T2 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -48,14 +51,17 @@ class TANF_T2Admin(admin.ModelAdmin):
     ]
 
 
-class TANF_T3Admin(admin.ModelAdmin):
+class TANF_T3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T3 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -66,14 +72,17 @@ class TANF_T3Admin(admin.ModelAdmin):
     ]
 
 
-class TANF_T4Admin(admin.ModelAdmin):
+class TANF_T4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T4 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -84,14 +93,17 @@ class TANF_T4Admin(admin.ModelAdmin):
     ]
 
 
-class TANF_T5Admin(admin.ModelAdmin):
+class TANF_T5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T5 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -102,14 +114,17 @@ class TANF_T5Admin(admin.ModelAdmin):
     ]
 
 
-class TANF_T6Admin(admin.ModelAdmin):
+class TANF_T6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T6 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
         'RPT_MONTH_YEAR',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -121,9 +136,11 @@ class TANF_T6Admin(admin.ModelAdmin):
     ]
 
 
-class TANF_T7Admin(admin.ModelAdmin):
+class TANF_T7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T7 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
@@ -132,6 +149,7 @@ class TANF_T7Admin(admin.ModelAdmin):
         'STRATUM',
         'FAMILIES_MONTH',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index 2e7f8865c..d7a648e6e 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -1,11 +1,14 @@
 """ModelAdmin classes for parsed TRIBAL data files."""
 from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
+from .mixins import ExportCsvMixin, SttCodeMixin
 
 
-class Tribal_TANF_T1Admin(admin.ModelAdmin):
+class Tribal_TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T1 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -14,6 +17,7 @@ class Tribal_TANF_T1Admin(admin.ModelAdmin):
         'ZIP_CODE',
         'STRATUM',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -26,14 +30,17 @@ class Tribal_TANF_T1Admin(admin.ModelAdmin):
     ]
 
 
-class Tribal_TANF_T2Admin(admin.ModelAdmin):
+class Tribal_TANF_T2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T2 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -44,14 +51,17 @@ class Tribal_TANF_T2Admin(admin.ModelAdmin):
     ]
 
 
-class Tribal_TANF_T3Admin(admin.ModelAdmin):
+class Tribal_TANF_T3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T3 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -61,14 +71,17 @@ class Tribal_TANF_T3Admin(admin.ModelAdmin):
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T4Admin(admin.ModelAdmin):
+class Tribal_TANF_T4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T4 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -77,14 +90,17 @@ class Tribal_TANF_T4Admin(admin.ModelAdmin):
         CreationDateFilter,
         'RPT_MONTH_YEAR',
     ]
-class Tribal_TANF_T5Admin(admin.ModelAdmin):
+class Tribal_TANF_T5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T5 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
         'CASE_NUMBER',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -94,14 +110,17 @@ class Tribal_TANF_T5Admin(admin.ModelAdmin):
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T6Admin(admin.ModelAdmin):
+class Tribal_TANF_T6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal T6 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
         'RPT_MONTH_YEAR',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [
@@ -112,9 +131,11 @@ class Tribal_TANF_T6Admin(admin.ModelAdmin):
         'RPT_MONTH_YEAR'
     ]
 
-class Tribal_TANF_T7Admin(admin.ModelAdmin):
+class Tribal_TANF_T7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal T7 data files."""
 
+    actions = ["export_as_csv"]
+
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
@@ -123,6 +144,7 @@ class Tribal_TANF_T7Admin(admin.ModelAdmin):
         'STRATUM',
         'FAMILIES_MONTH',
         'datafile',
+        'stt_code',
     ]
 
     list_filter = [

From df25f0855bd15403b77b02500064f2b5d613475f Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Tue, 2 Apr 2024 14:07:42 -0600
Subject: [PATCH 029/127] - fix liint

---
 .../tdpservice/search_indexes/admin/mixins.py         | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index d6cb42372..cb7686b51 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -8,14 +8,12 @@ class ExportCsvMixin:
     """Mixin class to support CSV exporting."""
 
     class Echo:
-        """An object that implements just the write method of the file-like
-        interface.
-        """
+        """An object that implements just the write method of the file-like interface."""
 
         def write(self, value):
             """Write the value by returning it, instead of storing in a buffer."""
             return value
-    
+
     class RowIterator:
         """Iterator class to support custom CSV row generation."""
 
@@ -34,10 +32,9 @@ def __init_header_row(self, field_names):
                 if name == "datafile":
                     header_row.append("STT")
             return header_row
-                    
-        
+
         def __iter__(self):
-            """Yield the next row in the csv export.""" 
+            """Yield the next row in the csv export."""
             for obj in self.queryset:
                 row = []
 

From 40f2ecc1430768af020c7d32a5109bc9630cb44c Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 3 Apr 2024 10:54:25 -0400
Subject: [PATCH 030/127] lint

---
 .../tdpservice/parsers/test/test_parse.py       | 17 +++++++++--------
 tdrs-backend/tdpservice/parsers/validators.py   |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 54752e31f..39c841693 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1673,12 +1673,13 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
 
     assert parser_errors.count() == 2
 
-    for error in parser_errors:
-        assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK
+    error1 = parser_errors[0]
+    error2 = parser_errors[1]
 
-        if (error.field_name == None):
-            assert error.error_message == "No records created."
-        elif (error.field_name == "Record_Type"):
-            assert error.error_message == "Reporting month year None does not match file reporting year:2020, quarter:Q1."
-        else:
-            assert False, f"Unexpected field name {error.field_name}"
+    assert error1.error_type == ParserErrorCategoryChoices.PRE_CHECK
+    assert error1.field_name is None
+    assert error1.error_message == "No records created."
+
+    assert error2.error_type == ParserErrorCategoryChoices.PRE_CHECK
+    assert error2.field_name == "Record_Type"
+    assert error2.error_message == "Reporting month year None does not match file reporting year:2020, quarter:Q1."
diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py
index 1254ef06d..1c7449898 100644
--- a/tdrs-backend/tdpservice/parsers/validators.py
+++ b/tdrs-backend/tdpservice/parsers/validators.py
@@ -178,7 +178,7 @@ def validate_reporting_month_year_fields_header(line, row_schema_instance):
         field_month_year = row_schema_instance.get_field_values_by_names(line, ['RPT_MONTH_YEAR']).get('RPT_MONTH_YEAR')
         df_quarter = row_schema_instance.datafile.quarter
         df_year = row_schema_instance.datafile.year
-        
+
         # get reporting month year from header
         field_year, field_quarter = year_month_to_year_quarter(f"{field_month_year}")
         file_calendar_year, file_calendar_qtr = fiscal_to_calendar(df_year, f"{df_quarter}")

From 03d8f9f48b9fe4f45e60aa3eca500d3289573123 Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 3 Apr 2024 11:06:16 -0400
Subject: [PATCH 031/127] lint

---
 .../tdpservice/parsers/test/test_parse.py      | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 39c841693..434e69a3c 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1673,13 +1673,13 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
 
     assert parser_errors.count() == 2
 
-    error1 = parser_errors[0]
-    error2 = parser_errors[1]
-
-    assert error1.error_type == ParserErrorCategoryChoices.PRE_CHECK
-    assert error1.field_name is None
-    assert error1.error_message == "No records created."
+    msg = "Reporting month year None does not match file reporting year:2020, quarter:Q1."
+    for error in parser_errors:
+        assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK
 
-    assert error2.error_type == ParserErrorCategoryChoices.PRE_CHECK
-    assert error2.field_name == "Record_Type"
-    assert error2.error_message == "Reporting month year None does not match file reporting year:2020, quarter:Q1."
+        if (error.field_name is None):
+            assert error.error_message == "No records created."
+        elif (error.field_name == "Record_Type"):
+            assert error.error_message == msg
+        else:
+            assert False, f"Unexpected field name {error.field_name}"

From a1d5cf57fb39bab0c6e0f42a89ff3a8480b66cc3 Mon Sep 17 00:00:00 2001
From: Andrew Trimpe <atrimpe@teamraft.com>
Date: Wed, 3 Apr 2024 14:24:49 -0400
Subject: [PATCH 032/127] update 2019 -> 2020 and fix datafiles and tests

---
 .../tdpservice/parsers/schema_defs/ssp/m6.py  |  6 +--
 .../tdpservice/parsers/schema_defs/ssp/m7.py  |  5 +--
 .../tdpservice/parsers/schema_defs/tanf/t6.py |  5 +--
 .../tdpservice/parsers/schema_defs/tanf/t7.py |  4 +-
 .../parsers/schema_defs/tribal_tanf/t6.py     |  5 +--
 .../parsers/schema_defs/tribal_tanf/t7.py     |  4 +-
 .../parsers/test/data/ADS.E2J.FTP3.TS06       |  4 +-
 .../parsers/test/data/ADS.E2J.FTP3.TS142      |  4 +-
 .../parsers/test/data/ADS.E2J.FTP4.TS06       |  4 +-
 .../parsers/test/data/ADS.E2J.NDM3.MS24       |  4 +-
 .../parsers/test/data/ADS.E2J.NDM4.MS24       |  4 +-
 .../test/data/tanf_section4_with_errors.txt   |  4 +-
 .../test/data/tribal_section_4_fake.txt       |  4 +-
 .../tdpservice/parsers/test/test_parse.py     | 41 ++++++++++---------
 tdrs-backend/tdpservice/parsers/validators.py |  1 -
 15 files changed, 44 insertions(+), 55 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
index 722105c86..57ae9262f 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -48,7 +48,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(2019),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid()
             ]
         ),
@@ -211,7 +211,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(2019),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid()
             ]
         ),
@@ -374,7 +374,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(2019),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid()
             ]
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
index d4d28aed9..9ddd4a212 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
@@ -5,9 +5,6 @@
 from ...transforms import calendar_quarter_to_rpt_month_year
 from ... import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument
-import datetime
-
-minYear = datetime.date.today().year - 5
 
 schemas = []
 
@@ -51,7 +48,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(minYear),
+                        validators.dateYearIsLargerThan(2020),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
index d6a96a948..c7e020ae3 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
@@ -6,9 +6,6 @@
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T6DataSubmissionDocument
-import datetime
-
-minYear = datetime.date.today().year - 5
 
 s1 = RowSchema(
     document=TANF_T6DataSubmissionDocument(),
@@ -57,7 +54,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(minYear),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid(),
             ],
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
index f45439f29..714587acc 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
@@ -5,9 +5,7 @@
 from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T7DataSubmissionDocument
-import datetime
 
-minYear = datetime.date.today().year - 5
 schemas = []
 
 validator_index = 7
@@ -50,7 +48,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(minYear),
+                        validators.dateYearIsLargerThan(2020),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
index dca8a9158..ddd0af8d0 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
@@ -6,9 +6,6 @@
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T6DataSubmissionDocument
-import datetime
-
-minYear = datetime.date.today().year - 5
 
 s1 = RowSchema(
     document=Tribal_TANF_T6DataSubmissionDocument(),
@@ -45,7 +42,7 @@
             endIndex=7,
             required=True,
             validators=[
-                validators.dateYearIsLargerThan(minYear),
+                validators.dateYearIsLargerThan(2020),
                 validators.quarterIsValid(),
             ],
         ),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
index 2575cb26c..97b1b8e5a 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
@@ -5,9 +5,7 @@
 from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T7DataSubmissionDocument
-import datetime
 
-minYear = datetime.date.today().year - 5
 schemas = []
 
 validator_index = 7
@@ -50,7 +48,7 @@
                     endIndex=7,
                     required=True,
                     validators=[
-                        validators.dateYearIsLargerThan(minYear),
+                        validators.dateYearIsLargerThan(2020),
                         validators.quarterIsValid(),
                     ],
                 ),
diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06
index 8e88b1fb0..1fe4531ab 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06
+++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS06
@@ -1,3 +1,3 @@
-HEADER20204G06   TAN1 N
-T620204000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884
+HEADER20214G06   TAN1 N
+T620214000127470001104500011146000043010000397700003924000084460000706800007222000055042849000055141378000056643253000755810007592100075542000009810000097000000968000392980003934900038972000353020003560200035602001684470016904700168107000464480004649800046203001219990012254900121904000001630000014900000151000003440000033100000276000002580000024100000187000054530000388100003884
 TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142
index 7ebd8bbcc..00ec788ff 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142
+++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP3.TS142
@@ -1,3 +1,3 @@
-HEADER20204G00142TAN1EU
-T620204       5       1       1       5       1       1       0       0       0       21097       22915       21441      40      41      40       5       6       4       8       9      11      27      26      25      85      89      85      18      21      19      67      68      66       0       0       0       0       0       0       0       0       0       3       3       0
+HEADER20214G00142TAN1EU
+T620214       5       1       1       5       1       1       0       0       0       21097       22915       21441      40      41      40       5       6       4       8       9      11      27      26      25      85      89      85      18      21      19      67      68      66       0       0       0       0       0       0       0       0       0       3       3       0
 TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06
index 5c344cf42..b38ed125a 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06
+++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP4.TS06
@@ -1,3 +1,3 @@
-HEADER20204S06   TAN1 N
-T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+HEADER20214S06   TAN1 N
+T720214101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
 TRAILER0000001         
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24
index 6ea30625f..9bd4925f6 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24
+++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM3.MS24
@@ -1,3 +1,3 @@
-HEADER20204G24   SSP1 N
-M620204000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229
+HEADER20214G24   SSP1 N
+M620214000158690001600800015956000008610000085100000845000149050001505500015013000001030000010200000098000513550005169600051348000157070001581400015766000356480003588200035582000000000000000000000000000000000000000000000000000000000000000012020000118900001229
 TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24 b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24
index fbf049ce9..74a14b2d0 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24
+++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.NDM4.MS24
@@ -1,3 +1,3 @@
-HEADER20204S24   SSP1 N
-M720204101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229                                                                                                                                                
+HEADER20214S24   SSP1 N
+M720214101000176900013100001111102000074800007670000768103001335200139310014077200000120200011890001229                                                                                                                                                
 TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt b/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt
index 53590eccf..afe9ef1bf 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt
+++ b/tdrs-backend/tdpservice/parsers/test/data/tanf_section4_with_errors.txt
@@ -1,3 +1,3 @@
-HEADER20204S06   TAN1 N
-T720204700006853700680540068454103000312400000000003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+HEADER20214S06   TAN1 N
+T720214700006853700680540068454103000312400000000003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
 TRAILER0000001         
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt
index 9ec1943d3..41fab8910 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt
+++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake.txt
@@ -1,3 +1,3 @@
-HEADER20204S00142TAN1EU
-T720204101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
+HEADER20214S00142TAN1EU
+T720214101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
 TRAILER0000001         
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 434e69a3c..a92b849fd 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1020,7 +1020,7 @@ def tanf_section3_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tanf_section3_file(tanf_section3_file, dfs):
     """Test parsing TANF Section 3 submission."""
-    tanf_section3_file.year = 2021
+    tanf_section3_file.year = 2022
     tanf_section3_file.quarter = 'Q1'
 
     dfs.datafile = tanf_section3_file
@@ -1049,9 +1049,9 @@ def test_parse_tanf_section3_file(tanf_section3_file, dfs):
     second = t6_objs[1]
     third = t6_objs[2]
 
-    assert first.RPT_MONTH_YEAR == 202012
-    assert second.RPT_MONTH_YEAR == 202011
-    assert third.RPT_MONTH_YEAR == 202010
+    assert first.RPT_MONTH_YEAR == 202112
+    assert second.RPT_MONTH_YEAR == 202111
+    assert third.RPT_MONTH_YEAR == 202110
 
     assert first.NUM_APPROVED == 3924
     assert second.NUM_APPROVED == 3977
@@ -1102,7 +1102,7 @@ def tanf_section4_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tanf_section4_file(tanf_section4_file, dfs):
     """Test parsing TANF Section 4 submission."""
-    tanf_section4_file.year = 2021
+    tanf_section4_file.year = 2022
     tanf_section4_file.quarter = 'Q1'
 
     dfs.datafile = tanf_section4_file
@@ -1130,8 +1130,8 @@ def test_parse_tanf_section4_file(tanf_section4_file, dfs):
     first = t7_objs.first()
     sixth = t7_objs[5]
 
-    assert first.RPT_MONTH_YEAR == 202011
-    assert sixth.RPT_MONTH_YEAR == 202012
+    assert first.RPT_MONTH_YEAR == 202111
+    assert sixth.RPT_MONTH_YEAR == 202112
 
     assert first.TDRS_SECTION_IND == '2'
     assert sixth.TDRS_SECTION_IND == '2'
@@ -1189,7 +1189,7 @@ def ssp_section4_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_ssp_section4_file(ssp_section4_file, dfs):
     """Test parsing SSP Section 4 submission."""
-    ssp_section4_file.year = 2021
+    ssp_section4_file.year = 2022
     ssp_section4_file.quarter = 'Q1'
 
     dfs.datafile = ssp_section4_file
@@ -1210,7 +1210,7 @@ def test_parse_ssp_section4_file(ssp_section4_file, dfs):
     assert m7_objs.count() == 12
 
     first = m7_objs.first()
-    assert first.RPT_MONTH_YEAR == 202010
+    assert first.RPT_MONTH_YEAR == 202110
     assert first.FAMILIES_MONTH == 748
 
 @pytest.fixture
@@ -1277,7 +1277,7 @@ def ssp_section3_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_ssp_section3_file(ssp_section3_file, dfs):
     """Test parsing TANF Section 3 submission."""
-    ssp_section3_file.year = 2021
+    ssp_section3_file.year = 2022
     ssp_section3_file.quarter = 'Q1'
 
     dfs.datafile = ssp_section3_file
@@ -1305,9 +1305,9 @@ def test_parse_ssp_section3_file(ssp_section3_file, dfs):
     second = m6_objs[1]
     third = m6_objs[2]
 
-    assert first.RPT_MONTH_YEAR == 202010
-    assert second.RPT_MONTH_YEAR == 202011
-    assert third.RPT_MONTH_YEAR == 202012
+    assert first.RPT_MONTH_YEAR == 202110
+    assert second.RPT_MONTH_YEAR == 202111
+    assert third.RPT_MONTH_YEAR == 202112
 
     assert first.SSPMOE_FAMILIES == 15869
     assert second.SSPMOE_FAMILIES == 16008
@@ -1460,7 +1460,7 @@ def tribal_section_3_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tribal_section_3_file(tribal_section_3_file, dfs):
     """Test parsing Tribal TANF Section 3 submission."""
-    tribal_section_3_file.year = 2021
+    tribal_section_3_file.year = 2022
     tribal_section_3_file.quarter = 'Q1'
 
     dfs.datafile = tribal_section_3_file
@@ -1496,7 +1496,7 @@ def tribal_section_4_file(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tribal_section_4_file(tribal_section_4_file, dfs):
     """Test parsing Tribal TANF Section 4 submission."""
-    tribal_section_4_file.year = 2021
+    tribal_section_4_file.year = 2022
     tribal_section_4_file.quarter = 'Q1'
 
     dfs.datafile = tribal_section_4_file
@@ -1519,8 +1519,8 @@ def test_parse_tribal_section_4_file(tribal_section_4_file, dfs):
     first = t7_objs.first()
     sixth = t7_objs[5]
 
-    assert first.RPT_MONTH_YEAR == 202011
-    assert sixth.RPT_MONTH_YEAR == 202012
+    assert first.RPT_MONTH_YEAR == 202111
+    assert sixth.RPT_MONTH_YEAR == 202112
 
     assert first.TDRS_SECTION_IND == '2'
     assert sixth.TDRS_SECTION_IND == '2'
@@ -1593,6 +1593,8 @@ def tanf_section_4_file_with_errors(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, dfs):
     """Test parsing TANF Section 4 submission."""
+    tanf_section_4_file_with_errors.year = 2022
+    tanf_section_4_file_with_errors.quarter = 'Q1'
     dfs.datafile = tanf_section_4_file_with_errors
 
     parse.parse_datafile(tanf_section_4_file_with_errors, dfs)
@@ -1611,6 +1613,7 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d
     assert TANF_T7.objects.all().count() == 18
 
     parser_errors = ParserError.objects.filter(file=tanf_section_4_file_with_errors)
+
     assert parser_errors.count() == 6
 
     t7_objs = TANF_T7.objects.all().order_by('FAMILIES_MONTH')
@@ -1618,8 +1621,8 @@ def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, d
     first = t7_objs.first()
     sixth = t7_objs[5]
 
-    assert first.RPT_MONTH_YEAR == 202011
-    assert sixth.RPT_MONTH_YEAR == 202010
+    assert first.RPT_MONTH_YEAR == 202111
+    assert sixth.RPT_MONTH_YEAR == 202110
 
     assert first.TDRS_SECTION_IND == '1'
     assert sixth.TDRS_SECTION_IND == '1'
diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py
index 1c7449898..e90cf7762 100644
--- a/tdrs-backend/tdpservice/parsers/validators.py
+++ b/tdrs-backend/tdpservice/parsers/validators.py
@@ -182,7 +182,6 @@ def validate_reporting_month_year_fields_header(line, row_schema_instance):
         # get reporting month year from header
         field_year, field_quarter = year_month_to_year_quarter(f"{field_month_year}")
         file_calendar_year, file_calendar_qtr = fiscal_to_calendar(df_year, f"{df_quarter}")
-
         return (True, None) if str(file_calendar_year) == str(field_year) and file_calendar_qtr == field_quarter else (
             False, f"Reporting month year {field_month_year} " +
             f"does not match file reporting year:{df_year}, quarter:{df_quarter}.",

From 705deb603cd13f8a653caa228b7e14523e0c92d6 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 3 Apr 2024 14:01:11 -0600
Subject: [PATCH 033/127] - Add multiselector listfilter - Update admin models

---
 .../search_indexes/admin/filters.py           | 79 ++++++++++++++-----
 .../tdpservice/search_indexes/admin/mixins.py | 17 +++-
 .../tdpservice/search_indexes/admin/ssp.py    | 30 +++----
 .../tdpservice/search_indexes/admin/tanf.py   | 30 +++----
 .../tdpservice/search_indexes/admin/tribal.py | 30 +++----
 .../templates/multiselectlistfilter.html      | 24 ++++++
 6 files changed, 146 insertions(+), 64 deletions(-)
 create mode 100644 tdrs-backend/tdpservice/search_indexes/templates/multiselectlistfilter.html

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index c9f1762e4..0461dfba4 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -4,6 +4,58 @@
 from tdpservice.stts.models import STT
 import datetime
 
+
+class MultipleChoiceListFilter(SimpleListFilter):
+    template = 'multiselectlistfilter.html'
+
+    def lookups(self, request, model_admin):
+        """
+        Must be overridden to return a list of tuples (value, verbose value)
+        """
+        raise NotImplementedError(
+            'The MultipleChoiceListFilter.lookups() method must be overridden to '
+            'return a list of tuples (value, verbose value).'
+        )
+
+    def queryset(self, request, queryset):
+        if request.GET.get(self.parameter_name):
+            kwargs = {self.parameter_name: request.GET[self.parameter_name].split(',')}
+            queryset = queryset.filter(**kwargs)
+        return queryset
+
+    def value_as_list(self):
+        return self.value().split(',') if self.value() else []
+
+    def choices(self, changelist):
+
+        def amend_query_string(include=None, exclude=None):
+            selections = self.value_as_list()
+            if include and include not in selections:
+                selections.append(include)
+            if exclude and exclude in selections:
+                selections.remove(exclude)
+            if selections:
+                csv = ','.join(selections)
+                return changelist.get_query_string({self.parameter_name: csv})
+            else:
+                return changelist.get_query_string(remove=[self.parameter_name])
+
+        yield {
+            'selected': self.value() is None,
+            'query_string': changelist.get_query_string(remove=[self.parameter_name]),
+            'display': 'All',
+            'reset': True,
+        }
+        for lookup, title in self.lookup_choices:
+            yield {
+                'selected': str(lookup) in self.value_as_list(),
+                'query_string': changelist.get_query_string({self.parameter_name: lookup}),
+                'include_query_string': amend_query_string(include=str(lookup)),
+                'exclude_query_string': amend_query_string(exclude=str(lookup)),
+                'display': title,
+            }
+
+
 class CreationDateFilter(SimpleListFilter):
     """Simple filter class to show newest created datafile records."""
 
@@ -37,7 +89,7 @@ def queryset(self, request, queryset):
         return queryset
 
 
-class STTFilter(SimpleListFilter):
+class STTFilter(MultipleChoiceListFilter):
     """Simple filter class to show records based on stt."""
 
     title = _('STT Code')
@@ -46,29 +98,20 @@ class STTFilter(SimpleListFilter):
 
     def lookups(self, request, model_admin):
         """Available options in dropdown."""
-        options = [(None, _('All'))]
+        options = []
         for obj in STT.objects.all():
             options.append((obj.stt_code, _(obj.name)))
         return options
 
-    def choices(self, cl):
-        """Update query string based on selection."""
-        for lookup, title in self.lookup_choices:
-            yield {
-                'selected': self.value() == lookup,
-                'query_string': cl.get_query_string({
-                    self.parameter_name: lookup,
-                }, []),
-                'display': title,
-            }
-
     def queryset(self, request, queryset):
         """Return queryset of records based on stt code."""
-        val = self.value()
-        if val is not None and queryset.exists():
-            if len(val) == 1:
-                val = f"0{val}"
-            queryset = queryset.filter(datafile__stt__stt_code=val)
+
+        if self.value() is not None and queryset.exists():
+            stts = self.value().split(',')
+            print(stts)
+            print(queryset.count())
+            queryset = queryset.filter(datafile__stt__stt_code__in=stts)
+            print(queryset.count())
         return queryset
 
 class FiscalPeriodFilter(SimpleListFilter):
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index cb7686b51..b1b4b99c3 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -1,5 +1,5 @@
 """Mixin classes supproting custom functionality."""
-
+from django.contrib import admin
 from django.http import StreamingHttpResponse
 import csv
 
@@ -71,3 +71,18 @@ class SttCodeMixin:
     def stt_code(self, obj):
         """Return stt code."""
         return obj.datafile.stt.stt_code
+
+
+class AdminModelActionMixinBase(admin.ModelAdmin):
+    pass
+
+class AdminModelDisableDeleteActionMixin(AdminModelActionMixinBase):
+    """Mixin class to conditionally disable model deletion."""
+
+    def get_actions(self, request):
+        """Toggle the delete action."""
+        print("\n\nINSIDE GET ACTIONS\n\n")
+        actions = super().get_actions(request)
+        if 'delete_selected' in actions:
+            del actions['delete_selected']
+        return actions
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index 3685069b8..12a7f5d40 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -1,10 +1,10 @@
 """ModelAdmin classes for parsed SSP data files."""
 from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import ExportCsvMixin, SttCodeMixin
+from .mixins import AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
 
 
-class SSP_M1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M1 data files."""
 
     actions = ["export_as_csv"]
@@ -21,16 +21,16 @@ class SSP_M1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
         'ZIP_CODE',
         'STRATUM',
     ]
 
 
-class SSP_M2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M2 data files."""
 
     actions = ["export_as_csv"]
@@ -44,14 +44,14 @@ class SSP_M2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
 
-class SSP_M3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -65,13 +65,13 @@ class SSP_M3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -85,13 +85,13 @@ class SSP_M4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -105,13 +105,13 @@ class SSP_M5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M6 data files."""
 
     actions = ["export_as_csv"]
@@ -126,13 +126,13 @@ class SSP_M6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
 
     list_filter = [
         'CALENDAR_QUARTER',
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR'
     ]
 
-class SSP_M7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M7Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M7 data files."""
 
     actions = ["export_as_csv"]
@@ -150,8 +150,8 @@ class SSP_M7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
 
     list_filter = [
         'CALENDAR_QUARTER',
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 164a664fa..5fb87e160 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,10 +1,10 @@
 """ModelAdmin classes for parsed TANF data files."""
 from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import ExportCsvMixin, SttCodeMixin
+from .mixins import AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
 
 
-class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T1 data files."""
 
     actions = ["export_as_csv"]
@@ -21,16 +21,16 @@ class TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
         'ZIP_CODE',
         'STRATUM',
     ]
 
 
-class TANF_T2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T2 data files."""
 
     actions = ["export_as_csv"]
@@ -44,14 +44,14 @@ class TANF_T2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
 
-class TANF_T3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T3 data files."""
 
     actions = ["export_as_csv"]
@@ -65,14 +65,14 @@ class TANF_T3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
 
-class TANF_T4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T4 data files."""
 
     actions = ["export_as_csv"]
@@ -86,14 +86,14 @@ class TANF_T4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
 
-class TANF_T5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T5 data files."""
 
     actions = ["export_as_csv"]
@@ -107,14 +107,14 @@ class TANF_T5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
 
-class TANF_T6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T6 data files."""
 
     actions = ["export_as_csv"]
@@ -129,14 +129,14 @@ class TANF_T6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
 
     list_filter = [
         'CALENDAR_QUARTER',
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR'
     ]
 
 
-class TANF_T7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T7Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T7 data files."""
 
     actions = ["export_as_csv"]
@@ -154,8 +154,8 @@ class TANF_T7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
 
     list_filter = [
         'CALENDAR_QUARTER',
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index d7a648e6e..1571432fc 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -1,10 +1,10 @@
 """ModelAdmin classes for parsed TRIBAL data files."""
 from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import ExportCsvMixin, SttCodeMixin
+from .mixins import AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
 
 
-class Tribal_TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T1 data files."""
 
     actions = ["export_as_csv"]
@@ -21,16 +21,16 @@ class Tribal_TANF_T1Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
         'ZIP_CODE',
         'STRATUM',
     ]
 
 
-class Tribal_TANF_T2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T2 data files."""
 
     actions = ["export_as_csv"]
@@ -44,14 +44,14 @@ class Tribal_TANF_T2Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
 
-class Tribal_TANF_T3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T3 data files."""
 
     actions = ["export_as_csv"]
@@ -65,13 +65,13 @@ class Tribal_TANF_T3Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T4 data files."""
 
     actions = ["export_as_csv"]
@@ -85,12 +85,12 @@ class Tribal_TANF_T4Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
-class Tribal_TANF_T5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T5 data files."""
 
     actions = ["export_as_csv"]
@@ -104,13 +104,13 @@ class Tribal_TANF_T5Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
     list_filter = [
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal T6 data files."""
 
     actions = ["export_as_csv"]
@@ -125,13 +125,13 @@ class Tribal_TANF_T6Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
 
     list_filter = [
         'CALENDAR_QUARTER',
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR'
     ]
 
-class Tribal_TANF_T7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T7Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal T7 data files."""
 
     actions = ["export_as_csv"]
@@ -149,8 +149,8 @@ class Tribal_TANF_T7Admin(admin.ModelAdmin, ExportCsvMixin, SttCodeMixin):
 
     list_filter = [
         'CALENDAR_QUARTER',
-        STTFilter,
         FiscalPeriodFilter,
         CreationDateFilter,
+        STTFilter,
         'RPT_MONTH_YEAR',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/templates/multiselectlistfilter.html b/tdrs-backend/tdpservice/search_indexes/templates/multiselectlistfilter.html
new file mode 100644
index 000000000..d6b4a45c6
--- /dev/null
+++ b/tdrs-backend/tdpservice/search_indexes/templates/multiselectlistfilter.html
@@ -0,0 +1,24 @@
+{% load i18n %}
+<h3>{% blocktrans with filter_title=title %} By {{ filter_title }} {% endblocktrans %}</h3>
+<ul class="mulitple-choice">
+{% for choice in choices %}
+    {% if choice.reset %}
+        <li{% if choice.selected %} class="selected"{% endif %}>
+            <a href="{{ choice.query_string|iriencode }}" title="{{ choice.display }}">{{ choice.display }}</a>
+        </li>
+    {% endif %}
+{% endfor %}
+{% for choice in choices %}
+    {% if not choice.reset %}
+        <li{% if choice.selected %} class="selected"{% endif %}>
+          <a href="{{ choice.query_string|iriencode }}" title="{{ choice.display }}" style="display:inline">{{ choice.display }}</a>
+          {% if choice.selected and choice.exclude_query_string %}
+            <a class="small" href="{{ choice.exclude_query_string|iriencode }}" style="display:inline">(exclude)</a>
+          {% endif %}
+          {% if not choice.selected and choice.include_query_string %}
+            <a class="small" href="{{ choice.include_query_string|iriencode }}" style="display:inline">(include)</a>
+          {% endif %}
+        </li>
+    {% endif %}
+{% endfor %}
+</ul>
\ No newline at end of file

From bbba7989a429a15421787b1c46f7c729adea08c1 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 3 Apr 2024 14:07:09 -0600
Subject: [PATCH 034/127] - Rename Mixin - Create base Mixin to appeal to MRO

---
 .../tdpservice/search_indexes/admin/mixins.py    |  8 ++++----
 .../tdpservice/search_indexes/admin/ssp.py       | 16 ++++++++--------
 .../tdpservice/search_indexes/admin/tanf.py      | 16 ++++++++--------
 .../tdpservice/search_indexes/admin/tribal.py    | 16 ++++++++--------
 4 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index b1b4b99c3..b27baf119 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -73,15 +73,15 @@ def stt_code(self, obj):
         return obj.datafile.stt.stt_code
 
 
-class AdminModelActionMixinBase(admin.ModelAdmin):
+class AdminModelMixin(admin.ModelAdmin):
+    """Base class for all mixin classes needing to modify ModelAdmin methods."""
     pass
 
-class AdminModelDisableDeleteActionMixin(AdminModelActionMixinBase):
-    """Mixin class to conditionally disable model deletion."""
+class DisableDeleteActionMixin(AdminModelMixin):
+    """Mixin class to disable model deletion."""
 
     def get_actions(self, request):
         """Toggle the delete action."""
-        print("\n\nINSIDE GET ACTIONS\n\n")
         actions = super().get_actions(request)
         if 'delete_selected' in actions:
             del actions['delete_selected']
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index 12a7f5d40..8a5eebe05 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -1,10 +1,10 @@
 """ModelAdmin classes for parsed SSP data files."""
 from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
+from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
 
 
-class SSP_M1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M1 data files."""
 
     actions = ["export_as_csv"]
@@ -30,7 +30,7 @@ class SSP_M1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMix
     ]
 
 
-class SSP_M2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M2 data files."""
 
     actions = ["export_as_csv"]
@@ -51,7 +51,7 @@ class SSP_M2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMix
     ]
 
 
-class SSP_M3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -71,7 +71,7 @@ class SSP_M3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMix
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -91,7 +91,7 @@ class SSP_M4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMix
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -111,7 +111,7 @@ class SSP_M5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMix
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M6 data files."""
 
     actions = ["export_as_csv"]
@@ -132,7 +132,7 @@ class SSP_M6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMix
         'RPT_MONTH_YEAR'
     ]
 
-class SSP_M7Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M7 data files."""
 
     actions = ["export_as_csv"]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 5fb87e160..5ebabec3d 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,10 +1,10 @@
 """ModelAdmin classes for parsed TANF data files."""
 from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
+from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin, AdminModelMixin
 
 
-class TANF_T1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T1 data files."""
 
     actions = ["export_as_csv"]
@@ -30,7 +30,7 @@ class TANF_T1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMi
     ]
 
 
-class TANF_T2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T2 data files."""
 
     actions = ["export_as_csv"]
@@ -51,7 +51,7 @@ class TANF_T2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMi
     ]
 
 
-class TANF_T3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T3 data files."""
 
     actions = ["export_as_csv"]
@@ -72,7 +72,7 @@ class TANF_T3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMi
     ]
 
 
-class TANF_T4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T4 data files."""
 
     actions = ["export_as_csv"]
@@ -93,7 +93,7 @@ class TANF_T4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMi
     ]
 
 
-class TANF_T5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T5 data files."""
 
     actions = ["export_as_csv"]
@@ -114,7 +114,7 @@ class TANF_T5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMi
     ]
 
 
-class TANF_T6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T6 data files."""
 
     actions = ["export_as_csv"]
@@ -136,7 +136,7 @@ class TANF_T6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMi
     ]
 
 
-class TANF_T7Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T7 data files."""
 
     actions = ["export_as_csv"]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index 1571432fc..fb7e0d331 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -1,10 +1,10 @@
 """ModelAdmin classes for parsed TRIBAL data files."""
 from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
+from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
 
 
-class Tribal_TANF_T1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T1 data files."""
 
     actions = ["export_as_csv"]
@@ -30,7 +30,7 @@ class Tribal_TANF_T1Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, St
     ]
 
 
-class Tribal_TANF_T2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T2 data files."""
 
     actions = ["export_as_csv"]
@@ -51,7 +51,7 @@ class Tribal_TANF_T2Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, St
     ]
 
 
-class Tribal_TANF_T3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T3 data files."""
 
     actions = ["export_as_csv"]
@@ -71,7 +71,7 @@ class Tribal_TANF_T3Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, St
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T4 data files."""
 
     actions = ["export_as_csv"]
@@ -90,7 +90,7 @@ class Tribal_TANF_T4Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, St
         STTFilter,
         'RPT_MONTH_YEAR',
     ]
-class Tribal_TANF_T5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T5 data files."""
 
     actions = ["export_as_csv"]
@@ -110,7 +110,7 @@ class Tribal_TANF_T5Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, St
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal T6 data files."""
 
     actions = ["export_as_csv"]
@@ -131,7 +131,7 @@ class Tribal_TANF_T6Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, St
         'RPT_MONTH_YEAR'
     ]
 
-class Tribal_TANF_T7Admin(AdminModelDisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal T7 data files."""
 
     actions = ["export_as_csv"]

From e3ecc0d9086954a3fc4c5ff5b2da9b62b55f1888 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 3 Apr 2024 14:07:44 -0600
Subject: [PATCH 035/127] - Remove unused includes

---
 tdrs-backend/tdpservice/search_indexes/admin/ssp.py    | 1 -
 tdrs-backend/tdpservice/search_indexes/admin/tanf.py   | 1 -
 tdrs-backend/tdpservice/search_indexes/admin/tribal.py | 1 -
 3 files changed, 3 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index 8a5eebe05..f0183d05e 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -1,5 +1,4 @@
 """ModelAdmin classes for parsed SSP data files."""
-from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
 from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
 
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 5ebabec3d..5b9e091ca 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,5 +1,4 @@
 """ModelAdmin classes for parsed TANF data files."""
-from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
 from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin, AdminModelMixin
 
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index fb7e0d331..0d5f9163f 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -1,5 +1,4 @@
 """ModelAdmin classes for parsed TRIBAL data files."""
-from django.contrib import admin
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
 from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
 

From ca995ce50cbc03f039fd2f510f175774ee26118c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 3 Apr 2024 14:13:32 -0600
Subject: [PATCH 036/127] - lint

---
 .../tdpservice/search_indexes/admin/filters.py   | 16 +++++++---------
 .../tdpservice/search_indexes/admin/mixins.py    |  3 ++-
 .../tdpservice/search_indexes/admin/tanf.py      |  2 +-
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index 0461dfba4..1278a9682 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -6,28 +6,30 @@
 
 
 class MultipleChoiceListFilter(SimpleListFilter):
+    """Filter class allowing multiple filter options."""
+
     template = 'multiselectlistfilter.html'
 
     def lookups(self, request, model_admin):
-        """
-        Must be overridden to return a list of tuples (value, verbose value)
-        """
+        """Must be overridden to return a list of tuples (value, verbose value)."""
         raise NotImplementedError(
             'The MultipleChoiceListFilter.lookups() method must be overridden to '
             'return a list of tuples (value, verbose value).'
         )
 
     def queryset(self, request, queryset):
+        """Return queryset based on selected parameters."""
         if request.GET.get(self.parameter_name):
             kwargs = {self.parameter_name: request.GET[self.parameter_name].split(',')}
             queryset = queryset.filter(**kwargs)
         return queryset
 
     def value_as_list(self):
+        """Convert multiple filter fields to list."""
         return self.value().split(',') if self.value() else []
 
     def choices(self, changelist):
-
+        """Overriden choices method."""
         def amend_query_string(include=None, exclude=None):
             selections = self.value_as_list()
             if include and include not in selections:
@@ -104,14 +106,10 @@ def lookups(self, request, model_admin):
         return options
 
     def queryset(self, request, queryset):
-        """Return queryset of records based on stt code."""
-
+        """Return queryset of records based on stt code(s)."""
         if self.value() is not None and queryset.exists():
             stts = self.value().split(',')
-            print(stts)
-            print(queryset.count())
             queryset = queryset.filter(datafile__stt__stt_code__in=stts)
-            print(queryset.count())
         return queryset
 
 class FiscalPeriodFilter(SimpleListFilter):
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index b27baf119..2c357d4bc 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -75,6 +75,7 @@ def stt_code(self, obj):
 
 class AdminModelMixin(admin.ModelAdmin):
     """Base class for all mixin classes needing to modify ModelAdmin methods."""
+
     pass
 
 class DisableDeleteActionMixin(AdminModelMixin):
@@ -85,4 +86,4 @@ def get_actions(self, request):
         actions = super().get_actions(request)
         if 'delete_selected' in actions:
             del actions['delete_selected']
-        return actions
\ No newline at end of file
+        return actions
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 5b9e091ca..2632abef9 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,6 +1,6 @@
 """ModelAdmin classes for parsed TANF data files."""
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin, AdminModelMixin
+from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
 
 
 class TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):

From 7e6979f3e3f7e10b0a27abfb2af2f874fcd00864 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 3 Apr 2024 14:37:24 -0600
Subject: [PATCH 037/127] - Order by stt_code

---
 .../tdpservice/search_indexes/admin/filters.py | 18 +++++-------------
 .../tdpservice/search_indexes/admin/ssp.py     |  7 +++++++
 .../tdpservice/search_indexes/admin/tanf.py    |  7 +++++++
 .../tdpservice/search_indexes/admin/tribal.py  |  7 +++++++
 4 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index 1278a9682..c0b4a5d6b 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -72,22 +72,14 @@ def lookups(self, request, model_admin):
             ('all', _('All')),
         )
 
-    def choices(self, cl):
-        """Update query string based on selection."""
-        for lookup, title in self.lookup_choices:
-            yield {
-                'selected': self.value() == lookup,
-                'query_string': cl.get_query_string({
-                    self.parameter_name: lookup,
-                }, []),
-                'display': title,
-            }
-
     def queryset(self, request, queryset):
         """Sort queryset to show latest records."""
         if self.value() is None and queryset.exists():
-            datafile = queryset.order_by("-datafile__id").first().datafile
-            return queryset.filter(datafile=datafile)
+            datafiles = []
+            for record in queryset.order_by("datafile__stt__stt_code", "-datafile__id")\
+                .distinct("datafile__stt__stt_code"):
+                datafiles.append(record.datafile)
+            return queryset.filter(datafile__in=datafiles)
         return queryset
 
 
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index f0183d05e..193ba76d3 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -7,6 +7,7 @@ class SSP_M1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M1 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -33,6 +34,7 @@ class SSP_M2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M2 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -54,6 +56,7 @@ class SSP_M3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -74,6 +77,7 @@ class SSP_M4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -94,6 +98,7 @@ class SSP_M5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -114,6 +119,7 @@ class SSP_M6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M6 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -135,6 +141,7 @@ class SSP_M7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M7 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 2632abef9..3a9521396 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -7,6 +7,7 @@ class TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T1 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -33,6 +34,7 @@ class TANF_T2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T2 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -54,6 +56,7 @@ class TANF_T3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T3 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -75,6 +78,7 @@ class TANF_T4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T4 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -96,6 +100,7 @@ class TANF_T5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T5 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -117,6 +122,7 @@ class TANF_T6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T6 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -139,6 +145,7 @@ class TANF_T7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T7 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index 0d5f9163f..c8a1782c1 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -7,6 +7,7 @@ class Tribal_TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     """ModelAdmin class for parsed Tribal_T1 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -33,6 +34,7 @@ class Tribal_TANF_T2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     """ModelAdmin class for parsed Tribal_T2 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -54,6 +56,7 @@ class Tribal_TANF_T3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     """ModelAdmin class for parsed Tribal_T3 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -74,6 +77,7 @@ class Tribal_TANF_T4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     """ModelAdmin class for parsed Tribal_T4 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -93,6 +97,7 @@ class Tribal_TANF_T5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     """ModelAdmin class for parsed Tribal_T5 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -113,6 +118,7 @@ class Tribal_TANF_T6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     """ModelAdmin class for parsed Tribal T6 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',
@@ -134,6 +140,7 @@ class Tribal_TANF_T7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     """ModelAdmin class for parsed Tribal T7 data files."""
 
     actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',

From 5940b734b4b0f83b5b32c6fbd931b9d16c7d8077 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 3 Apr 2024 14:46:40 -0600
Subject: [PATCH 038/127] - Make admin read only

---
 .../tdpservice/search_indexes/admin/mixins.py | 28 +++++++++++++------
 .../tdpservice/search_indexes/admin/ssp.py    | 16 +++++------
 .../tdpservice/search_indexes/admin/tanf.py   | 16 +++++------
 .../tdpservice/search_indexes/admin/tribal.py | 16 +++++------
 4 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index 2c357d4bc..2c751f337 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -78,12 +78,22 @@ class AdminModelMixin(admin.ModelAdmin):
 
     pass
 
-class DisableDeleteActionMixin(AdminModelMixin):
-    """Mixin class to disable model deletion."""
-
-    def get_actions(self, request):
-        """Toggle the delete action."""
-        actions = super().get_actions(request)
-        if 'delete_selected' in actions:
-            del actions['delete_selected']
-        return actions
+
+class ReadOnlyAdmin(AdminModelMixin):
+    """Force ModelAdmin to be READ only."""
+    readonly_fields = []
+
+    def get_readonly_fields(self, request, obj=None):
+        """Force all fields to read only."""
+        return list(self.readonly_fields) + \
+               [field.name for field in obj._meta.fields] + \
+               [field.name for field in obj._meta.many_to_many]
+
+
+    def has_add_permission(self, request):
+        """Deny add permisison."""
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        """Deny delete permission."""
+        return False
\ No newline at end of file
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index 193ba76d3..e220a449a 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -1,9 +1,9 @@
 """ModelAdmin classes for parsed SSP data files."""
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
+from .mixins import ExportCsvMixin, SttCodeMixin, ReadOnlyAdmin
 
 
-class SSP_M1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M1 data files."""
 
     actions = ["export_as_csv"]
@@ -30,7 +30,7 @@ class SSP_M1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class SSP_M2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M2 data files."""
 
     actions = ["export_as_csv"]
@@ -52,7 +52,7 @@ class SSP_M2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class SSP_M3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -73,7 +73,7 @@ class SSP_M3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -94,7 +94,7 @@ class SSP_M4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M3 data files."""
 
     actions = ["export_as_csv"]
@@ -115,7 +115,7 @@ class SSP_M5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M6 data files."""
 
     actions = ["export_as_csv"]
@@ -137,7 +137,7 @@ class SSP_M6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR'
     ]
 
-class SSP_M7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class SSP_M7Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed M7 data files."""
 
     actions = ["export_as_csv"]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 3a9521396..9b27dd1c3 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,9 +1,9 @@
 """ModelAdmin classes for parsed TANF data files."""
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
+from .mixins import ExportCsvMixin, ReadOnlyAdmin, SttCodeMixin
 
 
-class TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T1 data files."""
 
     actions = ["export_as_csv"]
@@ -30,7 +30,7 @@ class TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T2 data files."""
 
     actions = ["export_as_csv"]
@@ -52,7 +52,7 @@ class TANF_T2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T3 data files."""
 
     actions = ["export_as_csv"]
@@ -74,7 +74,7 @@ class TANF_T3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T4 data files."""
 
     actions = ["export_as_csv"]
@@ -96,7 +96,7 @@ class TANF_T4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T5 data files."""
 
     actions = ["export_as_csv"]
@@ -118,7 +118,7 @@ class TANF_T5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T6 data files."""
 
     actions = ["export_as_csv"]
@@ -141,7 +141,7 @@ class TANF_T6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class TANF_T7Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed T7 data files."""
 
     actions = ["export_as_csv"]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index c8a1782c1..d061ddd28 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -1,9 +1,9 @@
 """ModelAdmin classes for parsed TRIBAL data files."""
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
+from .mixins import ExportCsvMixin, SttCodeMixin, ReadOnlyAdmin
 
 
-class Tribal_TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T1 data files."""
 
     actions = ["export_as_csv"]
@@ -30,7 +30,7 @@ class Tribal_TANF_T1Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     ]
 
 
-class Tribal_TANF_T2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T2 data files."""
 
     actions = ["export_as_csv"]
@@ -52,7 +52,7 @@ class Tribal_TANF_T2Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
     ]
 
 
-class Tribal_TANF_T3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T3 data files."""
 
     actions = ["export_as_csv"]
@@ -73,7 +73,7 @@ class Tribal_TANF_T3Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T4 data files."""
 
     actions = ["export_as_csv"]
@@ -93,7 +93,7 @@ class Tribal_TANF_T4Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
         STTFilter,
         'RPT_MONTH_YEAR',
     ]
-class Tribal_TANF_T5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal_T5 data files."""
 
     actions = ["export_as_csv"]
@@ -114,7 +114,7 @@ class Tribal_TANF_T5Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal T6 data files."""
 
     actions = ["export_as_csv"]
@@ -136,7 +136,7 @@ class Tribal_TANF_T6Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin
         'RPT_MONTH_YEAR'
     ]
 
-class Tribal_TANF_T7Admin(DisableDeleteActionMixin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T7Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     """ModelAdmin class for parsed Tribal T7 data files."""
 
     actions = ["export_as_csv"]

From d8c5619ed6489dbcedb1ff741184925ba5a5afc6 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 3 Apr 2024 14:53:49 -0600
Subject: [PATCH 039/127] - Removing unnecessary method

---
 .../tdpservice/search_indexes/admin/filters.py        | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index c0b4a5d6b..15fdb1639 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -127,17 +127,6 @@ def lookups(self, request, model_admin):
 
         return options
 
-    def choices(self, cl):
-        """Update query string based on selection."""
-        for lookup, title in self.lookup_choices:
-            yield {
-                'selected': self.value() == lookup,
-                'query_string': cl.get_query_string({
-                    self.parameter_name: lookup,
-                }, []),
-                'display': title,
-            }
-
     def queryset(self, request, queryset):
         """Filter queryset to show records matching selected fiscal year."""
         if self.value() is not None and queryset.exists():

From d4dac92b4741af5df6fe853f4da10ccaa75b0633 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 3 Apr 2024 16:18:29 -0600
Subject: [PATCH 040/127] - fix lint

---
 tdrs-backend/tdpservice/search_indexes/admin/filters.py | 2 +-
 tdrs-backend/tdpservice/search_indexes/admin/mixins.py  | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index 15fdb1639..d5205bd55 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -77,7 +77,7 @@ def queryset(self, request, queryset):
         if self.value() is None and queryset.exists():
             datafiles = []
             for record in queryset.order_by("datafile__stt__stt_code", "-datafile__id")\
-                .distinct("datafile__stt__stt_code"):
+                    .distinct("datafile__stt__stt_code"):
                 datafiles.append(record.datafile)
             return queryset.filter(datafile__in=datafiles)
         return queryset
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index 2c751f337..e51c56fc3 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -81,14 +81,13 @@ class AdminModelMixin(admin.ModelAdmin):
 
 class ReadOnlyAdmin(AdminModelMixin):
     """Force ModelAdmin to be READ only."""
+
     readonly_fields = []
 
     def get_readonly_fields(self, request, obj=None):
         """Force all fields to read only."""
-        return list(self.readonly_fields) + \
-               [field.name for field in obj._meta.fields] + \
-               [field.name for field in obj._meta.many_to_many]
-
+        return list(self.readonly_fields) + [field.name for field in obj._meta.fields] +\
+            [field.name for field in obj._meta.many_to_many]
 
     def has_add_permission(self, request):
         """Deny add permisison."""
@@ -96,4 +95,4 @@ def has_add_permission(self, request):
 
     def has_delete_permission(self, request, obj=None):
         """Deny delete permission."""
-        return False
\ No newline at end of file
+        return False

From a9407d4b8cc971ffa593237b6066f20d1a07b92c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 5 Apr 2024 08:54:07 -0600
Subject: [PATCH 041/127] - Aggregate Mixin classes - Aggregate overrides to
 respective classes

---
 .../tdpservice/search_indexes/admin/mixins.py |  9 ++++-
 .../tdpservice/search_indexes/admin/ssp.py    | 37 ++++---------------
 .../tdpservice/search_indexes/admin/tanf.py   | 37 ++++---------------
 .../tdpservice/search_indexes/admin/tribal.py | 36 ++++--------------
 4 files changed, 31 insertions(+), 88 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index e51c56fc3..e856fcbb9 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -74,12 +74,17 @@ def stt_code(self, obj):
 
 
 class AdminModelMixin(admin.ModelAdmin):
-    """Base class for all mixin classes needing to modify ModelAdmin methods."""
+    """Base class for all mixin classes needing to modify ModelAdmin methods. Needed to satisfy Python MRO."""
 
     pass
 
+class CsvExportAdminMixin(AdminModelMixin, ExportCsvMixin, SttCodeMixin):
+    """Class to encapsulate CSV related mixins."""
+    actions = ["export_as_csv"]
+    ordering = ['datafile__stt__stt_code']
 
-class ReadOnlyAdmin(AdminModelMixin):
+
+class ReadOnlyAdminMixin(AdminModelMixin):
     """Force ModelAdmin to be READ only."""
 
     readonly_fields = []
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index e220a449a..e9f389640 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -1,14 +1,11 @@
 """ModelAdmin classes for parsed SSP data files."""
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import ExportCsvMixin, SttCodeMixin, ReadOnlyAdmin
+from .mixins import CsvExportAdminMixin, ReadOnlyAdminMixin
 
 
-class SSP_M1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed M1 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -30,12 +27,9 @@ class SSP_M1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class SSP_M2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed M2 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -52,12 +46,9 @@ class SSP_M2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class SSP_M3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed M3 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -73,12 +64,9 @@ class SSP_M3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed M3 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -94,12 +82,9 @@ class SSP_M4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed M3 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -115,12 +100,9 @@ class SSP_M5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR',
     ]
 
-class SSP_M6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed M6 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
@@ -137,12 +119,9 @@ class SSP_M6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR'
     ]
 
-class SSP_M7Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class SSP_M7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed M7 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 9b27dd1c3..0562e174c 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -1,14 +1,11 @@
 """ModelAdmin classes for parsed TANF data files."""
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import ExportCsvMixin, ReadOnlyAdmin, SttCodeMixin
+from .mixins import CsvExportAdminMixin, ReadOnlyAdminMixin
 
 
-class TANF_T1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed T1 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -30,12 +27,9 @@ class TANF_T1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed T2 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -52,12 +46,9 @@ class TANF_T2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed T3 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -74,12 +65,9 @@ class TANF_T3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed T4 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -96,12 +84,9 @@ class TANF_T4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed T5 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -118,12 +103,9 @@ class TANF_T5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed T6 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
@@ -141,12 +123,9 @@ class TANF_T6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class TANF_T7Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed T7 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index d061ddd28..a690de366 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -1,14 +1,11 @@
 """ModelAdmin classes for parsed TRIBAL data files."""
 from .filters import CreationDateFilter, FiscalPeriodFilter, STTFilter
-from .mixins import ExportCsvMixin, SttCodeMixin, ReadOnlyAdmin
+from .mixins import CsvExportAdminMixin, ReadOnlyAdminMixin
 
 
-class Tribal_TANF_T1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal_T1 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -30,12 +27,9 @@ class Tribal_TANF_T1Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class Tribal_TANF_T2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal_T2 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -52,12 +46,9 @@ class Tribal_TANF_T2Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
     ]
 
 
-class Tribal_TANF_T3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal_T3 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -73,12 +64,9 @@ class Tribal_TANF_T3Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal_T4 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -93,12 +81,9 @@ class Tribal_TANF_T4Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
         STTFilter,
         'RPT_MONTH_YEAR',
     ]
-class Tribal_TANF_T5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal_T5 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
@@ -114,12 +99,9 @@ class Tribal_TANF_T5Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR',
     ]
 
-class Tribal_TANF_T6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal T6 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
-
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
@@ -136,11 +118,9 @@ class Tribal_TANF_T6Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
         'RPT_MONTH_YEAR'
     ]
 
-class Tribal_TANF_T7Admin(ReadOnlyAdmin, ExportCsvMixin, SttCodeMixin):
+class Tribal_TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal T7 data files."""
 
-    actions = ["export_as_csv"]
-    ordering = ['datafile__stt__stt_code']
 
     list_display = [
         'RecordType',

From 62b17225ca6b33819a2cc5d1c88be0e2740b063b Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 5 Apr 2024 09:23:59 -0600
Subject: [PATCH 042/127] - lint

---
 tdrs-backend/tdpservice/search_indexes/admin/mixins.py | 1 +
 tdrs-backend/tdpservice/search_indexes/admin/tribal.py | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
index e856fcbb9..758b5eab7 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/mixins.py
@@ -80,6 +80,7 @@ class AdminModelMixin(admin.ModelAdmin):
 
 class CsvExportAdminMixin(AdminModelMixin, ExportCsvMixin, SttCodeMixin):
     """Class to encapsulate CSV related mixins."""
+
     actions = ["export_as_csv"]
     ordering = ['datafile__stt__stt_code']
 
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index a690de366..6988a59b4 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -121,7 +121,6 @@ class Tribal_TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
 class Tribal_TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal T7 data files."""
 
-
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',

From c3368ee3e53e5895c65e158a69090c57d4327321 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 8 Apr 2024 11:34:12 -0600
Subject: [PATCH 043/127] - Update tests

---
 .../tdpservice/parsers/test/test_validators.py         | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py
index ce25840d9..ecc8e97e8 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_validators.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py
@@ -123,9 +123,9 @@ def test_validate__FAM_AFF__SSN():
 def test_quarterIsValid(value, valid):
     """Test `quarterIsValid`."""
     val = validators.quarterIsValid()
-    result = val(value)
+    result = val(value, RowSchema(), "friendly_name", "item_no")
 
-    errorText = None if valid else f"{value[-1:]} is not a valid quarter."
+    errorText = None if valid else f"T1: {value[-1:]} is not a valid quarter."
     assert result == (valid, errorText)
 
 def test_validateSSN():
@@ -472,7 +472,7 @@ def test_notEmpty_returns_nonexistent_substring():
 def test_quarterIsValid_returns_true_if_valid(test_input):
     """Test `quarterIsValid` gives a valid result for values 1-4."""
     validator = validators.quarterIsValid()
-    is_valid, error = validator(test_input)
+    is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no")
 
     assert is_valid is True
     assert error is None
@@ -482,10 +482,10 @@ def test_quarterIsValid_returns_true_if_valid(test_input):
 def test_quarterIsValid_returns_false_if_invalid(test_input):
     """Test `quarterIsValid` gives an invalid result for values not 1-4."""
     validator = validators.quarterIsValid()
-    is_valid, error = validator(test_input)
+    is_valid, error = validator(test_input, RowSchema(), "friendly_name", "item_no")
 
     assert is_valid is False
-    assert error == f"{test_input} is not a valid quarter."
+    assert error == f"T1: {test_input} is not a valid quarter."
 
 @pytest.mark.usefixtures('db')
 class TestCat3ValidatorsBase:

From 83ab2f17b25f97474751a8eb0669ebb12c84f97c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 8 Apr 2024 11:38:44 -0600
Subject: [PATCH 044/127] - Fix parse test

---
 tdrs-backend/tdpservice/parsers/test/test_parse.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 50d80cb63..b849dd52c 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1675,12 +1675,9 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
     parse.parse_datafile(tribal_section_4_bad_quarter, dfs)
     parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter)
 
-    for error in parser_errors:
-        print(f"{error}")
-
     assert parser_errors.count() == 2
 
-    msg = "Reporting month year None does not match file reporting year:2020, quarter:Q1."
+    msg = "T7: Reporting month year None does not match file reporting year:2020, quarter:Q1."
     for error in parser_errors:
         assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK
 

From 6a09786c29e692797ec855a074fe73b2f31e8fe6 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Tue, 9 Apr 2024 10:52:03 -0600
Subject: [PATCH 045/127] - Add cat1 validator for calendar quarter - Update
 test

---
 .../tdpservice/parsers/schema_defs/ssp/m6.py  |  3 +++
 .../tdpservice/parsers/schema_defs/ssp/m7.py  |  1 +
 .../tdpservice/parsers/schema_defs/tanf/t6.py |  3 +++
 .../tdpservice/parsers/schema_defs/tanf/t7.py |  1 +
 .../parsers/schema_defs/tribal_tanf/t6.py     |  3 +++
 .../parsers/schema_defs/tribal_tanf/t7.py     |  1 +
 .../tribal_section_4_fake_bad_quarter.txt     |  4 ++--
 .../tdpservice/parsers/test/test_parse.py     | 17 +++++------------
 .../parsers/test/test_validators.py           | 19 +++++++++++++++++++
 tdrs-backend/tdpservice/parsers/validators.py | 12 ++++++++++++
 10 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
index 979ca091e..e85bc8fb1 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py
@@ -13,6 +13,7 @@
     preparsing_validators=[
         validators.recordHasLength(259),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -177,6 +178,7 @@
     preparsing_validators=[
         validators.recordHasLength(259),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -341,6 +343,7 @@
     preparsing_validators=[
         validators.recordHasLength(259),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
index 8d6dc85fb..8d6664a43 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py
@@ -27,6 +27,7 @@
                 validators.notEmpty(0, 7),
                 validators.notEmpty(validator_index, validator_index + 24),
                 validators.field_year_month_with_header_year_quarter(),
+                validators.calendarQuarterIsValid(2, 7),
             ],
             postparsing_validators=[],
             fields=[
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
index 6d94e5255..e2e3b1a7f 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t6.py
@@ -13,6 +13,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -233,6 +234,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
@@ -447,6 +449,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual(
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
index 42fe3ec2a..581cd3883 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t7.py
@@ -27,6 +27,7 @@
                 validators.notEmpty(0, 7),
                 validators.notEmpty(validator_index, validator_index + 24),
                 validators.field_year_month_with_header_year_quarter(),
+                validators.calendarQuarterIsValid(2, 7),
             ],
             postparsing_validators=[],
             fields=[
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
index 50b7bb123..d32ed3693 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t6.py
@@ -13,6 +13,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]),
@@ -221,6 +222,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]),
@@ -423,6 +425,7 @@
     preparsing_validators=[
         validators.recordHasLength(379),
         validators.field_year_month_with_header_year_quarter(),
+        validators.calendarQuarterIsValid(2, 7),
     ],
     postparsing_validators=[
         validators.sumIsEqual("NUM_APPLICATIONS", ["NUM_APPROVED", "NUM_DENIED"]),
diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
index 164fbf322..dfbebb3bf 100644
--- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
+++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t7.py
@@ -27,6 +27,7 @@
                 validators.notEmpty(0, 7),
                 validators.notEmpty(validator_index, validator_index + 24),
                 validators.field_year_month_with_header_year_quarter(),
+                validators.calendarQuarterIsValid(2, 7),
             ],
             postparsing_validators=[],
             fields=[
diff --git a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt
index c4b6851d7..e8ad6c8f5 100644
--- a/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt
+++ b/tdrs-backend/tdpservice/parsers/test/data/tribal_section_4_fake_bad_quarter.txt
@@ -1,3 +1,3 @@
-HEADER20194S00142TAN1EU
+HEADER20204S00142TAN1EU
 T72020 101006853700680540068454103000312400037850003180104000347400036460003583106000044600004360000325299000506200036070003385202000039100002740000499                                                                                                
-TRAILER0000001         
\ No newline at end of file
+TRAILER0000001         
diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index b849dd52c..2e98be41a 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1668,22 +1668,15 @@ def tribal_section_4_bad_quarter(stt_user, stt):
 @pytest.mark.django_db()
 def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
     """Test handling invalid quarter value that raises a ValueError exception."""
-    tribal_section_4_bad_quarter.year = 2020
+    tribal_section_4_bad_quarter.year = 2021
     tribal_section_4_bad_quarter.quarter = 'Q1'
     dfs.datafile = tribal_section_4_bad_quarter
 
     parse.parse_datafile(tribal_section_4_bad_quarter, dfs)
-    parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter)
+    parser_errors = ParserError.objects.filter(file=tribal_section_4_bad_quarter).order_by('id')
 
-    assert parser_errors.count() == 2
+    assert parser_errors.count() == 3
 
-    msg = "T7: Reporting month year None does not match file reporting year:2020, quarter:Q1."
-    for error in parser_errors:
-        assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK
+    parser_errors.first().error_message == "T7: 2020  is an invalid value for the CALENDAR_QUARTER field."
 
-        if (error.field_name is None):
-            assert error.error_message == "No records created."
-        elif (error.field_name == "Record_Type"):
-            assert error.error_message == msg
-        else:
-            assert False, f"Unexpected field name {error.field_name}"
+    Tribal_TANF_T7.objects.count() == 0
diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py
index ecc8e97e8..350cde40f 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_validators.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py
@@ -487,6 +487,25 @@ def test_quarterIsValid_returns_false_if_invalid(test_input):
     assert is_valid is False
     assert error == f"T1: {test_input} is not a valid quarter."
 
+@pytest.mark.parametrize("value", ["T72020 ", "T720194", "T720200", "T720207", "T72020$"])
+def test_calendarQuarterIsValid_returns_invalid(value):
+    """Test `calendarQuarterIsValid` returns false on invalid input."""
+    val = validators.calendarQuarterIsValid(2, 7)
+    is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no")
+
+    assert is_valid is False
+    assert error_msg == f"T1: {value[2:7]} is an invalid value for the CALENDAR_QUARTER field."
+
+
+@pytest.mark.parametrize("value", ["T720201", "T720202", "T720203", "T720204"])
+def test_calendarQuarterIsValid_returns_valid(value):
+    """Test `calendarQuarterIsValid` returns false on invalid input."""
+    val = validators.calendarQuarterIsValid(2, 7)
+    is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no")
+
+    assert is_valid is True
+    assert error_msg is None
+
 @pytest.mark.usefixtures('db')
 class TestCat3ValidatorsBase:
     """A base test class for tests that evaluate category three validators."""
diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py
index 7fa36c581..2b1ce0ff5 100644
--- a/tdrs-backend/tdpservice/parsers/validators.py
+++ b/tdrs-backend/tdpservice/parsers/validators.py
@@ -370,6 +370,18 @@ def caseNumberNotEmpty(start=0, end=None):
     )
 
 
+def calendarQuarterIsValid(start=0, end=None):
+    """Validate that the calendar quarter value is valid."""
+    return make_validator(
+        lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 \
+              and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5,
+        lambda value,
+        row_schema,
+        friendly_name,
+        item_num: f"{row_schema.record_type}: {value[start:end]} is an invalid value for the CALENDAR_QUARTER field.",
+    )
+
+
 def isEmpty(start=0, end=None):
     """Validate that string value is only blanks."""
     return make_validator(

From b82621c9bc5347ebb9201b34fd5a903514f6a3ba Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Tue, 9 Apr 2024 10:54:54 -0600
Subject: [PATCH 046/127] - fix lint

---
 tdrs-backend/tdpservice/parsers/validators.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py
index 2b1ce0ff5..463895e63 100644
--- a/tdrs-backend/tdpservice/parsers/validators.py
+++ b/tdrs-backend/tdpservice/parsers/validators.py
@@ -373,8 +373,8 @@ def caseNumberNotEmpty(start=0, end=None):
 def calendarQuarterIsValid(start=0, end=None):
     """Validate that the calendar quarter value is valid."""
     return make_validator(
-        lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020 \
-              and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5,
+        lambda value: value[start:end].isnumeric() and int(value[start:end - 1]) >= 2020
+        and int(value[end - 1:end]) > 0 and int(value[end - 1:end]) < 5,
         lambda value,
         row_schema,
         friendly_name,

From 6a23258df6badf1d5573e7656d594a56f9d80a5e Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 10 Apr 2024 10:50:35 -0600
Subject: [PATCH 047/127] - remove trailing `/`

---
 scripts/deploy-backend.sh             | 4 ++--
 tdrs-backend/clamav-router/nginx.conf | 7 ++++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh
index 884f3eb63..0dc5ba5b4 100755
--- a/scripts/deploy-backend.sh
+++ b/scripts/deploy-backend.sh
@@ -131,10 +131,10 @@ update_backend()
     cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL"
     
     if [ "$CF_SPACE" = "tanf-prod" ]; then
-      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan/"
+      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan"
     else
       # Add environment varilables for clamav
-      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan/"
+      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan"
     fi
 
     if [ "$1" = "rolling" ] ; then
diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf
index d0ce5ae0d..bd75526c5 100644
--- a/tdrs-backend/clamav-router/nginx.conf
+++ b/tdrs-backend/clamav-router/nginx.conf
@@ -20,12 +20,13 @@ http{
                             '"proxy_add_x_forwarded_for": $proxy_add_x_forwarded_for, '
                            '"http_referer": $http_referer, '
                            '"http_user_agent": $http_user_agent, '
-                           '"cookies=$http_cookie;"  "server=$server_name" "http_host=$http_host"';
+                           '"cookies=$http_cookie;"  "server=$server_name" "http_host=$http_host"'
+                           ' Proxy: "$proxy_host" "$upstream_addr"';
 
     server {
         listen {{port}};
         client_max_body_size 100m;
-        location ~* ^/scan/(.*)$ {
+        location ~* ^/scan(.*)$ {
             set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;
@@ -35,7 +36,7 @@ http{
     server {
         listen 9000;
         client_max_body_size 100m;
-        location ~* ^/scan/(.*)$ {
+        location ~* ^/scan(.*)$ {
             set $clamav http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass                      $clamav$1$is_args$args;
             proxy_pass_request_headers      on;

From 2fbc96d85ca23efd7534a5673cc4d1e3d9a700ec Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 08:52:17 -0600
Subject: [PATCH 048/127] - update run-task structure

---
 .circleci/owasp/jobs.yml | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 31fbcb734..64d35f121 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -88,20 +88,9 @@
 
             # These environment variables are exported to Circle CI's BASH_ENV
             # by the zap-scanner.sh script for each respective app target.
-            CMD_ARGS=(
-              "$CIRCLE_BUILD_NUM"
-              --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0}
-              --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0}
-              --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0}
-              --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0}
-              --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0}
-              --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0}
-              --project-slug $PROJECT_SLUG
-            )
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan ${CMD_ARGS[@]}"
+            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
+
             # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-<< parameters.target_env >> \
-              --command "$CMD" \
-              --name nightly-owasp-scan
+            cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan

From 7728ba3ae1ed47bdb0de55752c4a31f64c98151f Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 08:53:38 -0600
Subject: [PATCH 049/127] - overriding workflow

---
 .circleci/build-and-test/workflows.yml | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml
index 7c0e559b0..109f910ae 100644
--- a/.circleci/build-and-test/workflows.yml
+++ b/.circleci/build-and-test/workflows.yml
@@ -3,15 +3,8 @@
     when: << pipeline.parameters.build_and_test_all >>
     jobs:
       - secrets-check
-      - test-backend:
-          requires:
-            - secrets-check      
-      - test-frontend:
-          requires:
-            - secrets-check
-      - test-e2e:
-          requires:
-            - secrets-check
+      - nightly-owasp-scan:
+          target_env: develop
 
   ci-build-and-test-all:
     jobs:
@@ -30,7 +23,7 @@
                 - master
                 - /^release.*/
           requires:
-            - secrets-check      
+            - secrets-check
       - test-frontend:
           filters:
             branches:

From ccbc4f99fa43a6bf71f491fc71d2e8ab4cfb4078 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 08:59:04 -0600
Subject: [PATCH 050/127] - adding todo reminder

---
 .circleci/build-and-test/workflows.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml
index 109f910ae..de60699fd 100644
--- a/.circleci/build-and-test/workflows.yml
+++ b/.circleci/build-and-test/workflows.yml
@@ -3,6 +3,7 @@
     when: << pipeline.parameters.build_and_test_all >>
     jobs:
       - secrets-check
+      # TODO: Revert back to tests after review
       - nightly-owasp-scan:
           target_env: develop
 

From 943fc6948ac5d535b0acff0d0f054b08c5ee0f32 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 10:58:47 -0600
Subject: [PATCH 051/127] - removing wait

---
 .circleci/owasp/jobs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 64d35f121..c3d1af59b 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -93,4 +93,4 @@
             CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
 
             # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan
+            cf run-task tdp-backend-<< parameters.target_env >> --command "$CMD" --name nightly-owasp-scan

From fdfed7ea34f55921a803f26829ceb1bc65cf76c3 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 11:18:00 -0600
Subject: [PATCH 052/127] - Breakup args

---
 .circleci/owasp/jobs.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index c3d1af59b..338e49270 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -90,7 +90,9 @@
             # by the zap-scanner.sh script for each respective app target.
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
+            BACKEND="--backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0}"
+            FRONTEND="--frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0}"
+            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM $BACKEND $FRONTEND --project-slug $PROJECT_SLUG"
 
             # Submit a CF Task for execution that will run the necessary command
             cf run-task tdp-backend-<< parameters.target_env >> --command "$CMD" --name nightly-owasp-scan

From 5c0b00e07dfb4d92a783bc420f9372a4b54e4d20 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 11:23:39 -0600
Subject: [PATCH 053/127] - Adding verbosity to zap script - testing staging

---
 .circleci/build-and-test/workflows.yml | 2 +-
 scripts/zap-scanner.sh                 | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml
index de60699fd..1ecb5e9ae 100644
--- a/.circleci/build-and-test/workflows.yml
+++ b/.circleci/build-and-test/workflows.yml
@@ -5,7 +5,7 @@
       - secrets-check
       # TODO: Revert back to tests after review
       - nightly-owasp-scan:
-          target_env: develop
+          target_env: staging
 
   ci-build-and-test-all:
     jobs:
diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index d66f12371..ce351cb00 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # pipefail is needed to correctly carry over the exit code from zap-full-scan.py
-set -o pipefail
+set -uxo pipefail
 
 TARGET=$1
 ENVIRONMENT=$2
@@ -39,7 +39,7 @@ fi
 cd "$TARGET_DIR" || exit 2
 
 
-if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then 
+if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then
   docker network create external-net
 fi
 

From b77392eff5d878b7f8e8596a7cb7e28c70d64b57 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 14:06:57 -0600
Subject: [PATCH 054/127] - debugging

---
 .circleci/build-and-test/jobs.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml
index c143ea8d7..8fbefebdb 100644
--- a/.circleci/build-and-test/jobs.yml
+++ b/.circleci/build-and-test/jobs.yml
@@ -53,12 +53,12 @@
       - disable-npm-audit
       - install-nodejs-packages:
           app-dir: tdrs-frontend
-      - run:
-          name: Wait for backend to become available
-          command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \
-              "PATH=$PATH:/home/zap/.local/bin &&
-               pip install wait-for-it &&
-               wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
+    #   - run:
+    #       name: Wait for backend to become available
+    #       command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \
+    #           "PATH=$PATH:/home/zap/.local/bin &&
+    #            pip install wait-for-it &&
+    #            wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
       - run:
           name: apply the migrations
           command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate"

From 05e979cbba8750f7ee4be5a14ced83df8de914bb Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 14:09:34 -0600
Subject: [PATCH 055/127] Revert "- debugging"

This reverts commit b77392eff5d878b7f8e8596a7cb7e28c70d64b57.
---
 .circleci/build-and-test/jobs.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml
index 8fbefebdb..c143ea8d7 100644
--- a/.circleci/build-and-test/jobs.yml
+++ b/.circleci/build-and-test/jobs.yml
@@ -53,12 +53,12 @@
       - disable-npm-audit
       - install-nodejs-packages:
           app-dir: tdrs-frontend
-    #   - run:
-    #       name: Wait for backend to become available
-    #       command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \
-    #           "PATH=$PATH:/home/zap/.local/bin &&
-    #            pip install wait-for-it &&
-    #            wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
+      - run:
+          name: Wait for backend to become available
+          command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \
+              "PATH=$PATH:/home/zap/.local/bin &&
+               pip install wait-for-it &&
+               wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
       - run:
           name: apply the migrations
           command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate"

From de81b27ab2b9c3a8d8e6751b1864747226ed9aa8 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 14:36:04 -0600
Subject: [PATCH 056/127] - commenting failing piece

---
 scripts/zap-scanner.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index ce351cb00..e2428b373 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -44,10 +44,10 @@ if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then
 fi
 
 # Ensure the APP_URL is reachable from the zaproxy container
-if ! docker-compose run --rm zaproxy curl -Is "$APP_URL" > /dev/null 2>&1; then
-  echo "Target application at $APP_URL is unreachable by ZAP scanner"
-  exit 3
-fi
+# if ! docker-compose run --rm zaproxy curl -Is "$APP_URL" > /dev/null 2>&1; then
+#   echo "Target application at $APP_URL is unreachable by ZAP scanner"
+#   exit 3
+# fi
 
 echo "================== OWASP ZAP tests =================="
 

From 236ab9ef5d112fb5eff4a371d9cd992729e921fa Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 15:44:22 -0600
Subject: [PATCH 057/127] - debug

---
 .circleci/owasp/jobs.yml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 338e49270..bdea722e4 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -72,14 +72,14 @@
           cf-space: <<parameters.cf_space>>
           cf-org: <<parameters.cf_org>>
           cf-username: <<parameters.cf_username>>
-      - run-owasp-scan:
-          environment: nightly
-          target: backend
-          target_env: <<parameters.target_env>>
-      - run-owasp-scan:
-          environment: nightly
-          target: frontend
-          target_env: <<parameters.target_env>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: backend
+      #     target_env: <<parameters.target_env>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: frontend
+      #     target_env: <<parameters.target_env>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |
@@ -93,6 +93,7 @@
             BACKEND="--backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0}"
             FRONTEND="--frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0}"
             CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM $BACKEND $FRONTEND --project-slug $PROJECT_SLUG"
+            echo $CMD
 
             # Submit a CF Task for execution that will run the necessary command
             cf run-task tdp-backend-<< parameters.target_env >> --command "$CMD" --name nightly-owasp-scan

From 11902e48264ca3a0bfd5b158e8a6d19f21943026 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:02:47 -0600
Subject: [PATCH 058/127] - testing whether scripts exist

---
 scripts/zap-scanner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index e2428b373..237e216d1 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -171,7 +171,7 @@ else
 fi
 
 # Run the ZAP full scan and store output for further processing if needed.
-ZAP_OUTPUT=$(docker-compose run --rm zaproxy "$ZAP_SCRIPT" "${ZAP_ARGS[@]}" | tee /dev/tty)
+ZAP_OUTPUT=$(docker-compose run --rm zaproxy bash -c "pwd && ls" | tee /dev/tty)
 ZAP_EXIT=$?
 
 if [ "$ZAP_EXIT" -eq 0 ]; then

From 2f814705d4ba2de9f25b81ff1c17501f61b62961 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:05:08 -0600
Subject: [PATCH 059/127] - enable scan

---
 .circleci/owasp/jobs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index bdea722e4..30735cea2 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -72,10 +72,10 @@
           cf-space: <<parameters.cf_space>>
           cf-org: <<parameters.cf_org>>
           cf-username: <<parameters.cf_username>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: backend
-      #     target_env: <<parameters.target_env>>
+      - run-owasp-scan:
+          environment: nightly
+          target: backend
+          target_env: <<parameters.target_env>>
       # - run-owasp-scan:
       #     environment: nightly
       #     target: frontend

From 5e6d03b2d4c6dafa3c96552a3b2474982e938b16 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:18:42 -0600
Subject: [PATCH 060/127] - debug

---
 scripts/zap-scanner.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 237e216d1..1955b5c1e 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -171,7 +171,8 @@ else
 fi
 
 # Run the ZAP full scan and store output for further processing if needed.
-ZAP_OUTPUT=$(docker-compose run --rm zaproxy bash -c "pwd && ls" | tee /dev/tty)
+ZAP_OUTPUT=$(docker-compose run --rm zaproxy "$ZAP_SCRIPT" "${ZAP_ARGS[@]}" | tee /dev/tty)
+echo $ZAP_OUTPUT
 ZAP_EXIT=$?
 
 if [ "$ZAP_EXIT" -eq 0 ]; then

From 10f65604503e491b32a75ffbfe2c71f5aa5b552e Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:23:29 -0600
Subject: [PATCH 061/127] - remove exit

---
 scripts/zap-scanner.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 1955b5c1e..e4276383c 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -44,10 +44,9 @@ if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then
 fi
 
 # Ensure the APP_URL is reachable from the zaproxy container
-# if ! docker-compose run --rm zaproxy curl -Is "$APP_URL" > /dev/null 2>&1; then
-#   echo "Target application at $APP_URL is unreachable by ZAP scanner"
-#   exit 3
-# fi
+if ! docker-compose run --rm zaproxy curl -Is "$APP_URL" > /dev/null 2>&1; then
+  echo "Target application at $APP_URL is unreachable by ZAP scanner"
+fi
 
 echo "================== OWASP ZAP tests =================="
 

From efd4df424d76a009129108b177ad5892987f7d3c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:40:34 -0600
Subject: [PATCH 062/127] - Update zap image to a supported image

---
 tdrs-backend/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
index 80e746aca..53a70eccb 100644
--- a/tdrs-backend/docker-compose.yml
+++ b/tdrs-backend/docker-compose.yml
@@ -3,7 +3,7 @@ version: "3.4"
 
 services:
   zaproxy:
-    image: owasp/zap2docker-stable:2.13.0
+    image: softwaresecurityproject/zap-stable:2.13.0
     command: sleep 3600
     depends_on:
       - web

From f1e51588738dab8e8239fee8a25a2ff80ffc0d7e Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:44:35 -0600
Subject: [PATCH 063/127] - remove echo

---
 scripts/zap-scanner.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index e4276383c..4a7f81f87 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -171,7 +171,6 @@ fi
 
 # Run the ZAP full scan and store output for further processing if needed.
 ZAP_OUTPUT=$(docker-compose run --rm zaproxy "$ZAP_SCRIPT" "${ZAP_ARGS[@]}" | tee /dev/tty)
-echo $ZAP_OUTPUT
 ZAP_EXIT=$?
 
 if [ "$ZAP_EXIT" -eq 0 ]; then

From 046d995c594f5dd9e6d98c75ccf59dd0520ae626 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:57:07 -0600
Subject: [PATCH 064/127] - add exit back in

---
 scripts/zap-scanner.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 4a7f81f87..ce351cb00 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -46,6 +46,7 @@ fi
 # Ensure the APP_URL is reachable from the zaproxy container
 if ! docker-compose run --rm zaproxy curl -Is "$APP_URL" > /dev/null 2>&1; then
   echo "Target application at $APP_URL is unreachable by ZAP scanner"
+  exit 3
 fi
 
 echo "================== OWASP ZAP tests =================="

From 7d489b507e962045dd9feb6fbc62b915d1b2e46c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:57:24 -0600
Subject: [PATCH 065/127] - 2 minute crawling

---
 scripts/zap-scanner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index ce351cb00..4cb8810c4 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -137,7 +137,7 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(21\).enabled=true \
   -config spider.postform=true"
 # How long ZAP will crawl the app with the spider process
-ZAP_SPIDER_MINS=10
+ZAP_SPIDER_MINS=2
 
 ZAP_ARGS=(-t "$APP_URL" -m "$ZAP_SPIDER_MINS" -r "$REPORT_NAME" -z "$ZAP_CLI_OPTIONS")
 if [ -z ${CONFIG_FILE+x} ]; then

From 3ed5a80dff5a361000d8f7961257fa8f4baca0fd Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:58:17 -0600
Subject: [PATCH 066/127] Revert "- Breakup args"

This reverts commit fdfed7ea34f55921a803f26829ceb1bc65cf76c3.
---
 .circleci/owasp/jobs.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 30735cea2..8d5832310 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -90,9 +90,7 @@
             # by the zap-scanner.sh script for each respective app target.
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            BACKEND="--backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0}"
-            FRONTEND="--frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0}"
-            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM $BACKEND $FRONTEND --project-slug $PROJECT_SLUG"
+            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
             echo $CMD
 
             # Submit a CF Task for execution that will run the necessary command

From c0b246b95df605f7da911cf56ef054af7abd51e8 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 16:58:37 -0600
Subject: [PATCH 067/127] - all scans

---
 .circleci/owasp/jobs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 8d5832310..0eda7c4ba 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -76,10 +76,10 @@
           environment: nightly
           target: backend
           target_env: <<parameters.target_env>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: frontend
-      #     target_env: <<parameters.target_env>>
+      - run-owasp-scan:
+          environment: nightly
+          target: frontend
+          target_env: <<parameters.target_env>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |

From cce6cac4669920839e3c7a85356f7a36526fc625 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 11 Apr 2024 17:10:32 -0600
Subject: [PATCH 068/127] - comment out exit again

---
 scripts/zap-scanner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 4cb8810c4..4f81b027a 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -46,7 +46,7 @@ fi
 # Ensure the APP_URL is reachable from the zaproxy container
 if ! docker-compose run --rm zaproxy curl -Is "$APP_URL" > /dev/null 2>&1; then
   echo "Target application at $APP_URL is unreachable by ZAP scanner"
-  exit 3
+  # exit 3
 fi
 
 echo "================== OWASP ZAP tests =================="

From 5e8f427df3ca3a3e869afb2494ebc321b61afcaa Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 07:19:14 -0600
Subject: [PATCH 069/127] - disable backend

---
 .circleci/owasp/jobs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 0eda7c4ba..b0bce5c90 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -72,10 +72,10 @@
           cf-space: <<parameters.cf_space>>
           cf-org: <<parameters.cf_org>>
           cf-username: <<parameters.cf_username>>
-      - run-owasp-scan:
-          environment: nightly
-          target: backend
-          target_env: <<parameters.target_env>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: backend
+      #     target_env: <<parameters.target_env>>
       - run-owasp-scan:
           environment: nightly
           target: frontend

From c2ae8f7756ccb73db726537205ac809f2c8cd8fc Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 07:27:23 -0600
Subject: [PATCH 070/127] - removing cli opts

---
 scripts/zap-scanner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 4f81b027a..7b0aaf6e8 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -139,7 +139,7 @@ ZAP_CLI_OPTIONS="\
 # How long ZAP will crawl the app with the spider process
 ZAP_SPIDER_MINS=2
 
-ZAP_ARGS=(-t "$APP_URL" -m "$ZAP_SPIDER_MINS" -r "$REPORT_NAME" -z "$ZAP_CLI_OPTIONS")
+ZAP_ARGS=(-t "$APP_URL" -m "$ZAP_SPIDER_MINS" -r "$REPORT_NAME") # -z "$ZAP_CLI_OPTIONS")
 if [ -z ${CONFIG_FILE+x} ]; then
     echo "No config file, defaulting all alerts to WARN"
 else

From 559de612bc397a2b649ce96d235afca613e28de3 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 08:07:06 -0600
Subject: [PATCH 071/127] - enable all opts

---
 scripts/zap-scanner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 7b0aaf6e8..4f81b027a 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -139,7 +139,7 @@ ZAP_CLI_OPTIONS="\
 # How long ZAP will crawl the app with the spider process
 ZAP_SPIDER_MINS=2
 
-ZAP_ARGS=(-t "$APP_URL" -m "$ZAP_SPIDER_MINS" -r "$REPORT_NAME") # -z "$ZAP_CLI_OPTIONS")
+ZAP_ARGS=(-t "$APP_URL" -m "$ZAP_SPIDER_MINS" -r "$REPORT_NAME" -z "$ZAP_CLI_OPTIONS")
 if [ -z ${CONFIG_FILE+x} ]; then
     echo "No config file, defaulting all alerts to WARN"
 else

From 44bc2baa114b033157010191d0d6f452d121161b Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 08:28:40 -0600
Subject: [PATCH 072/127] - testing with no scan

---
 .circleci/owasp/jobs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index b0bce5c90..b5bd1f08c 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -76,10 +76,10 @@
       #     environment: nightly
       #     target: backend
       #     target_env: <<parameters.target_env>>
-      - run-owasp-scan:
-          environment: nightly
-          target: frontend
-          target_env: <<parameters.target_env>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: frontend
+      #     target_env: <<parameters.target_env>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |

From 317021c6eab936d7b014f842ae0584bc15d52bdc Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 08:32:08 -0600
Subject: [PATCH 073/127] - enable scan, force cmd params

---
 .circleci/owasp/jobs.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index b5bd1f08c..736235353 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -76,10 +76,10 @@
       #     environment: nightly
       #     target: backend
       #     target_env: <<parameters.target_env>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: frontend
-      #     target_env: <<parameters.target_env>>
+      - run-owasp-scan:
+          environment: nightly
+          target: frontend
+          target_env: <<parameters.target_env>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |
@@ -90,7 +90,7 @@
             # by the zap-scanner.sh script for each respective app target.
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
+            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count 0 --backend-warn-count 0 --backend-fail-count 0 --frontend-pass-count 0 --frontend-warn-count 0 --frontend-fail-count 0 --project-slug raft-tech/TANF-app"
             echo $CMD
 
             # Submit a CF Task for execution that will run the necessary command

From 17c3bdeaf214098510f124f9c8c16584dada9c9d Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 08:52:13 -0600
Subject: [PATCH 074/127] - testing a sleep

---
 .circleci/owasp/jobs.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 736235353..e99b80a39 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -90,8 +90,10 @@
             # by the zap-scanner.sh script for each respective app target.
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count 0 --backend-warn-count 0 --backend-fail-count 0 --frontend-pass-count 0 --frontend-warn-count 0 --frontend-fail-count 0 --project-slug raft-tech/TANF-app"
+            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
             echo $CMD
 
+            sleep 30
+
             # Submit a CF Task for execution that will run the necessary command
             cf run-task tdp-backend-<< parameters.target_env >> --command "$CMD" --name nightly-owasp-scan

From c011884e12cd66b3fdc837ded52ba75aac617717 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 08:53:01 -0600
Subject: [PATCH 075/127] - readding wait

---
 .circleci/owasp/jobs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index e99b80a39..a1faf102f 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -96,4 +96,4 @@
             sleep 30
 
             # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-<< parameters.target_env >> --command "$CMD" --name nightly-owasp-scan
+            cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan

From a8dddf5bdf41602c49417d79c347448a7eec03e7 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 09:11:59 -0600
Subject: [PATCH 076/127] - remove sleep

---
 .circleci/owasp/jobs.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index a1faf102f..b36faade8 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -93,7 +93,5 @@
             CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
             echo $CMD
 
-            sleep 30
-
             # Submit a CF Task for execution that will run the necessary command
             cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan

From b3c4295f2a934cb9db8212f5b9a8f97647c345b0 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 09:14:56 -0600
Subject: [PATCH 077/127] - add an echo

---
 scripts/zap-scanner.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 4f81b027a..679b42c92 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -198,4 +198,5 @@ if [ "$ENVIRONMENT" = "nightly" ]; then
     echo "export ZAP_${TARGET}_WARN_COUNT=$ZAP_WARN_COUNT"
     echo "export ZAP_${TARGET}_FAIL_COUNT=$ZAP_FAIL_COUNT"
   } >> "$BASH_ENV"
+  echo $BASH_ENV
 fi

From 6365965e6a037c269b3acdd5a889e4a202eb7b6f Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 09:24:00 -0600
Subject: [PATCH 078/127] - remove echo

---
 scripts/zap-scanner.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 679b42c92..4f81b027a 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -198,5 +198,4 @@ if [ "$ENVIRONMENT" = "nightly" ]; then
     echo "export ZAP_${TARGET}_WARN_COUNT=$ZAP_WARN_COUNT"
     echo "export ZAP_${TARGET}_FAIL_COUNT=$ZAP_FAIL_COUNT"
   } >> "$BASH_ENV"
-  echo $BASH_ENV
 fi

From ce82d12d7501e4e1512cc0dd5070cf697376e0b1 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 09:33:38 -0600
Subject: [PATCH 079/127] - abide by shellcheck

---
 .circleci/owasp/jobs.yml |  4 ++--
 scripts/temp.sh          | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 scripts/temp.sh

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index b36faade8..729bd316c 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -91,7 +91,7 @@
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
             CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
-            echo $CMD
+            echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan
+            cf run-task tdp-backend-staging --wait --command "$CMD" --name nightly-owasp-scan
diff --git a/scripts/temp.sh b/scripts/temp.sh
new file mode 100644
index 000000000..4cc8c981c
--- /dev/null
+++ b/scripts/temp.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Construct the project slug from the current branch name and user
+PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
+
+# These environment variables are exported to Circle CI's BASH_ENV
+# by the zap-scanner.sh script for each respective app target.
+# Evaluate the full command before passing it in so it doesn't
+# get improperly interpolated by Cloud.gov.
+CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
+echo "$CMD"
+
+# Submit a CF Task for execution that will run the necessary command
+cf run-task tdp-backend-staging --wait --command "$CMD" --name nightly-owasp-scan

From d2af94ca612fd61c2c4e8a17dc7f725275802ef9 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 09:34:03 -0600
Subject: [PATCH 080/127] - remove temp

---
 scripts/temp.sh | 13 -------------
 1 file changed, 13 deletions(-)
 delete mode 100644 scripts/temp.sh

diff --git a/scripts/temp.sh b/scripts/temp.sh
deleted file mode 100644
index 4cc8c981c..000000000
--- a/scripts/temp.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-# Construct the project slug from the current branch name and user
-PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-
-# These environment variables are exported to Circle CI's BASH_ENV
-# by the zap-scanner.sh script for each respective app target.
-# Evaluate the full command before passing it in so it doesn't
-# get improperly interpolated by Cloud.gov.
-CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
-echo "$CMD"
-
-# Submit a CF Task for execution that will run the necessary command
-cf run-task tdp-backend-staging --wait --command "$CMD" --name nightly-owasp-scan

From 62dfd5632d213d615051dc737a5e5b6c602ee057 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 09:49:06 -0600
Subject: [PATCH 081/127] - revert to old command style that abides to
 shellcheck

---
 .circleci/owasp/jobs.yml | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 729bd316c..373238268 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -88,10 +88,20 @@
 
             # These environment variables are exported to Circle CI's BASH_ENV
             # by the zap-scanner.sh script for each respective app target.
+            CMD_ARGS=(
+              "$CIRCLE_BUILD_NUM"
+              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
+              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
+              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
+              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
+              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
+              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+              --project-slug "$PROJECT_SLUG"
+            )
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
-            echo "$CMD"
-
+            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
             # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-staging --wait --command "$CMD" --name nightly-owasp-scan
+            cf run-task tdp-backend-<< parameters.target_env >> \
+              --command "$CMD" \
+              --name nightly-owasp-scan
\ No newline at end of file

From e9beaea6b1d01c9b105df230267668f0472a9c43 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 10:01:39 -0600
Subject: [PATCH 082/127] - disable scan

---
 .circleci/owasp/jobs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 373238268..264d06743 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -76,10 +76,10 @@
       #     environment: nightly
       #     target: backend
       #     target_env: <<parameters.target_env>>
-      - run-owasp-scan:
-          environment: nightly
-          target: frontend
-          target_env: <<parameters.target_env>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: frontend
+      #     target_env: <<parameters.target_env>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |

From 76c3049027b6a8236519006edf189108e5feb83f Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 10:04:06 -0600
Subject: [PATCH 083/127] - remove comment and change indent

---
 .circleci/owasp/jobs.yml | 207 +++++++++++++++++++--------------------
 1 file changed, 103 insertions(+), 104 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 264d06743..d40fd161c 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -1,107 +1,106 @@
-# jobs:
-  backend-owasp-scan:
-    executor: large-machine-executor
-    working_directory: ~/tdp-apps
-    steps:
-      - checkout
-      - docker-compose-check
-      - docker-compose-up-backend
-      - docker-compose-up-frontend
-      - run:
-          name: Wait for Django to become available
-          command: |
-            cd tdrs-backend;
-            docker-compose run --rm zaproxy bash -c \
-              "PATH=$PATH:/home/zap/.local/bin &&
-               pip install wait-for-it &&
-               wait-for-it --service http://web:8080 \
-                           --timeout 60 \
-                           -- echo \"Django is ready\""
-      - run-owasp-scan:
-          environment: circle
-          target: backend
+backend-owasp-scan:
+  executor: large-machine-executor
+  working_directory: ~/tdp-apps
+  steps:
+    - checkout
+    - docker-compose-check
+    - docker-compose-up-backend
+    - docker-compose-up-frontend
+    - run:
+        name: Wait for Django to become available
+        command: |
+          cd tdrs-backend;
+          docker-compose run --rm zaproxy bash -c \
+            "PATH=$PATH:/home/zap/.local/bin &&
+              pip install wait-for-it &&
+              wait-for-it --service http://web:8080 \
+                          --timeout 60 \
+                          -- echo \"Django is ready\""
+    - run-owasp-scan:
+        environment: circle
+        target: backend
 
-  frontend-owasp-scan:
-    executor: large-machine-executor
-    working_directory: ~/tdp-apps
-    steps:
-      - checkout
-      - docker-compose-check
-      - docker-compose-up-backend
-      - docker-compose-up-frontend
-      - run:
-          name: Wait for frontend to become available
-          command: |
-            cd tdrs-frontend;
-            docker-compose run --rm zaproxy bash -c \
-              "PATH=$PATH:/home/zap/.local/bin &&
-               pip install wait-for-it &&
-               wait-for-it --service http://tdp-frontend/ \
-                           --timeout 60 \
-                           -- echo \"Frontend is ready\""
-      - run-owasp-scan:
-          environment: circle
-          target: frontend
+frontend-owasp-scan:
+  executor: large-machine-executor
+  working_directory: ~/tdp-apps
+  steps:
+    - checkout
+    - docker-compose-check
+    - docker-compose-up-backend
+    - docker-compose-up-frontend
+    - run:
+        name: Wait for frontend to become available
+        command: |
+          cd tdrs-frontend;
+          docker-compose run --rm zaproxy bash -c \
+            "PATH=$PATH:/home/zap/.local/bin &&
+              pip install wait-for-it &&
+              wait-for-it --service http://tdp-frontend/ \
+                          --timeout 60 \
+                          -- echo \"Frontend is ready\""
+    - run-owasp-scan:
+        environment: circle
+        target: frontend
 
-  nightly-owasp-scan:
-    executor: large-machine-executor
-    working_directory: ~/tdp-apps
-    parameters:
-      cf_password:
-        type: string
-        default: CF_PASSWORD_STAGING
-      cf_username:
-        type: string
-        default: CF_USERNAME_STAGING
-      cf_space:
-        type: string
-        default: tanf-staging
-      cf_org:
-        type: string
-        default: "CF_ORG"
-      target_env:
-        type: enum
-        enum: [ "staging", "develop", "prod" ]
-    steps:
-      - checkout
-      - sudo-check
-      - cf-check
-      - docker-compose-check
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf_password>>
-          cf-space: <<parameters.cf_space>>
-          cf-org: <<parameters.cf_org>>
-          cf-username: <<parameters.cf_username>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: backend
-      #     target_env: <<parameters.target_env>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: frontend
-      #     target_env: <<parameters.target_env>>
-      - run:
-          name: Run post-processing task to record OWASP ZAP results
-          command: |
-            # Construct the project slug from the current branch name and user
-            PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
+nightly-owasp-scan:
+  executor: large-machine-executor
+  working_directory: ~/tdp-apps
+  parameters:
+    cf_password:
+      type: string
+      default: CF_PASSWORD_STAGING
+    cf_username:
+      type: string
+      default: CF_USERNAME_STAGING
+    cf_space:
+      type: string
+      default: tanf-staging
+    cf_org:
+      type: string
+      default: "CF_ORG"
+    target_env:
+      type: enum
+      enum: [ "staging", "develop", "prod" ]
+  steps:
+    - checkout
+    - sudo-check
+    - cf-check
+    - docker-compose-check
+    - login-cloud-dot-gov:
+        cf-password: <<parameters.cf_password>>
+        cf-space: <<parameters.cf_space>>
+        cf-org: <<parameters.cf_org>>
+        cf-username: <<parameters.cf_username>>
+    # - run-owasp-scan:
+    #     environment: nightly
+    #     target: backend
+    #     target_env: <<parameters.target_env>>
+    # - run-owasp-scan:
+    #     environment: nightly
+    #     target: frontend
+    #     target_env: <<parameters.target_env>>
+    - run:
+        name: Run post-processing task to record OWASP ZAP results
+        command: |
+          # Construct the project slug from the current branch name and user
+          PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
 
-            # These environment variables are exported to Circle CI's BASH_ENV
-            # by the zap-scanner.sh script for each respective app target.
-            CMD_ARGS=(
-              "$CIRCLE_BUILD_NUM"
-              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
-              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
-              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
-              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
-              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
-              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
-              --project-slug "$PROJECT_SLUG"
-            )
-            # Evaluate the full command before passing it in so it doesn't
-            # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
-            # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-<< parameters.target_env >> \
-              --command "$CMD" \
-              --name nightly-owasp-scan
\ No newline at end of file
+          # These environment variables are exported to Circle CI's BASH_ENV
+          # by the zap-scanner.sh script for each respective app target.
+          CMD_ARGS=(
+            "$CIRCLE_BUILD_NUM"
+            --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
+            --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
+            --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
+            --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
+            --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
+            --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+            --project-slug "$PROJECT_SLUG"
+          )
+          # Evaluate the full command before passing it in so it doesn't
+          # get improperly interpolated by Cloud.gov.
+          CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+          # Submit a CF Task for execution that will run the necessary command
+          cf run-task tdp-backend-<< parameters.target_env >> \
+            --command "$CMD" \
+            --name nightly-owasp-scan
\ No newline at end of file

From 5a481b7ddd38548f070f49be1aec9f3d980bd5e5 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 10:05:44 -0600
Subject: [PATCH 084/127] - UNCOMMENT JOBS

---
 .circleci/owasp/jobs.yml | 207 ++++++++++++++++++++-------------------
 1 file changed, 104 insertions(+), 103 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index d40fd161c..f2c3f3e9f 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -1,106 +1,107 @@
-backend-owasp-scan:
-  executor: large-machine-executor
-  working_directory: ~/tdp-apps
-  steps:
-    - checkout
-    - docker-compose-check
-    - docker-compose-up-backend
-    - docker-compose-up-frontend
-    - run:
-        name: Wait for Django to become available
-        command: |
-          cd tdrs-backend;
-          docker-compose run --rm zaproxy bash -c \
-            "PATH=$PATH:/home/zap/.local/bin &&
-              pip install wait-for-it &&
-              wait-for-it --service http://web:8080 \
-                          --timeout 60 \
-                          -- echo \"Django is ready\""
-    - run-owasp-scan:
-        environment: circle
-        target: backend
+jobs:
+  backend-owasp-scan:
+    executor: large-machine-executor
+    working_directory: ~/tdp-apps
+    steps:
+      - checkout
+      - docker-compose-check
+      - docker-compose-up-backend
+      - docker-compose-up-frontend
+      - run:
+          name: Wait for Django to become available
+          command: |
+            cd tdrs-backend;
+            docker-compose run --rm zaproxy bash -c \
+              "PATH=$PATH:/home/zap/.local/bin &&
+               pip install wait-for-it &&
+               wait-for-it --service http://web:8080 \
+                           --timeout 60 \
+                           -- echo \"Django is ready\""
+      - run-owasp-scan:
+          environment: circle
+          target: backend
 
-frontend-owasp-scan:
-  executor: large-machine-executor
-  working_directory: ~/tdp-apps
-  steps:
-    - checkout
-    - docker-compose-check
-    - docker-compose-up-backend
-    - docker-compose-up-frontend
-    - run:
-        name: Wait for frontend to become available
-        command: |
-          cd tdrs-frontend;
-          docker-compose run --rm zaproxy bash -c \
-            "PATH=$PATH:/home/zap/.local/bin &&
-              pip install wait-for-it &&
-              wait-for-it --service http://tdp-frontend/ \
-                          --timeout 60 \
-                          -- echo \"Frontend is ready\""
-    - run-owasp-scan:
-        environment: circle
-        target: frontend
+  frontend-owasp-scan:
+    executor: large-machine-executor
+    working_directory: ~/tdp-apps
+    steps:
+      - checkout
+      - docker-compose-check
+      - docker-compose-up-backend
+      - docker-compose-up-frontend
+      - run:
+          name: Wait for frontend to become available
+          command: |
+            cd tdrs-frontend;
+            docker-compose run --rm zaproxy bash -c \
+              "PATH=$PATH:/home/zap/.local/bin &&
+               pip install wait-for-it &&
+               wait-for-it --service http://tdp-frontend/ \
+                           --timeout 60 \
+                           -- echo \"Frontend is ready\""
+      - run-owasp-scan:
+          environment: circle
+          target: frontend
 
-nightly-owasp-scan:
-  executor: large-machine-executor
-  working_directory: ~/tdp-apps
-  parameters:
-    cf_password:
-      type: string
-      default: CF_PASSWORD_STAGING
-    cf_username:
-      type: string
-      default: CF_USERNAME_STAGING
-    cf_space:
-      type: string
-      default: tanf-staging
-    cf_org:
-      type: string
-      default: "CF_ORG"
-    target_env:
-      type: enum
-      enum: [ "staging", "develop", "prod" ]
-  steps:
-    - checkout
-    - sudo-check
-    - cf-check
-    - docker-compose-check
-    - login-cloud-dot-gov:
-        cf-password: <<parameters.cf_password>>
-        cf-space: <<parameters.cf_space>>
-        cf-org: <<parameters.cf_org>>
-        cf-username: <<parameters.cf_username>>
-    # - run-owasp-scan:
-    #     environment: nightly
-    #     target: backend
-    #     target_env: <<parameters.target_env>>
-    # - run-owasp-scan:
-    #     environment: nightly
-    #     target: frontend
-    #     target_env: <<parameters.target_env>>
-    - run:
-        name: Run post-processing task to record OWASP ZAP results
-        command: |
-          # Construct the project slug from the current branch name and user
-          PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
+  nightly-owasp-scan:
+    executor: large-machine-executor
+    working_directory: ~/tdp-apps
+    parameters:
+      cf_password:
+        type: string
+        default: CF_PASSWORD_STAGING
+      cf_username:
+        type: string
+        default: CF_USERNAME_STAGING
+      cf_space:
+        type: string
+        default: tanf-staging
+      cf_org:
+        type: string
+        default: "CF_ORG"
+      target_env:
+        type: enum
+        enum: [ "staging", "develop", "prod" ]
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+      - docker-compose-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf_password>>
+          cf-space: <<parameters.cf_space>>
+          cf-org: <<parameters.cf_org>>
+          cf-username: <<parameters.cf_username>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: backend
+      #     target_env: <<parameters.target_env>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: frontend
+      #     target_env: <<parameters.target_env>>
+      - run:
+          name: Run post-processing task to record OWASP ZAP results
+          command: |
+            # Construct the project slug from the current branch name and user
+            PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
 
-          # These environment variables are exported to Circle CI's BASH_ENV
-          # by the zap-scanner.sh script for each respective app target.
-          CMD_ARGS=(
-            "$CIRCLE_BUILD_NUM"
-            --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
-            --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
-            --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
-            --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
-            --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
-            --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
-            --project-slug "$PROJECT_SLUG"
-          )
-          # Evaluate the full command before passing it in so it doesn't
-          # get improperly interpolated by Cloud.gov.
-          CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
-          # Submit a CF Task for execution that will run the necessary command
-          cf run-task tdp-backend-<< parameters.target_env >> \
-            --command "$CMD" \
-            --name nightly-owasp-scan
\ No newline at end of file
+            # These environment variables are exported to Circle CI's BASH_ENV
+            # by the zap-scanner.sh script for each respective app target.
+            CMD_ARGS=(
+              "$CIRCLE_BUILD_NUM"
+              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
+              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
+              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
+              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
+              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
+              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+              --project-slug "$PROJECT_SLUG"
+            )
+            # Evaluate the full command before passing it in so it doesn't
+            # get improperly interpolated by Cloud.gov.
+            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+            # Submit a CF Task for execution that will run the necessary command
+            cf run-task tdp-backend-<< parameters.target_env >> \
+              --command "$CMD" \
+              --name nightly-owasp-scan
\ No newline at end of file

From 750b5624798e747be8258c5a4bcf466a9a3b564c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 10:07:13 -0600
Subject: [PATCH 085/127] - comment jobs - newline

---
 .circleci/owasp/jobs.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index f2c3f3e9f..7352041e2 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -1,4 +1,4 @@
-jobs:
+# jobs:
   backend-owasp-scan:
     executor: large-machine-executor
     working_directory: ~/tdp-apps
@@ -104,4 +104,4 @@ jobs:
             # Submit a CF Task for execution that will run the necessary command
             cf run-task tdp-backend-<< parameters.target_env >> \
               --command "$CMD" \
-              --name nightly-owasp-scan
\ No newline at end of file
+              --name nightly-owasp-scan

From 440a19977590822eeadbfad83f4eeffbc5e11714 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 10:10:05 -0600
Subject: [PATCH 086/127] - enable frontend

---
 .circleci/owasp/jobs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 7352041e2..48dd94564 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -76,10 +76,10 @@
       #     environment: nightly
       #     target: backend
       #     target_env: <<parameters.target_env>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: frontend
-      #     target_env: <<parameters.target_env>>
+      - run-owasp-scan:
+          environment: nightly
+          target: frontend
+          target_env: <<parameters.target_env>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |

From 31db0962425d9a6ccb5d2ba7b1aba1b43b94edc9 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 10:21:33 -0600
Subject: [PATCH 087/127] - echo cmd - trailing slash

---
 .circleci/owasp/jobs.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 48dd94564..324851d79 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -101,7 +101,9 @@
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
             CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+            echo "$CMD"
+
             # Submit a CF Task for execution that will run the necessary command
             cf run-task tdp-backend-<< parameters.target_env >> \
               --command "$CMD" \
-              --name nightly-owasp-scan
+              --name nightly-owasp-scan \

From decdc245a1c59ddfff52d151cb2cf816f63ca04e Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 10:32:22 -0600
Subject: [PATCH 088/127] - adding wait - adding trailing echo

---
 .circleci/owasp/jobs.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 324851d79..b37accaf9 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -105,5 +105,7 @@
 
             # Submit a CF Task for execution that will run the necessary command
             cf run-task tdp-backend-<< parameters.target_env >> \
+              --wait \
               --command "$CMD" \
-              --name nightly-owasp-scan \
+              --name nightly-owasp-scan
+            echo DONE!
\ No newline at end of file

From 85acdea60515beabcf4b679453eed7c45ff05777 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 10:48:14 -0600
Subject: [PATCH 089/127] - firmat

---
 .circleci/owasp/jobs.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index b37accaf9..f871e4e50 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -104,8 +104,5 @@
             echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-<< parameters.target_env >> \
-              --wait \
-              --command "$CMD" \
-              --name nightly-owasp-scan
+            cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan
             echo DONE!
\ No newline at end of file

From d66887e1f4505b01da7ff7f0fb98ef99156b4df6 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 11:01:40 -0600
Subject: [PATCH 090/127] - dummy command

---
 .circleci/owasp/jobs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index f871e4e50..da7bc8fee 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -104,5 +104,5 @@
             echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan
+            cf run-task tdp-backend-<< parameters.target_env >> --wait --command "ls" --name nightly-owasp-scan
             echo DONE!
\ No newline at end of file

From 5b42977ee621f7ba655d577eed34115a79822dc6 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 11:12:20 -0600
Subject: [PATCH 091/127] - playing with echos

---
 .circleci/owasp/jobs.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index da7bc8fee..3bf4f78bc 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -104,5 +104,4 @@
             echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
-            cf run-task tdp-backend-<< parameters.target_env >> --wait --command "ls" --name nightly-owasp-scan
-            echo DONE!
\ No newline at end of file
+            echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!
\ No newline at end of file

From a3cb0374822d9ac6fda4f50d3b5ba93193778794 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 12:57:22 -0600
Subject: [PATCH 092/127] - remove comment

---
 .circleci/owasp/jobs.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 3bf4f78bc..90478f1ad 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   backend-owasp-scan:
     executor: large-machine-executor
     working_directory: ~/tdp-apps

From 785d689c51b6ac02b71d8a397b94918ae03e7cb4 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 13:11:15 -0600
Subject: [PATCH 093/127] - removing comments as test

---
 .circleci/build-and-test/jobs.yml | 1 -
 .circleci/deployment/jobs.yml     | 1 -
 .circleci/util/jobs.yml           | 3 +--
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml
index c143ea8d7..f9b602adf 100644
--- a/.circleci/build-and-test/jobs.yml
+++ b/.circleci/build-and-test/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   test-backend:
     executor: machine-executor
     steps:
diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml
index d6faf27b8..63d5bc070 100644
--- a/.circleci/deployment/jobs.yml
+++ b/.circleci/deployment/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   deploy-dev:
     parameters:
       target_env:
diff --git a/.circleci/util/jobs.yml b/.circleci/util/jobs.yml
index 9ba582f8e..3cd1bfe12 100644
--- a/.circleci/util/jobs.yml
+++ b/.circleci/util/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   make_erd:
     executor: machine-executor
     working_directory: ~/tdp_apps
@@ -9,7 +8,7 @@
           name: Run graph_models
           command: |
             cd tdrs-backend
-            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then
             docker network create external-net
             fi
             docker-compose run --rm web bash -c \

From 949b68e802e9c25fe5d744aadc5801fb663e8454 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 13:19:29 -0600
Subject: [PATCH 094/127] Revert "- removing comments as test"

This reverts commit 785d689c51b6ac02b71d8a397b94918ae03e7cb4.
---
 .circleci/build-and-test/jobs.yml | 1 +
 .circleci/deployment/jobs.yml     | 1 +
 .circleci/util/jobs.yml           | 3 ++-
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml
index f9b602adf..c143ea8d7 100644
--- a/.circleci/build-and-test/jobs.yml
+++ b/.circleci/build-and-test/jobs.yml
@@ -1,3 +1,4 @@
+# jobs:
   test-backend:
     executor: machine-executor
     steps:
diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml
index 63d5bc070..d6faf27b8 100644
--- a/.circleci/deployment/jobs.yml
+++ b/.circleci/deployment/jobs.yml
@@ -1,3 +1,4 @@
+# jobs:
   deploy-dev:
     parameters:
       target_env:
diff --git a/.circleci/util/jobs.yml b/.circleci/util/jobs.yml
index 3cd1bfe12..9ba582f8e 100644
--- a/.circleci/util/jobs.yml
+++ b/.circleci/util/jobs.yml
@@ -1,3 +1,4 @@
+# jobs:
   make_erd:
     executor: machine-executor
     working_directory: ~/tdp_apps
@@ -8,7 +9,7 @@
           name: Run graph_models
           command: |
             cd tdrs-backend
-            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
             docker network create external-net
             fi
             docker-compose run --rm web bash -c \

From b8381b03ba0227ec04f7ae8f4db98ba7939dc314 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 13:21:36 -0600
Subject: [PATCH 095/127] - newline

---
 .circleci/owasp/jobs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 90478f1ad..72ace6e4b 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -103,4 +103,4 @@
             echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
-            echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!
\ No newline at end of file
+            echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!

From 4b0f266547702ff2d865b8331b71dfe13c69560c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 13:21:48 -0600
Subject: [PATCH 096/127] Revert "Revert "- removing comments as test""

This reverts commit 949b68e802e9c25fe5d744aadc5801fb663e8454.
---
 .circleci/build-and-test/jobs.yml | 1 -
 .circleci/deployment/jobs.yml     | 1 -
 .circleci/util/jobs.yml           | 3 +--
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml
index c143ea8d7..f9b602adf 100644
--- a/.circleci/build-and-test/jobs.yml
+++ b/.circleci/build-and-test/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   test-backend:
     executor: machine-executor
     steps:
diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml
index d6faf27b8..63d5bc070 100644
--- a/.circleci/deployment/jobs.yml
+++ b/.circleci/deployment/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   deploy-dev:
     parameters:
       target_env:
diff --git a/.circleci/util/jobs.yml b/.circleci/util/jobs.yml
index 9ba582f8e..3cd1bfe12 100644
--- a/.circleci/util/jobs.yml
+++ b/.circleci/util/jobs.yml
@@ -1,4 +1,3 @@
-# jobs:
   make_erd:
     executor: machine-executor
     working_directory: ~/tdp_apps
@@ -9,7 +8,7 @@
           name: Run graph_models
           command: |
             cd tdrs-backend
-            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then
             docker network create external-net
             fi
             docker-compose run --rm web bash -c \

From 62bd03032781c5c21815451371880427a8d966fe Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 13:33:35 -0600
Subject: [PATCH 097/127] - catting config

---
 .circleci/owasp/jobs.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 72ace6e4b..78ea200ce 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -103,4 +103,5 @@
             echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
+            cat .circleci/generated_config.yml
             echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!

From 0243276ca6a50796a14b2c9b7ba2a79c4df945e6 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 14:44:59 -0600
Subject: [PATCH 098/127] - remove cat

---
 .circleci/generate_config.sh   |    2 +-
 .circleci/generated_config.yml | 1084 ++++++++++++++++++++++++++++++++
 .circleci/owasp/jobs.yml       |    1 -
 3 files changed, 1085 insertions(+), 2 deletions(-)
 mode change 100644 => 100755 .circleci/generate_config.sh
 create mode 100644 .circleci/generated_config.yml

diff --git a/.circleci/generate_config.sh b/.circleci/generate_config.sh
old mode 100644
new mode 100755
index 0349eb352..0ecb35e5b
--- a/.circleci/generate_config.sh
+++ b/.circleci/generate_config.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/sh
+#!/bin/bash
 
 cat base_config.yml > generated_config.yml
 
diff --git a/.circleci/generated_config.yml b/.circleci/generated_config.yml
new file mode 100644
index 000000000..60a2d2fe3
--- /dev/null
+++ b/.circleci/generated_config.yml
@@ -0,0 +1,1084 @@
+version: 2.1
+
+orbs:
+  node: circleci/node@5.1.0
+  terraform: circleci/terraform@2.1.0
+  jq: circleci/jq@2.2.0
+
+executors:
+  docker-executor:
+    docker:
+      - image: cimg/python:3.10.4
+        user: root
+  machine-executor:
+    machine:
+      docker_layer_caching: false
+      image: ubuntu-2204:2024.01.1
+  large-machine-executor:
+    machine:
+      docker_layer_caching: false
+      image: ubuntu-2204:2024.01.1
+    resource_class: large
+
+parameters:
+  build_and_test_all:
+    type: boolean
+    default: false
+  build_and_test_backend:
+    type: boolean
+    default: false
+  build_and_test_frontend:
+    type: boolean
+    default: false
+  deploy_infrastructure:
+    type: boolean
+    default: false
+  develop_branch_deploy:
+    type: boolean
+    default: false
+  run_dev_deployment:
+    type: boolean
+    default: false
+  run_nightly_owasp_scan:
+    type: boolean
+    default: false
+  run_owasp_scan:
+    type: boolean
+    default: false
+  target_env:
+    type: string
+    default: ''
+  triggered:
+    type: boolean
+    default: false
+  util_make_erd:
+    type: boolean
+    default: false
+commands:
+# commands:
+  upload-codecov:
+    description: Uploads testing code coverage results to Codecov
+    parameters:
+      component:
+        description: The component of the application being tested, either backend or frontend.
+        type: enum
+        enum: [ "backend", "frontend" ]
+      coverage-report:
+        description: The path to the coverage report being uploaded.
+        type: string
+    steps:
+      - run:
+          name: Ensure Codecov uploader is installed, otherwise install it.
+          command: ./scripts/codecov-check.sh
+      - run:
+          name: Determine Codecov metric flag
+          command: |
+            if [ "$CIRCLE_BRANCH" == "main" ] ; then
+              CURRENT_FLAG=main-<<parameters.component>>
+            elif [ "$CIRCLE_BRANCH" == "master" ] ; then
+              CURRENT_FLAG=master-<<parameters.component>>
+            else
+              CURRENT_FLAG=dev-<<parameters.component>>
+            fi
+            echo "export CURRENT_FLAG=$CURRENT_FLAG" >> $BASH_ENV
+      - run:
+          name: Upload code coverage report if target branch
+          command: codecov -t "$CODECOV_TOKEN" -f <<parameters.coverage-report>> -F "$CURRENT_FLAG"
+
+  install-nodejs-machine:
+    description: |
+      Installs our target version of Node.JS using NVM (Node Version Manager)
+      from the install location provided by machine executor images.
+    steps:
+      - run:
+          name: Install Node.JS
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y libgbm-dev
+            source /opt/circleci/.nvm/nvm.sh
+            nvm install v16.13
+            nvm alias default v16.13
+            echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV
+            echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV
+
+  disable-npm-audit:
+    steps:
+      - run:
+          name: Disable npm audit warnings in CI
+          command: npm set audit false
+          
+  # This allows us to use the node orb to install packages within other commands
+  install-nodejs-packages: node/install-packages
+# commands:
+  deploy-cloud-dot-gov:
+    parameters:
+      environment:
+        description: The environment to deploy to.
+        type: enum
+        enum: [ "development", "production" ]
+        default: development
+      backend-appname:
+        default: tdp-backend
+        type: string
+      cf-password:
+        default: CF_PASSWORD_DEV
+        type: env_var_name
+      cf-org:
+        default: CF_ORG
+        type: env_var_name
+      cf-space:
+        default: tanf-dev
+        type: string
+      cf-username:
+        default: CF_USERNAME_DEV
+        type: env_var_name
+      frontend-appname:
+        default: tdp-frontend
+        type: string
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-org: <<parameters.cf-org>>
+          cf-space: <<parameters.cf-space>>
+          cf-username: <<parameters.cf-username>>
+      - deploy-backend:
+          backend-appname: <<parameters.backend-appname>>
+          frontend-appname: <<parameters.frontend-appname>>
+          cf-space: <<parameters.cf-space>>
+      - deploy-frontend:
+          environment: <<parameters.environment>>
+          backend-appname: <<parameters.backend-appname>>
+          frontend-appname: <<parameters.frontend-appname>>
+          cf-space: <<parameters.cf-space>>
+
+  clamav-cloud-dot-gov:
+    parameters:
+      backend-appname:
+        default: tdp-backend
+        type: string
+      cf-password:
+        default: CF_PASSWORD_DEV
+        type: env_var_name
+      cf-org:
+        default: CF_ORG
+        type: env_var_name
+      cf-space:
+        default: tanf-dev
+        type: string
+      cf-username:
+        default: CF_USERNAME_DEV
+        type: env_var_name
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-org: <<parameters.cf-org>>
+          cf-space: <<parameters.cf-space>>
+          cf-username: <<parameters.cf-username>>
+      - deploy-clamav
+
+  deploy-backend:
+    parameters:
+      backend-appname:
+        default: tdp-backend
+        type: string
+      frontend-appname:
+        default: tdp-frontend
+        type: string
+      kibana-appname:
+        default: tdp-kibana
+        type: string
+      proxy-appname:
+        default: tdp-elastic-proxy
+        type: string
+      cf-space:
+        default: tanf-dev
+        type: string
+    steps:
+      - get-app-deploy-strategy:
+          appname: <<parameters.backend-appname>>
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt update
+            sudo apt-get install -y wget
+            sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_amd64 && chmod +x /usr/local/bin/yq
+      - run:
+          name: Apply database migrations
+          command: |
+            bash ./scripts/apply-remote-migrations.sh <<parameters.backend-appname>>
+      - run:
+          name: Deploy backend application
+          command: |
+            bash ./scripts/deploy-backend.sh \
+              $DEPLOY_STRATEGY \
+              <<parameters.frontend-appname>> \
+              <<parameters.backend-appname>> \
+              <<parameters.kibana-appname>> \
+              <<parameters.proxy-appname>> \
+              <<parameters.cf-space>>
+
+  deploy-clamav:
+    steps:
+      - run:
+          name: Deploy ClamAV REST application
+          command: |
+            cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \
+              --var cf-space=tanf-prod
+
+  deploy-frontend:
+    parameters:
+      environment:
+        description: The environment to deploy to.
+        type: enum
+        enum: [ "development", "production" ]
+        default: development
+      backend-appname:
+        default: tdp-backend
+        type: string
+      frontend-appname:
+        default: tdp-frontend
+        type: string
+      kibana-appname:
+        default: tdp-kibana
+        type: string
+# So the frontend knows what space its in for the banner.
+# I am unclear if the domain is a reliable metric to make this function
+# It seems like it might not be working
+      cf-space:
+        default: dev
+        type: string
+    steps:
+      - install-nodejs:
+          node-version: "16.13"
+      - disable-npm-audit
+      - install-nodejs-packages:
+          app-dir: tdrs-frontend
+      - get-app-deploy-strategy:
+          appname: <<parameters.frontend-appname>>
+      - run:
+          name: Deploy frontend application
+          command: |
+            bash ./scripts/deploy-frontend.sh \
+              $DEPLOY_STRATEGY \
+              <<parameters.frontend-appname>> \
+              <<parameters.backend-appname>> \
+              <<parameters.kibana-appname>> \
+              <<parameters.cf-space>> \
+              <<parameters.environment>>
+
+  get-app-deploy-strategy:
+    parameters:
+      appname:
+        type: string
+    steps:
+      - run:
+          name: Determine deploy strategy
+          command: |
+            # NOTE: The || true is a no-op included to suppress exit codes which
+            #       would cause the step to exit early due to use of pipefail
+            APP_GUID=$(cf app <<parameters.appname>> --guid || true)
+            if [ "$APP_GUID" == "FAILED" ]; then
+              echo "export DEPLOY_STRATEGY=initial" >> $BASH_ENV
+            else
+              echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV
+            fi
+
+  deploy-infrastructure:
+    parameters:
+      tf-path:
+        type: string
+        default: ./terraform/dev
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+      cf-space:
+        type: string
+        default: tanf-dev
+      cf-org:
+        type: env_var_name
+        default: CF_ORG
+      cf-app:
+        type: string
+        default: CF_APP
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            apk update
+            apk add jq
+            apk add curl
+            # TODO: Add Signature check
+            curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx
+            mv cf7 /usr/local/bin/cf
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-username: <<parameters.cf-username>>
+          cf-space: <<parameters.cf-space>>
+      - run:
+          name: Export S3 Credentials for TFState
+          command: |
+            S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2)
+            {
+              echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\""
+              echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\""
+              echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\""
+              echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\""
+            } >> ./backend_config.tfvars
+      - run:
+          name: Prepare Terraform Variables
+          command: |
+            S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2)
+            {
+              echo "cf_password = \"$<<parameters.cf-password>>\""
+              echo "cf_user = \"$<<parameters.cf-username>>\""
+              echo "cf_space_name = \"<<parameters.cf-space>>\""
+              echo "cf_app_name = \"<<parameters.cf-app>>\""
+            } >> ./variables.tfvars
+      - terraform/init:
+          path: <<parameters.tf-path>>
+          backend_config_file: ./backend_config.tfvars
+      - terraform/validate:
+          path: <<parameters.tf-path>>
+      - terraform/fmt:
+          path: <<parameters.tf-path>>
+      - terraform/plan:
+          path: <<parameters.tf-path>>
+          var_file: ./variables.tfvars
+      - terraform/apply:
+          path: <<parameters.tf-path>>
+          var_file: ./variables.tfvars
+
+  enable-s3-versioning:
+    parameters:
+      target_env:
+        type: string
+      cf-password:
+        type: env_var_name
+      cf-username:
+        type: env_var_name
+      cf-space:
+        type: string
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt update
+            sudo apt install jq
+            sudo apt install curl
+            # TODO: Add Signature check
+            curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx
+            sudo mv cf7 /usr/local/bin/cf
+            sudo chmod +x /usr/local/bin/cf
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-username: <<parameters.cf-username>>
+          cf-space: <<parameters.cf-space>>
+      - run:
+          name: Ensure s3 versioning key exists
+          # use cf to check if a service key exists for tdp-datafiles-dev(or prod)
+          # exit 0 if it does, create it if it doesn't
+          command: |
+            chmod +x ./scripts/create_s3_versioning_key.sh
+            ./scripts/create_s3_versioning_key.sh tdp-datafiles-<<parameters.target_env>> s3-versioning-key
+      - run:
+          name: Set up aws credentials
+          command: |
+            S3_CREDENTIALS=$(cf service-key tdp-datafiles-<<parameters.target_env>> s3-versioning-key | tail -n +2)
+            aws configure set aws_access_key_id $(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)
+            aws configure set aws_secret_access_key $(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)
+            aws configure set default.region $(echo "${S3_CREDENTIALS}" | jq -r '.region')
+      - run:
+          name: Enable S3 Versioning
+          command: |
+            S3_BUCKET_ID=cg-$(cf service tdp-datafiles-<<parameters.target_env>> --guid)
+            aws s3api put-bucket-versioning --bucket ${S3_BUCKET_ID} --versioning-configuration Status=Enabled
+# commands:
+  run-owasp-scan:
+    description: Runs OWASP ZAP scanner and stores resulting artifacts.
+    parameters:
+      environment:
+        description: The environment this script is being run for.
+        type: enum
+        enum: ["circle", "nightly"] 
+      target:
+        description: The target application to be scanned.
+        type: enum
+        enum: ["backend", "frontend"]
+      target_env:
+        description: The target environment to be scanned.
+        type: enum
+        default: "develop"
+        enum: ["staging", "develop", "prod"]
+    steps:
+      - run:
+          name: Execute OWASP ZAP vulnerability scan
+          #wait up to 60min for scan
+          no_output_timeout: 60m
+          command: ./scripts/zap-scanner.sh <<parameters.target>> <<parameters.environment>> <<parameters.target_env>>
+      - store_artifacts:
+          path: tdrs-<<parameters.target>>/reports/owasp_report.html
+
+  docker-compose-up-frontend:
+    steps:
+      - run:
+          name: Build and spin-up React application
+          command: |
+            cd tdrs-frontend 
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
+            docker network create external-net
+            fi
+            docker-compose up -d --build
+# commands:
+  docker-compose-check:
+    steps:
+      - run:
+          name: Ensure docker-compose exists, otherwise install it.
+          command: ./scripts/docker-compose-check.sh
+
+  docker-compose-up-backend:
+    steps:
+      - run:
+          name: Build and spin-up Django API service
+          command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
+
+  docker-compose-up-with-elastic-backend:
+    steps:
+      - run:
+          name: Build and spin-up Django API service
+          command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build
+
+  cf-check:
+    steps:
+      - run:
+          name: Ensure cf cli is installed, otherwise install it.
+          command: sudo ./scripts/cf-check.sh
+
+  # This allows us to use the orb stanza for node/install within other commands
+  # NOTE: This doesn't work correctly on machine executors
+  install-nodejs: node/install
+
+  sudo-check:
+    steps:
+      - run:
+          name: Ensure sudo is installed, otherwise install it.
+          command: ./scripts/sudo-check.sh
+
+  login-cloud-dot-gov:
+    description: Authenticates with Cloud.gov and sets org and space targets
+    parameters:
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-org:
+        type: env_var_name
+        default: CF_ORG
+      cf-space:
+        type: string
+        default: tanf-dev
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+    steps:
+      - run:
+          name: Login to Cloud.gov and set application targets
+          command: |
+            cf login -a https://api.fr.cloud.gov \
+              -u ${<<parameters.cf-username>>} \
+              -p ${<<parameters.cf-password>>} \
+              -o ${<<parameters.cf-org>>} \
+              -s <<parameters.cf-space>>
+jobs:
+  test-backend:
+    executor: machine-executor
+    steps:
+      - checkout
+      - docker-compose-check
+      - docker-compose-up-with-elastic-backend
+      - run:
+          name: Run Unit Tests And Create Code Coverage Report
+          command: |
+            cd tdrs-backend;
+            docker-compose run --rm web bash -c "./wait_for_services.sh && pytest --cov-report=xml"
+      - run:
+          name: Execute Python Linting Test
+          command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ."
+      - upload-codecov:
+          component: backend
+          coverage-report: ./tdrs-backend/coverage.xml
+
+  test-frontend:
+    executor: machine-executor
+    working_directory: ~/tdp-apps
+    steps:
+      - checkout
+      - install-nodejs-machine
+      - disable-npm-audit
+      - install-nodejs-packages:
+          app-dir: tdrs-frontend
+      - run:
+          name: Run ESLint
+          command: cd tdrs-frontend; npm run lint
+      - run:
+          name: Run Pa11y Accessibility Tests
+          command: cd tdrs-frontend; mkdir pa11y-screenshots/; export NODE_OPTIONS=--openssl-legacy-provider; npm run test:accessibility
+      - run:
+          name: Run Jest Unit Tests
+          command: cd tdrs-frontend; npm run test:ci
+      - upload-codecov:
+          component: frontend
+          coverage-report: ./tdrs-frontend/coverage/lcov.info
+      - store_artifacts:
+          path: tdrs-frontend/pa11y-screenshots/
+
+  test-e2e:
+    executor: large-machine-executor
+    working_directory: ~/tdp-apps
+    steps:
+      - checkout
+      - docker-compose-check
+      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-frontend
+      - install-nodejs-machine
+      - disable-npm-audit
+      - install-nodejs-packages:
+          app-dir: tdrs-frontend
+      - run:
+          name: Wait for backend to become available
+          command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \
+              "PATH=$PATH:/home/zap/.local/bin &&
+               pip install wait-for-it &&
+               wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
+      - run:
+          name: apply the migrations
+          command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate"
+      - run:
+          name: Remove existing cypress test users
+          command: cd tdrs-backend; docker-compose exec web python manage.py delete_cypress_users -usernames new-cypress@teamraft.com cypress-admin@teamraft.com
+      - run:
+          name: Setup cypress test users
+          command: cd tdrs-backend; docker-compose exec web python manage.py generate_cypress_users
+      - run:
+          name: Run Cypress e2e tests
+          command: cd tdrs-frontend; npm run test:e2e-ci
+      - store_artifacts:
+          path: tdrs-frontend/cypress/screenshots/
+      - store_artifacts:
+          path: tdrs-frontend/cypress/videos/
+
+  secrets-check:
+    executor: docker-executor
+    steps:
+      - checkout
+      - run:
+          name: "git-secrets: Scan repository for committed secrets"
+          command: ./scripts/git-secrets-check.sh
+      - run:
+          name: "trufflehog: Scan repository for committed secrets"
+          command: ./scripts/trufflehog-check.sh $CIRCLE_BRANCH
+  deploy-dev:
+    parameters:
+      target_env:
+        type: string
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-cloud-dot-gov:
+          backend-appname: tdp-backend-<< parameters.target_env >>
+          frontend-appname: tdp-frontend-<< parameters.target_env >>
+
+  deploy-staging:
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-cloud-dot-gov:
+          backend-appname: tdp-backend-staging
+          frontend-appname: tdp-frontend-staging
+          cf-password: CF_PASSWORD_STAGING
+          cf-space: tanf-staging
+          cf-username: CF_USERNAME_STAGING
+
+  deploy-develop:
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-cloud-dot-gov:
+          backend-appname: tdp-backend-develop
+          frontend-appname: tdp-frontend-develop
+          cf-password: CF_PASSWORD_STAGING
+          cf-space: tanf-staging
+          cf-username: CF_USERNAME_STAGING
+
+  deploy-infrastructure-dev:
+    parameters:
+      target_env:
+        type: string
+    executor: terraform/default
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-infrastructure:
+                cf-app: << parameters.target_env >>
+
+  deploy-infrastructure-staging:
+    executor: terraform/default
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-infrastructure:
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
+          tf-path: ./terraform/staging
+
+  deploy-infrastructure-production:
+    executor: terraform/default
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-infrastructure:
+          cf-password: CF_PASSWORD_PROD
+          cf-username: CF_USERNAME_PROD
+          cf-space: tanf-prod
+          tf-path: ./terraform/production
+
+  deploy-project-updates-site:
+    parameters:
+      cf-org:
+        default: CF_ORG
+        type: env_var_name
+      cf-space:
+        default: tanf-dev
+        type: string
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-org: <<parameters.cf-org>>
+          cf-space: <<parameters.cf-space>>
+          cf-username: <<parameters.cf-username>>
+      - run:
+          name: Deploy TDP Project Updates Site
+          command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates
+
+  enable-versioning:
+    executor: machine-executor
+    parameters:
+      target_env:
+        type: string
+        default: dev
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+      cf-space:
+        type: string
+        default: tanf-dev
+    steps:
+      - enable-s3-versioning:
+          target_env: <<parameters.target_env>>
+          cf-password: <<parameters.cf-password>>
+          cf-username: <<parameters.cf-username>>
+          cf-space: <<parameters.cf-space>>
+
+  test-deployment-e2e:
+    executor: machine-executor
+    working_directory: ~/tdp-apps
+    steps:
+      - checkout
+      - install-nodejs-machine
+      - disable-npm-audit
+      - install-nodejs-packages:
+          app-dir: tdrs-frontend
+      - run:
+          name: Run Cypress e2e tests
+          command: cd tdrs-frontend; npm run test:e2e-ci -- --config baseUrl="https://tdp-frontend-develop.acf.hhs.gov" --env cypressToken=$CYPRESS_TOKEN,apiUrl="https://tdp-frontend-develop.acf.hhs.gov/v1",adminUrl="https://tdp-frontend-develop.acf.hhs.gov/admin"
+      - store_artifacts:
+          path: tdrs-frontend/cypress/screenshots/
+      - store_artifacts:
+          path: tdrs-frontend/cypress/videos/
+
+  deploy-production:
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-cloud-dot-gov:
+          environment: production
+          backend-appname: tdp-backend-prod
+          frontend-appname: tdp-frontend-prod
+          cf-password: CF_PASSWORD_PROD
+          cf-space: tanf-prod
+          cf-username: CF_USERNAME_PROD
+  prod-deploy-clamav:
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - clamav-cloud-dot-gov:
+          backend-appname: tdp-backend-prod
+          cf-password: CF_PASSWORD_PROD
+          cf-space: tanf-prod
+          cf-username: CF_USERNAME_PROD
+  backend-owasp-scan:
+    executor: large-machine-executor
+    working_directory: ~/tdp-apps
+    steps:
+      - checkout
+      - docker-compose-check
+      - docker-compose-up-backend
+      - docker-compose-up-frontend
+      - run:
+          name: Wait for Django to become available
+          command: |
+            cd tdrs-backend;
+            docker-compose run --rm zaproxy bash -c \
+              "PATH=$PATH:/home/zap/.local/bin &&
+               pip install wait-for-it &&
+               wait-for-it --service http://web:8080 \
+                           --timeout 60 \
+                           -- echo \"Django is ready\""
+      - run-owasp-scan:
+          environment: circle
+          target: backend
+
+  frontend-owasp-scan:
+    executor: large-machine-executor
+    working_directory: ~/tdp-apps
+    steps:
+      - checkout
+      - docker-compose-check
+      - docker-compose-up-backend
+      - docker-compose-up-frontend
+      - run:
+          name: Wait for frontend to become available
+          command: |
+            cd tdrs-frontend;
+            docker-compose run --rm zaproxy bash -c \
+              "PATH=$PATH:/home/zap/.local/bin &&
+               pip install wait-for-it &&
+               wait-for-it --service http://tdp-frontend/ \
+                           --timeout 60 \
+                           -- echo \"Frontend is ready\""
+      - run-owasp-scan:
+          environment: circle
+          target: frontend
+
+  nightly-owasp-scan:
+    executor: large-machine-executor
+    working_directory: ~/tdp-apps
+    parameters:
+      cf_password:
+        type: string
+        default: CF_PASSWORD_STAGING
+      cf_username:
+        type: string
+        default: CF_USERNAME_STAGING
+      cf_space:
+        type: string
+        default: tanf-staging
+      cf_org:
+        type: string
+        default: "CF_ORG"
+      target_env:
+        type: enum
+        enum: [ "staging", "develop", "prod" ]
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+      - docker-compose-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf_password>>
+          cf-space: <<parameters.cf_space>>
+          cf-org: <<parameters.cf_org>>
+          cf-username: <<parameters.cf_username>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: backend
+      #     target_env: <<parameters.target_env>>
+      - run-owasp-scan:
+          environment: nightly
+          target: frontend
+          target_env: <<parameters.target_env>>
+      - run:
+          name: Run post-processing task to record OWASP ZAP results
+          command: |
+            # Construct the project slug from the current branch name and user
+            PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
+
+            # These environment variables are exported to Circle CI's BASH_ENV
+            # by the zap-scanner.sh script for each respective app target.
+            CMD_ARGS=(
+              "$CIRCLE_BUILD_NUM"
+              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
+              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
+              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
+              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
+              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
+              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+              --project-slug "$PROJECT_SLUG"
+            )
+            # Evaluate the full command before passing it in so it doesn't
+            # get improperly interpolated by Cloud.gov.
+            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+            echo "$CMD"
+
+            # Submit a CF Task for execution that will run the necessary command
+            echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!
+  make_erd:
+    executor: machine-executor
+    working_directory: ~/tdp_apps
+    steps:
+      - checkout
+      - docker-compose-check
+      - run:
+          name: Run graph_models
+          command: |
+            cd tdrs-backend
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then
+            docker network create external-net
+            fi
+            docker-compose run --rm web bash -c \
+            "./manage.py graph_models -a -g -o tdp_erd.png"
+      - store_artifacts:
+          path: tdrs-backend/tdp_erd.png
+workflows:
+# workflows:
+  build-and-test-all:
+    when: << pipeline.parameters.build_and_test_all >>
+    jobs:
+      - secrets-check
+      # TODO: Revert back to tests after review
+      - nightly-owasp-scan:
+          target_env: staging
+
+  ci-build-and-test-all:
+    jobs:
+      - secrets-check:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
+      - test-backend:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
+          requires:
+            - secrets-check
+      - test-frontend:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
+          requires:
+            - secrets-check
+      - test-e2e:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
+          requires:
+            - secrets-check
+
+  build-and-test-backend:
+    when: << pipeline.parameters.build_and_test_backend >>
+    jobs:
+      - secrets-check
+      - test-backend:
+          requires:
+            - secrets-check
+
+  build-and-test-frontend:
+    when: << pipeline.parameters.build_and_test_frontend >>
+    jobs:
+      - secrets-check
+      - test-frontend:
+          requires:
+            - secrets-check
+#workflows:
+  deployment:
+    when:
+      and:
+        - or:
+            - equal: [ master, << pipeline.git.branch >> ]
+            - equal: [ main, << pipeline.git.branch >> ]
+            - equal: [ develop, << pipeline.git.branch >> ]
+            - << pipeline.parameters.run_dev_deployment >>
+        - not: << pipeline.parameters.run_nightly_owasp_scan >>
+
+    jobs:
+      - deploy-project-updates-site:
+          filters:
+            branches:
+              only:
+                - develop
+      - deploy-infrastructure-dev:
+          target_env: << pipeline.parameters.target_env >>
+          filters:
+            branches:
+              ignore:
+                - develop
+                - main
+                - master
+      - deploy-infrastructure-staging:
+          filters:
+            branches:
+              only:
+                - develop
+                - main
+      - deploy-infrastructure-production:
+          filters:
+            branches:
+              only:
+                - master
+      - enable-versioning:
+          requires:
+            - deploy-infrastructure-dev
+          filters:
+            branches:
+              ignore:
+                - develop
+                - main
+                - master
+      - enable-versioning:
+          requires:
+            - deploy-infrastructure-staging
+          target_env: develop
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
+          filters:
+            branches:
+              only:
+                - develop
+      - enable-versioning:
+          requires:
+            - deploy-infrastructure-staging
+          target_env: staging
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
+          filters:
+            branches:
+              only:
+                - main
+      - enable-versioning:
+          requires:
+            - deploy-infrastructure-production
+          target_env: prod
+          cf-password: CF_PASSWORD_PROD
+          cf-username: CF_USERNAME_PROD
+          cf-space: tanf-prod
+          filters:
+            branches:
+              only:
+                - master
+      - prod-deploy-clamav:
+          requires:
+            - deploy-infrastructure-production
+          filters:
+            branches:
+              only:
+                - master
+      - deploy-dev:
+          target_env: << pipeline.parameters.target_env >>
+          requires:
+            - deploy-infrastructure-dev
+          filters:
+            branches:
+              ignore:
+                - develop
+                - main
+                - master
+      - deploy-develop:
+          requires:
+            - deploy-infrastructure-staging
+          filters:
+            branches:
+              only:
+                - develop
+      - deploy-staging:
+          requires:
+            - deploy-infrastructure-staging
+          filters:
+            branches:
+              only:
+                - main
+      - deploy-production:
+          requires:
+            - deploy-infrastructure-production
+          filters:
+            branches:
+              only:
+                - master
+      - test-deployment-e2e:
+          requires:
+            - deploy-develop
+          filters:
+            branches:
+              only:
+                - develop
+      - make_erd: # from ../util folder
+          filters:
+            branches:
+              only:
+                - develop
+                - master
+# workflows:
+  owasp-scan:
+    when: << pipeline.parameters.run_owasp_scan >>
+    jobs:
+      - backend-owasp-scan
+      - frontend-owasp-scan
+  
+  nightly:
+    when: << pipeline.parameters.run_nightly_owasp_scan >>
+    jobs:
+      - nightly-owasp-scan:
+          target_env: develop
+          filters:
+            branches:
+              only:
+                - develop
+      - nightly-owasp-scan:
+          target_env: staging
+          filters:
+            branches:
+              only:
+                - main
+      - nightly-owasp-scan:
+          target_env: prod
+          cf_password: CF_PASSWORD_PROD
+          cf_username: CF_USERNAME_PROD
+          cf_space: tanf-prod
+          filters:
+            branches:
+              only:
+                - master
diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 78ea200ce..72ace6e4b 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -103,5 +103,4 @@
             echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
-            cat .circleci/generated_config.yml
             echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!

From ff17e21353db7230d72d1657c522ff19f3b74662 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 14:45:23 -0600
Subject: [PATCH 099/127] Revert "- remove cat"

This reverts commit 0243276ca6a50796a14b2c9b7ba2a79c4df945e6.
---
 .circleci/generate_config.sh   |    2 +-
 .circleci/generated_config.yml | 1084 --------------------------------
 .circleci/owasp/jobs.yml       |    1 +
 3 files changed, 2 insertions(+), 1085 deletions(-)
 mode change 100755 => 100644 .circleci/generate_config.sh
 delete mode 100644 .circleci/generated_config.yml

diff --git a/.circleci/generate_config.sh b/.circleci/generate_config.sh
old mode 100755
new mode 100644
index 0ecb35e5b..0349eb352
--- a/.circleci/generate_config.sh
+++ b/.circleci/generate_config.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/sh
 
 cat base_config.yml > generated_config.yml
 
diff --git a/.circleci/generated_config.yml b/.circleci/generated_config.yml
deleted file mode 100644
index 60a2d2fe3..000000000
--- a/.circleci/generated_config.yml
+++ /dev/null
@@ -1,1084 +0,0 @@
-version: 2.1
-
-orbs:
-  node: circleci/node@5.1.0
-  terraform: circleci/terraform@2.1.0
-  jq: circleci/jq@2.2.0
-
-executors:
-  docker-executor:
-    docker:
-      - image: cimg/python:3.10.4
-        user: root
-  machine-executor:
-    machine:
-      docker_layer_caching: false
-      image: ubuntu-2204:2024.01.1
-  large-machine-executor:
-    machine:
-      docker_layer_caching: false
-      image: ubuntu-2204:2024.01.1
-    resource_class: large
-
-parameters:
-  build_and_test_all:
-    type: boolean
-    default: false
-  build_and_test_backend:
-    type: boolean
-    default: false
-  build_and_test_frontend:
-    type: boolean
-    default: false
-  deploy_infrastructure:
-    type: boolean
-    default: false
-  develop_branch_deploy:
-    type: boolean
-    default: false
-  run_dev_deployment:
-    type: boolean
-    default: false
-  run_nightly_owasp_scan:
-    type: boolean
-    default: false
-  run_owasp_scan:
-    type: boolean
-    default: false
-  target_env:
-    type: string
-    default: ''
-  triggered:
-    type: boolean
-    default: false
-  util_make_erd:
-    type: boolean
-    default: false
-commands:
-# commands:
-  upload-codecov:
-    description: Uploads testing code coverage results to Codecov
-    parameters:
-      component:
-        description: The component of the application being tested, either backend or frontend.
-        type: enum
-        enum: [ "backend", "frontend" ]
-      coverage-report:
-        description: The path to the coverage report being uploaded.
-        type: string
-    steps:
-      - run:
-          name: Ensure Codecov uploader is installed, otherwise install it.
-          command: ./scripts/codecov-check.sh
-      - run:
-          name: Determine Codecov metric flag
-          command: |
-            if [ "$CIRCLE_BRANCH" == "main" ] ; then
-              CURRENT_FLAG=main-<<parameters.component>>
-            elif [ "$CIRCLE_BRANCH" == "master" ] ; then
-              CURRENT_FLAG=master-<<parameters.component>>
-            else
-              CURRENT_FLAG=dev-<<parameters.component>>
-            fi
-            echo "export CURRENT_FLAG=$CURRENT_FLAG" >> $BASH_ENV
-      - run:
-          name: Upload code coverage report if target branch
-          command: codecov -t "$CODECOV_TOKEN" -f <<parameters.coverage-report>> -F "$CURRENT_FLAG"
-
-  install-nodejs-machine:
-    description: |
-      Installs our target version of Node.JS using NVM (Node Version Manager)
-      from the install location provided by machine executor images.
-    steps:
-      - run:
-          name: Install Node.JS
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y libgbm-dev
-            source /opt/circleci/.nvm/nvm.sh
-            nvm install v16.13
-            nvm alias default v16.13
-            echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV
-            echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV
-
-  disable-npm-audit:
-    steps:
-      - run:
-          name: Disable npm audit warnings in CI
-          command: npm set audit false
-          
-  # This allows us to use the node orb to install packages within other commands
-  install-nodejs-packages: node/install-packages
-# commands:
-  deploy-cloud-dot-gov:
-    parameters:
-      environment:
-        description: The environment to deploy to.
-        type: enum
-        enum: [ "development", "production" ]
-        default: development
-      backend-appname:
-        default: tdp-backend
-        type: string
-      cf-password:
-        default: CF_PASSWORD_DEV
-        type: env_var_name
-      cf-org:
-        default: CF_ORG
-        type: env_var_name
-      cf-space:
-        default: tanf-dev
-        type: string
-      cf-username:
-        default: CF_USERNAME_DEV
-        type: env_var_name
-      frontend-appname:
-        default: tdp-frontend
-        type: string
-    steps:
-      - checkout
-      - sudo-check
-      - cf-check
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf-password>>
-          cf-org: <<parameters.cf-org>>
-          cf-space: <<parameters.cf-space>>
-          cf-username: <<parameters.cf-username>>
-      - deploy-backend:
-          backend-appname: <<parameters.backend-appname>>
-          frontend-appname: <<parameters.frontend-appname>>
-          cf-space: <<parameters.cf-space>>
-      - deploy-frontend:
-          environment: <<parameters.environment>>
-          backend-appname: <<parameters.backend-appname>>
-          frontend-appname: <<parameters.frontend-appname>>
-          cf-space: <<parameters.cf-space>>
-
-  clamav-cloud-dot-gov:
-    parameters:
-      backend-appname:
-        default: tdp-backend
-        type: string
-      cf-password:
-        default: CF_PASSWORD_DEV
-        type: env_var_name
-      cf-org:
-        default: CF_ORG
-        type: env_var_name
-      cf-space:
-        default: tanf-dev
-        type: string
-      cf-username:
-        default: CF_USERNAME_DEV
-        type: env_var_name
-    steps:
-      - checkout
-      - sudo-check
-      - cf-check
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf-password>>
-          cf-org: <<parameters.cf-org>>
-          cf-space: <<parameters.cf-space>>
-          cf-username: <<parameters.cf-username>>
-      - deploy-clamav
-
-  deploy-backend:
-    parameters:
-      backend-appname:
-        default: tdp-backend
-        type: string
-      frontend-appname:
-        default: tdp-frontend
-        type: string
-      kibana-appname:
-        default: tdp-kibana
-        type: string
-      proxy-appname:
-        default: tdp-elastic-proxy
-        type: string
-      cf-space:
-        default: tanf-dev
-        type: string
-    steps:
-      - get-app-deploy-strategy:
-          appname: <<parameters.backend-appname>>
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt update
-            sudo apt-get install -y wget
-            sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_amd64 && chmod +x /usr/local/bin/yq
-      - run:
-          name: Apply database migrations
-          command: |
-            bash ./scripts/apply-remote-migrations.sh <<parameters.backend-appname>>
-      - run:
-          name: Deploy backend application
-          command: |
-            bash ./scripts/deploy-backend.sh \
-              $DEPLOY_STRATEGY \
-              <<parameters.frontend-appname>> \
-              <<parameters.backend-appname>> \
-              <<parameters.kibana-appname>> \
-              <<parameters.proxy-appname>> \
-              <<parameters.cf-space>>
-
-  deploy-clamav:
-    steps:
-      - run:
-          name: Deploy ClamAV REST application
-          command: |
-            cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \
-              --var cf-space=tanf-prod
-
-  deploy-frontend:
-    parameters:
-      environment:
-        description: The environment to deploy to.
-        type: enum
-        enum: [ "development", "production" ]
-        default: development
-      backend-appname:
-        default: tdp-backend
-        type: string
-      frontend-appname:
-        default: tdp-frontend
-        type: string
-      kibana-appname:
-        default: tdp-kibana
-        type: string
-# So the frontend knows what space its in for the banner.
-# I am unclear if the domain is a reliable metric to make this function
-# It seems like it might not be working
-      cf-space:
-        default: dev
-        type: string
-    steps:
-      - install-nodejs:
-          node-version: "16.13"
-      - disable-npm-audit
-      - install-nodejs-packages:
-          app-dir: tdrs-frontend
-      - get-app-deploy-strategy:
-          appname: <<parameters.frontend-appname>>
-      - run:
-          name: Deploy frontend application
-          command: |
-            bash ./scripts/deploy-frontend.sh \
-              $DEPLOY_STRATEGY \
-              <<parameters.frontend-appname>> \
-              <<parameters.backend-appname>> \
-              <<parameters.kibana-appname>> \
-              <<parameters.cf-space>> \
-              <<parameters.environment>>
-
-  get-app-deploy-strategy:
-    parameters:
-      appname:
-        type: string
-    steps:
-      - run:
-          name: Determine deploy strategy
-          command: |
-            # NOTE: The || true is a no-op included to suppress exit codes which
-            #       would cause the step to exit early due to use of pipefail
-            APP_GUID=$(cf app <<parameters.appname>> --guid || true)
-            if [ "$APP_GUID" == "FAILED" ]; then
-              echo "export DEPLOY_STRATEGY=initial" >> $BASH_ENV
-            else
-              echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV
-            fi
-
-  deploy-infrastructure:
-    parameters:
-      tf-path:
-        type: string
-        default: ./terraform/dev
-      cf-password:
-        type: env_var_name
-        default: CF_PASSWORD_DEV
-      cf-username:
-        type: env_var_name
-        default: CF_USERNAME_DEV
-      cf-space:
-        type: string
-        default: tanf-dev
-      cf-org:
-        type: env_var_name
-        default: CF_ORG
-      cf-app:
-        type: string
-        default: CF_APP
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            apk update
-            apk add jq
-            apk add curl
-            # TODO: Add Signature check
-            curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx
-            mv cf7 /usr/local/bin/cf
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf-password>>
-          cf-username: <<parameters.cf-username>>
-          cf-space: <<parameters.cf-space>>
-      - run:
-          name: Export S3 Credentials for TFState
-          command: |
-            S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2)
-            {
-              echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\""
-              echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\""
-              echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\""
-              echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\""
-            } >> ./backend_config.tfvars
-      - run:
-          name: Prepare Terraform Variables
-          command: |
-            S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2)
-            {
-              echo "cf_password = \"$<<parameters.cf-password>>\""
-              echo "cf_user = \"$<<parameters.cf-username>>\""
-              echo "cf_space_name = \"<<parameters.cf-space>>\""
-              echo "cf_app_name = \"<<parameters.cf-app>>\""
-            } >> ./variables.tfvars
-      - terraform/init:
-          path: <<parameters.tf-path>>
-          backend_config_file: ./backend_config.tfvars
-      - terraform/validate:
-          path: <<parameters.tf-path>>
-      - terraform/fmt:
-          path: <<parameters.tf-path>>
-      - terraform/plan:
-          path: <<parameters.tf-path>>
-          var_file: ./variables.tfvars
-      - terraform/apply:
-          path: <<parameters.tf-path>>
-          var_file: ./variables.tfvars
-
-  enable-s3-versioning:
-    parameters:
-      target_env:
-        type: string
-      cf-password:
-        type: env_var_name
-      cf-username:
-        type: env_var_name
-      cf-space:
-        type: string
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt update
-            sudo apt install jq
-            sudo apt install curl
-            # TODO: Add Signature check
-            curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx
-            sudo mv cf7 /usr/local/bin/cf
-            sudo chmod +x /usr/local/bin/cf
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf-password>>
-          cf-username: <<parameters.cf-username>>
-          cf-space: <<parameters.cf-space>>
-      - run:
-          name: Ensure s3 versioning key exists
-          # use cf to check if a service key exists for tdp-datafiles-dev(or prod)
-          # exit 0 if it does, create it if it doesn't
-          command: |
-            chmod +x ./scripts/create_s3_versioning_key.sh
-            ./scripts/create_s3_versioning_key.sh tdp-datafiles-<<parameters.target_env>> s3-versioning-key
-      - run:
-          name: Set up aws credentials
-          command: |
-            S3_CREDENTIALS=$(cf service-key tdp-datafiles-<<parameters.target_env>> s3-versioning-key | tail -n +2)
-            aws configure set aws_access_key_id $(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)
-            aws configure set aws_secret_access_key $(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)
-            aws configure set default.region $(echo "${S3_CREDENTIALS}" | jq -r '.region')
-      - run:
-          name: Enable S3 Versioning
-          command: |
-            S3_BUCKET_ID=cg-$(cf service tdp-datafiles-<<parameters.target_env>> --guid)
-            aws s3api put-bucket-versioning --bucket ${S3_BUCKET_ID} --versioning-configuration Status=Enabled
-# commands:
-  run-owasp-scan:
-    description: Runs OWASP ZAP scanner and stores resulting artifacts.
-    parameters:
-      environment:
-        description: The environment this script is being run for.
-        type: enum
-        enum: ["circle", "nightly"] 
-      target:
-        description: The target application to be scanned.
-        type: enum
-        enum: ["backend", "frontend"]
-      target_env:
-        description: The target environment to be scanned.
-        type: enum
-        default: "develop"
-        enum: ["staging", "develop", "prod"]
-    steps:
-      - run:
-          name: Execute OWASP ZAP vulnerability scan
-          #wait up to 60min for scan
-          no_output_timeout: 60m
-          command: ./scripts/zap-scanner.sh <<parameters.target>> <<parameters.environment>> <<parameters.target_env>>
-      - store_artifacts:
-          path: tdrs-<<parameters.target>>/reports/owasp_report.html
-
-  docker-compose-up-frontend:
-    steps:
-      - run:
-          name: Build and spin-up React application
-          command: |
-            cd tdrs-frontend 
-            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
-            docker network create external-net
-            fi
-            docker-compose up -d --build
-# commands:
-  docker-compose-check:
-    steps:
-      - run:
-          name: Ensure docker-compose exists, otherwise install it.
-          command: ./scripts/docker-compose-check.sh
-
-  docker-compose-up-backend:
-    steps:
-      - run:
-          name: Build and spin-up Django API service
-          command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
-
-  docker-compose-up-with-elastic-backend:
-    steps:
-      - run:
-          name: Build and spin-up Django API service
-          command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build
-
-  cf-check:
-    steps:
-      - run:
-          name: Ensure cf cli is installed, otherwise install it.
-          command: sudo ./scripts/cf-check.sh
-
-  # This allows us to use the orb stanza for node/install within other commands
-  # NOTE: This doesn't work correctly on machine executors
-  install-nodejs: node/install
-
-  sudo-check:
-    steps:
-      - run:
-          name: Ensure sudo is installed, otherwise install it.
-          command: ./scripts/sudo-check.sh
-
-  login-cloud-dot-gov:
-    description: Authenticates with Cloud.gov and sets org and space targets
-    parameters:
-      cf-password:
-        type: env_var_name
-        default: CF_PASSWORD_DEV
-      cf-org:
-        type: env_var_name
-        default: CF_ORG
-      cf-space:
-        type: string
-        default: tanf-dev
-      cf-username:
-        type: env_var_name
-        default: CF_USERNAME_DEV
-    steps:
-      - run:
-          name: Login to Cloud.gov and set application targets
-          command: |
-            cf login -a https://api.fr.cloud.gov \
-              -u ${<<parameters.cf-username>>} \
-              -p ${<<parameters.cf-password>>} \
-              -o ${<<parameters.cf-org>>} \
-              -s <<parameters.cf-space>>
-jobs:
-  test-backend:
-    executor: machine-executor
-    steps:
-      - checkout
-      - docker-compose-check
-      - docker-compose-up-with-elastic-backend
-      - run:
-          name: Run Unit Tests And Create Code Coverage Report
-          command: |
-            cd tdrs-backend;
-            docker-compose run --rm web bash -c "./wait_for_services.sh && pytest --cov-report=xml"
-      - run:
-          name: Execute Python Linting Test
-          command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ."
-      - upload-codecov:
-          component: backend
-          coverage-report: ./tdrs-backend/coverage.xml
-
-  test-frontend:
-    executor: machine-executor
-    working_directory: ~/tdp-apps
-    steps:
-      - checkout
-      - install-nodejs-machine
-      - disable-npm-audit
-      - install-nodejs-packages:
-          app-dir: tdrs-frontend
-      - run:
-          name: Run ESLint
-          command: cd tdrs-frontend; npm run lint
-      - run:
-          name: Run Pa11y Accessibility Tests
-          command: cd tdrs-frontend; mkdir pa11y-screenshots/; export NODE_OPTIONS=--openssl-legacy-provider; npm run test:accessibility
-      - run:
-          name: Run Jest Unit Tests
-          command: cd tdrs-frontend; npm run test:ci
-      - upload-codecov:
-          component: frontend
-          coverage-report: ./tdrs-frontend/coverage/lcov.info
-      - store_artifacts:
-          path: tdrs-frontend/pa11y-screenshots/
-
-  test-e2e:
-    executor: large-machine-executor
-    working_directory: ~/tdp-apps
-    steps:
-      - checkout
-      - docker-compose-check
-      - docker-compose-up-with-elastic-backend
-      - docker-compose-up-frontend
-      - install-nodejs-machine
-      - disable-npm-audit
-      - install-nodejs-packages:
-          app-dir: tdrs-frontend
-      - run:
-          name: Wait for backend to become available
-          command: cd tdrs-backend; docker-compose run --rm zaproxy bash -c \
-              "PATH=$PATH:/home/zap/.local/bin &&
-               pip install wait-for-it &&
-               wait-for-it --service http://web:8080 --timeout 180 -- echo \"Django is ready\""
-      - run:
-          name: apply the migrations
-          command: cd tdrs-backend; docker-compose exec web bash -c "python manage.py makemigrations; python manage.py migrate"
-      - run:
-          name: Remove existing cypress test users
-          command: cd tdrs-backend; docker-compose exec web python manage.py delete_cypress_users -usernames new-cypress@teamraft.com cypress-admin@teamraft.com
-      - run:
-          name: Setup cypress test users
-          command: cd tdrs-backend; docker-compose exec web python manage.py generate_cypress_users
-      - run:
-          name: Run Cypress e2e tests
-          command: cd tdrs-frontend; npm run test:e2e-ci
-      - store_artifacts:
-          path: tdrs-frontend/cypress/screenshots/
-      - store_artifacts:
-          path: tdrs-frontend/cypress/videos/
-
-  secrets-check:
-    executor: docker-executor
-    steps:
-      - checkout
-      - run:
-          name: "git-secrets: Scan repository for committed secrets"
-          command: ./scripts/git-secrets-check.sh
-      - run:
-          name: "trufflehog: Scan repository for committed secrets"
-          command: ./scripts/trufflehog-check.sh $CIRCLE_BRANCH
-  deploy-dev:
-    parameters:
-      target_env:
-        type: string
-    executor: docker-executor
-    working_directory: ~/tdp-deploy
-    steps:
-      - deploy-cloud-dot-gov:
-          backend-appname: tdp-backend-<< parameters.target_env >>
-          frontend-appname: tdp-frontend-<< parameters.target_env >>
-
-  deploy-staging:
-    executor: docker-executor
-    working_directory: ~/tdp-deploy
-    steps:
-      - deploy-cloud-dot-gov:
-          backend-appname: tdp-backend-staging
-          frontend-appname: tdp-frontend-staging
-          cf-password: CF_PASSWORD_STAGING
-          cf-space: tanf-staging
-          cf-username: CF_USERNAME_STAGING
-
-  deploy-develop:
-    executor: docker-executor
-    working_directory: ~/tdp-deploy
-    steps:
-      - deploy-cloud-dot-gov:
-          backend-appname: tdp-backend-develop
-          frontend-appname: tdp-frontend-develop
-          cf-password: CF_PASSWORD_STAGING
-          cf-space: tanf-staging
-          cf-username: CF_USERNAME_STAGING
-
-  deploy-infrastructure-dev:
-    parameters:
-      target_env:
-        type: string
-    executor: terraform/default
-    working_directory: ~/tdp-deploy
-    steps:
-      - deploy-infrastructure:
-                cf-app: << parameters.target_env >>
-
-  deploy-infrastructure-staging:
-    executor: terraform/default
-    working_directory: ~/tdp-deploy
-    steps:
-      - deploy-infrastructure:
-          cf-password: CF_PASSWORD_STAGING
-          cf-username: CF_USERNAME_STAGING
-          cf-space: tanf-staging
-          tf-path: ./terraform/staging
-
-  deploy-infrastructure-production:
-    executor: terraform/default
-    working_directory: ~/tdp-deploy
-    steps:
-      - deploy-infrastructure:
-          cf-password: CF_PASSWORD_PROD
-          cf-username: CF_USERNAME_PROD
-          cf-space: tanf-prod
-          tf-path: ./terraform/production
-
-  deploy-project-updates-site:
-    parameters:
-      cf-org:
-        default: CF_ORG
-        type: env_var_name
-      cf-space:
-        default: tanf-dev
-        type: string
-      cf-password:
-        type: env_var_name
-        default: CF_PASSWORD_DEV
-      cf-username:
-        type: env_var_name
-        default: CF_USERNAME_DEV
-    executor: docker-executor
-    working_directory: ~/tdp-deploy
-    steps:
-      - checkout
-      - sudo-check
-      - cf-check
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf-password>>
-          cf-org: <<parameters.cf-org>>
-          cf-space: <<parameters.cf-space>>
-          cf-username: <<parameters.cf-username>>
-      - run:
-          name: Deploy TDP Project Updates Site
-          command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates
-
-  enable-versioning:
-    executor: machine-executor
-    parameters:
-      target_env:
-        type: string
-        default: dev
-      cf-password:
-        type: env_var_name
-        default: CF_PASSWORD_DEV
-      cf-username:
-        type: env_var_name
-        default: CF_USERNAME_DEV
-      cf-space:
-        type: string
-        default: tanf-dev
-    steps:
-      - enable-s3-versioning:
-          target_env: <<parameters.target_env>>
-          cf-password: <<parameters.cf-password>>
-          cf-username: <<parameters.cf-username>>
-          cf-space: <<parameters.cf-space>>
-
-  test-deployment-e2e:
-    executor: machine-executor
-    working_directory: ~/tdp-apps
-    steps:
-      - checkout
-      - install-nodejs-machine
-      - disable-npm-audit
-      - install-nodejs-packages:
-          app-dir: tdrs-frontend
-      - run:
-          name: Run Cypress e2e tests
-          command: cd tdrs-frontend; npm run test:e2e-ci -- --config baseUrl="https://tdp-frontend-develop.acf.hhs.gov" --env cypressToken=$CYPRESS_TOKEN,apiUrl="https://tdp-frontend-develop.acf.hhs.gov/v1",adminUrl="https://tdp-frontend-develop.acf.hhs.gov/admin"
-      - store_artifacts:
-          path: tdrs-frontend/cypress/screenshots/
-      - store_artifacts:
-          path: tdrs-frontend/cypress/videos/
-
-  deploy-production:
-    executor: docker-executor
-    working_directory: ~/tdp-deploy
-    steps:
-      - deploy-cloud-dot-gov:
-          environment: production
-          backend-appname: tdp-backend-prod
-          frontend-appname: tdp-frontend-prod
-          cf-password: CF_PASSWORD_PROD
-          cf-space: tanf-prod
-          cf-username: CF_USERNAME_PROD
-  prod-deploy-clamav:
-    executor: docker-executor
-    working_directory: ~/tdp-deploy
-    steps:
-      - clamav-cloud-dot-gov:
-          backend-appname: tdp-backend-prod
-          cf-password: CF_PASSWORD_PROD
-          cf-space: tanf-prod
-          cf-username: CF_USERNAME_PROD
-  backend-owasp-scan:
-    executor: large-machine-executor
-    working_directory: ~/tdp-apps
-    steps:
-      - checkout
-      - docker-compose-check
-      - docker-compose-up-backend
-      - docker-compose-up-frontend
-      - run:
-          name: Wait for Django to become available
-          command: |
-            cd tdrs-backend;
-            docker-compose run --rm zaproxy bash -c \
-              "PATH=$PATH:/home/zap/.local/bin &&
-               pip install wait-for-it &&
-               wait-for-it --service http://web:8080 \
-                           --timeout 60 \
-                           -- echo \"Django is ready\""
-      - run-owasp-scan:
-          environment: circle
-          target: backend
-
-  frontend-owasp-scan:
-    executor: large-machine-executor
-    working_directory: ~/tdp-apps
-    steps:
-      - checkout
-      - docker-compose-check
-      - docker-compose-up-backend
-      - docker-compose-up-frontend
-      - run:
-          name: Wait for frontend to become available
-          command: |
-            cd tdrs-frontend;
-            docker-compose run --rm zaproxy bash -c \
-              "PATH=$PATH:/home/zap/.local/bin &&
-               pip install wait-for-it &&
-               wait-for-it --service http://tdp-frontend/ \
-                           --timeout 60 \
-                           -- echo \"Frontend is ready\""
-      - run-owasp-scan:
-          environment: circle
-          target: frontend
-
-  nightly-owasp-scan:
-    executor: large-machine-executor
-    working_directory: ~/tdp-apps
-    parameters:
-      cf_password:
-        type: string
-        default: CF_PASSWORD_STAGING
-      cf_username:
-        type: string
-        default: CF_USERNAME_STAGING
-      cf_space:
-        type: string
-        default: tanf-staging
-      cf_org:
-        type: string
-        default: "CF_ORG"
-      target_env:
-        type: enum
-        enum: [ "staging", "develop", "prod" ]
-    steps:
-      - checkout
-      - sudo-check
-      - cf-check
-      - docker-compose-check
-      - login-cloud-dot-gov:
-          cf-password: <<parameters.cf_password>>
-          cf-space: <<parameters.cf_space>>
-          cf-org: <<parameters.cf_org>>
-          cf-username: <<parameters.cf_username>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: backend
-      #     target_env: <<parameters.target_env>>
-      - run-owasp-scan:
-          environment: nightly
-          target: frontend
-          target_env: <<parameters.target_env>>
-      - run:
-          name: Run post-processing task to record OWASP ZAP results
-          command: |
-            # Construct the project slug from the current branch name and user
-            PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-
-            # These environment variables are exported to Circle CI's BASH_ENV
-            # by the zap-scanner.sh script for each respective app target.
-            CMD_ARGS=(
-              "$CIRCLE_BUILD_NUM"
-              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
-              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
-              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
-              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
-              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
-              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
-              --project-slug "$PROJECT_SLUG"
-            )
-            # Evaluate the full command before passing it in so it doesn't
-            # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
-            echo "$CMD"
-
-            # Submit a CF Task for execution that will run the necessary command
-            echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!
-  make_erd:
-    executor: machine-executor
-    working_directory: ~/tdp_apps
-    steps:
-      - checkout
-      - docker-compose-check
-      - run:
-          name: Run graph_models
-          command: |
-            cd tdrs-backend
-            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then
-            docker network create external-net
-            fi
-            docker-compose run --rm web bash -c \
-            "./manage.py graph_models -a -g -o tdp_erd.png"
-      - store_artifacts:
-          path: tdrs-backend/tdp_erd.png
-workflows:
-# workflows:
-  build-and-test-all:
-    when: << pipeline.parameters.build_and_test_all >>
-    jobs:
-      - secrets-check
-      # TODO: Revert back to tests after review
-      - nightly-owasp-scan:
-          target_env: staging
-
-  ci-build-and-test-all:
-    jobs:
-      - secrets-check:
-          filters:
-            branches:
-              only:
-                - main
-                - master
-                - /^release.*/
-      - test-backend:
-          filters:
-            branches:
-              only:
-                - main
-                - master
-                - /^release.*/
-          requires:
-            - secrets-check
-      - test-frontend:
-          filters:
-            branches:
-              only:
-                - main
-                - master
-                - /^release.*/
-          requires:
-            - secrets-check
-      - test-e2e:
-          filters:
-            branches:
-              only:
-                - main
-                - master
-                - /^release.*/
-          requires:
-            - secrets-check
-
-  build-and-test-backend:
-    when: << pipeline.parameters.build_and_test_backend >>
-    jobs:
-      - secrets-check
-      - test-backend:
-          requires:
-            - secrets-check
-
-  build-and-test-frontend:
-    when: << pipeline.parameters.build_and_test_frontend >>
-    jobs:
-      - secrets-check
-      - test-frontend:
-          requires:
-            - secrets-check
-#workflows:
-  deployment:
-    when:
-      and:
-        - or:
-            - equal: [ master, << pipeline.git.branch >> ]
-            - equal: [ main, << pipeline.git.branch >> ]
-            - equal: [ develop, << pipeline.git.branch >> ]
-            - << pipeline.parameters.run_dev_deployment >>
-        - not: << pipeline.parameters.run_nightly_owasp_scan >>
-
-    jobs:
-      - deploy-project-updates-site:
-          filters:
-            branches:
-              only:
-                - develop
-      - deploy-infrastructure-dev:
-          target_env: << pipeline.parameters.target_env >>
-          filters:
-            branches:
-              ignore:
-                - develop
-                - main
-                - master
-      - deploy-infrastructure-staging:
-          filters:
-            branches:
-              only:
-                - develop
-                - main
-      - deploy-infrastructure-production:
-          filters:
-            branches:
-              only:
-                - master
-      - enable-versioning:
-          requires:
-            - deploy-infrastructure-dev
-          filters:
-            branches:
-              ignore:
-                - develop
-                - main
-                - master
-      - enable-versioning:
-          requires:
-            - deploy-infrastructure-staging
-          target_env: develop
-          cf-password: CF_PASSWORD_STAGING
-          cf-username: CF_USERNAME_STAGING
-          cf-space: tanf-staging
-          filters:
-            branches:
-              only:
-                - develop
-      - enable-versioning:
-          requires:
-            - deploy-infrastructure-staging
-          target_env: staging
-          cf-password: CF_PASSWORD_STAGING
-          cf-username: CF_USERNAME_STAGING
-          cf-space: tanf-staging
-          filters:
-            branches:
-              only:
-                - main
-      - enable-versioning:
-          requires:
-            - deploy-infrastructure-production
-          target_env: prod
-          cf-password: CF_PASSWORD_PROD
-          cf-username: CF_USERNAME_PROD
-          cf-space: tanf-prod
-          filters:
-            branches:
-              only:
-                - master
-      - prod-deploy-clamav:
-          requires:
-            - deploy-infrastructure-production
-          filters:
-            branches:
-              only:
-                - master
-      - deploy-dev:
-          target_env: << pipeline.parameters.target_env >>
-          requires:
-            - deploy-infrastructure-dev
-          filters:
-            branches:
-              ignore:
-                - develop
-                - main
-                - master
-      - deploy-develop:
-          requires:
-            - deploy-infrastructure-staging
-          filters:
-            branches:
-              only:
-                - develop
-      - deploy-staging:
-          requires:
-            - deploy-infrastructure-staging
-          filters:
-            branches:
-              only:
-                - main
-      - deploy-production:
-          requires:
-            - deploy-infrastructure-production
-          filters:
-            branches:
-              only:
-                - master
-      - test-deployment-e2e:
-          requires:
-            - deploy-develop
-          filters:
-            branches:
-              only:
-                - develop
-      - make_erd: # from ../util folder
-          filters:
-            branches:
-              only:
-                - develop
-                - master
-# workflows:
-  owasp-scan:
-    when: << pipeline.parameters.run_owasp_scan >>
-    jobs:
-      - backend-owasp-scan
-      - frontend-owasp-scan
-  
-  nightly:
-    when: << pipeline.parameters.run_nightly_owasp_scan >>
-    jobs:
-      - nightly-owasp-scan:
-          target_env: develop
-          filters:
-            branches:
-              only:
-                - develop
-      - nightly-owasp-scan:
-          target_env: staging
-          filters:
-            branches:
-              only:
-                - main
-      - nightly-owasp-scan:
-          target_env: prod
-          cf_password: CF_PASSWORD_PROD
-          cf_username: CF_USERNAME_PROD
-          cf_space: tanf-prod
-          filters:
-            branches:
-              only:
-                - master
diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 72ace6e4b..78ea200ce 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -103,4 +103,5 @@
             echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
+            cat .circleci/generated_config.yml
             echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!

From 332da65f48d5e4a06ba742e3999047c3a695f1ae Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 14:45:54 -0600
Subject: [PATCH 100/127] - remove cat

---
 .circleci/owasp/jobs.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 78ea200ce..72ace6e4b 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -103,5 +103,4 @@
             echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
-            cat .circleci/generated_config.yml
             echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!

From e41fb86f126380d703483114efb41fb47b82d08d Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 15:09:33 -0600
Subject: [PATCH 101/127] - trying with sscript

---
 .circleci/owasp/jobs.yml |  9 ++++++---
 scripts/cf-runtask.sh    | 14 ++++++++++++++
 2 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100755 scripts/cf-runtask.sh

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 72ace6e4b..c867b5bad 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -97,10 +97,13 @@
               --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
               --project-slug "$PROJECT_SLUG"
             )
+
+            ./scripts/cf-runtask.sh "$PROJECT_SLUG" "$CMD_ARGS" << parameters.target_env >>
+
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
-            echo "$CMD"
+            # CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+            # echo "$CMD"
 
             # Submit a CF Task for execution that will run the necessary command
-            echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!
+            # echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!
diff --git a/scripts/cf-runtask.sh b/scripts/cf-runtask.sh
new file mode 100755
index 000000000..c15c92621
--- /dev/null
+++ b/scripts/cf-runtask.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# pipefail is needed to correctly carry over the exit code from zap-full-scan.py
+set -uxo pipefail
+
+PROJECT_SLUG=$1
+CMD_ARGS=$2
+TARGET=$3
+
+CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+echo $CMD
+echo START!
+cf run-task tdp-backend-$TARGET --wait --command "$CMD" --name nightly-owasp-scan
+echo DONE!
\ No newline at end of file

From 239ec16fc6b0e7a2dc373dff023df5fe949170b6 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 15:24:31 -0600
Subject: [PATCH 102/127] - all params

---
 .circleci/owasp/jobs.yml | 2 +-
 scripts/cf-runtask.sh    | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index c867b5bad..8132f0083 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -98,7 +98,7 @@
               --project-slug "$PROJECT_SLUG"
             )
 
-            ./scripts/cf-runtask.sh "$PROJECT_SLUG" "$CMD_ARGS" << parameters.target_env >>
+            ./scripts/cf-runtask.sh "$PROJECT_SLUG" "$CMD_ARGS" << parameters.target_env >> "$CIRCLE_BUILD_NUM" "${ZAP_BACKEND_PASS_COUNT:-0}" "${ZAP_BACKEND_WARN_COUNT:-0}" "${ZAP_BACKEND_FAIL_COUNT:-0}" "${ZAP_FRONTEND_PASS_COUNT:-0}" "${ZAP_FRONTEND_WARN_COUNT:-0}" "${ZAP_FRONTEND_FAIL_COUNT:-0}"
 
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
diff --git a/scripts/cf-runtask.sh b/scripts/cf-runtask.sh
index c15c92621..796034ae0 100755
--- a/scripts/cf-runtask.sh
+++ b/scripts/cf-runtask.sh
@@ -6,8 +6,15 @@ set -uxo pipefail
 PROJECT_SLUG=$1
 CMD_ARGS=$2
 TARGET=$3
+BUILD_NO=$4
+ZAP_BACKEND_PASS_COUNT=$5
+ZAP_BACKEND_WARN_COUNT=$6
+ZAP_BACKEND_FAIL_COUNT=$7
+ZAP_FRONTEND_PASS_COUNT=$8
+ZAP_FRONTEND_WARN_COUNT=$9
+ZAP_FRONTEND_FAIL_COUNT=$10
 
-CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
 echo $CMD
 echo START!
 cf run-task tdp-backend-$TARGET --wait --command "$CMD" --name nightly-owasp-scan

From 389f9e96ea268e90b1e5e58af2a7e2b90c219e68 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 15:39:46 -0600
Subject: [PATCH 103/127] - more logging

---
 .circleci/owasp/jobs.yml | 24 +-----------------------
 scripts/cf-runtask.sh    | 24 +++++++++++++-----------
 2 files changed, 14 insertions(+), 34 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 8132f0083..69e021ae1 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -84,26 +84,4 @@
           command: |
             # Construct the project slug from the current branch name and user
             PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-
-            # These environment variables are exported to Circle CI's BASH_ENV
-            # by the zap-scanner.sh script for each respective app target.
-            CMD_ARGS=(
-              "$CIRCLE_BUILD_NUM"
-              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
-              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
-              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
-              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
-              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
-              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
-              --project-slug "$PROJECT_SLUG"
-            )
-
-            ./scripts/cf-runtask.sh "$PROJECT_SLUG" "$CMD_ARGS" << parameters.target_env >> "$CIRCLE_BUILD_NUM" "${ZAP_BACKEND_PASS_COUNT:-0}" "${ZAP_BACKEND_WARN_COUNT:-0}" "${ZAP_BACKEND_FAIL_COUNT:-0}" "${ZAP_FRONTEND_PASS_COUNT:-0}" "${ZAP_FRONTEND_WARN_COUNT:-0}" "${ZAP_FRONTEND_FAIL_COUNT:-0}"
-
-            # Evaluate the full command before passing it in so it doesn't
-            # get improperly interpolated by Cloud.gov.
-            # CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
-            # echo "$CMD"
-
-            # Submit a CF Task for execution that will run the necessary command
-            # echo START! && cf run-task tdp-backend-<< parameters.target_env >> --wait --command "$CMD" --name nightly-owasp-scan && echo DONE!
+            ./scripts/cf-runtask.sh "$PROJECT_SLUG" << parameters.target_env >> "$CIRCLE_BUILD_NUM" "${ZAP_BACKEND_PASS_COUNT:-0}" "${ZAP_BACKEND_WARN_COUNT:-0}" "${ZAP_BACKEND_FAIL_COUNT:-0}" "${ZAP_FRONTEND_PASS_COUNT:-0}" "${ZAP_FRONTEND_WARN_COUNT:-0}" "${ZAP_FRONTEND_FAIL_COUNT:-0}"
diff --git a/scripts/cf-runtask.sh b/scripts/cf-runtask.sh
index 796034ae0..18494d161 100755
--- a/scripts/cf-runtask.sh
+++ b/scripts/cf-runtask.sh
@@ -4,18 +4,20 @@
 set -uxo pipefail
 
 PROJECT_SLUG=$1
-CMD_ARGS=$2
-TARGET=$3
-BUILD_NO=$4
-ZAP_BACKEND_PASS_COUNT=$5
-ZAP_BACKEND_WARN_COUNT=$6
-ZAP_BACKEND_FAIL_COUNT=$7
-ZAP_FRONTEND_PASS_COUNT=$8
-ZAP_FRONTEND_WARN_COUNT=$9
-ZAP_FRONTEND_FAIL_COUNT=$10
+TARGET=$2
+BUILD_NO=$3
+ZAP_BACKEND_PASS_COUNT=$4
+ZAP_BACKEND_WARN_COUNT=$5
+ZAP_BACKEND_FAIL_COUNT=$6
+ZAP_FRONTEND_PASS_COUNT=$7
+ZAP_FRONTEND_WARN_COUNT=$8
+ZAP_FRONTEND_FAIL_COUNT=$9
 
 CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
-echo $CMD
+echo .
+echo Executing command: $CMD
+echo .
 echo START!
-cf run-task tdp-backend-$TARGET --wait --command "$CMD" --name nightly-owasp-scan
+APP=tdp-backend-$TARGET
+cf run-task "$APP" --wait --command "$CMD" --name "nightly-owasp-scan"
 echo DONE!
\ No newline at end of file

From 26f432e98e2f3497f632415bf0f39a12b31bd0c5 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 15:56:42 -0600
Subject: [PATCH 104/127] - testing different command - no scan

---
 .circleci/owasp/jobs.yml | 8 ++++----
 scripts/cf-runtask.sh    | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 69e021ae1..92478d972 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -75,10 +75,10 @@
       #     environment: nightly
       #     target: backend
       #     target_env: <<parameters.target_env>>
-      - run-owasp-scan:
-          environment: nightly
-          target: frontend
-          target_env: <<parameters.target_env>>
+      # - run-owasp-scan:
+      #     environment: nightly
+      #     target: frontend
+      #     target_env: <<parameters.target_env>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |
diff --git a/scripts/cf-runtask.sh b/scripts/cf-runtask.sh
index 18494d161..c61569951 100755
--- a/scripts/cf-runtask.sh
+++ b/scripts/cf-runtask.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # pipefail is needed to correctly carry over the exit code from zap-full-scan.py
-set -uxo pipefail
+set -uo pipefail
 
 PROJECT_SLUG=$1
 TARGET=$2
@@ -19,5 +19,5 @@ echo Executing command: $CMD
 echo .
 echo START!
 APP=tdp-backend-$TARGET
-cf run-task "$APP" --wait --command "$CMD" --name "nightly-owasp-scan"
-echo DONE!
\ No newline at end of file
+cf run-task "$APP" --wait --command "python -V" --name "nightly-owasp-scan"
+echo DONE!

From 44e50cebf566e528090641df13439d396cd99c3a Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 15:58:30 -0600
Subject: [PATCH 105/127] - enable scan

---
 .circleci/owasp/jobs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 92478d972..69e021ae1 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -75,10 +75,10 @@
       #     environment: nightly
       #     target: backend
       #     target_env: <<parameters.target_env>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: frontend
-      #     target_env: <<parameters.target_env>>
+      - run-owasp-scan:
+          environment: nightly
+          target: frontend
+          target_env: <<parameters.target_env>>
       - run:
           name: Run post-processing task to record OWASP ZAP results
           command: |

From 97442873568fc4436e7a04726a2709f88de65591 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Fri, 12 Apr 2024 18:27:13 -0600
Subject: [PATCH 106/127] - remove unnecessary fields from list display

---
 tdrs-backend/tdpservice/search_indexes/admin/ssp.py    | 8 --------
 tdrs-backend/tdpservice/search_indexes/admin/tanf.py   | 9 ---------
 tdrs-backend/tdpservice/search_indexes/admin/tribal.py | 8 --------
 3 files changed, 25 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index e9f389640..e5f1b0f89 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -9,10 +9,6 @@ class SSP_M1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
-        'COUNTY_FIPS_CODE',
-        'ZIP_CODE',
-        'STRATUM',
         'datafile',
         'stt_code',
     ]
@@ -33,7 +29,6 @@ class SSP_M2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -52,7 +47,6 @@ class SSP_M3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -70,7 +64,6 @@ class SSP_M4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -88,7 +81,6 @@ class SSP_M5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index 0562e174c..b9fe84bf9 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -9,10 +9,6 @@ class TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
-        'COUNTY_FIPS_CODE',
-        'ZIP_CODE',
-        'STRATUM',
         'datafile',
         'stt_code',
     ]
@@ -33,7 +29,6 @@ class TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -52,7 +47,6 @@ class TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -71,7 +65,6 @@ class TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -90,7 +83,6 @@ class TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -109,7 +101,6 @@ class TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'CALENDAR_QUARTER',
-        'RPT_MONTH_YEAR',
         'datafile',
         'stt_code',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index 6988a59b4..e4ca82f06 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -9,10 +9,6 @@ class Tribal_TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
-        'COUNTY_FIPS_CODE',
-        'ZIP_CODE',
-        'STRATUM',
         'datafile',
         'stt_code',
     ]
@@ -33,7 +29,6 @@ class Tribal_TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -52,7 +47,6 @@ class Tribal_TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -70,7 +64,6 @@ class Tribal_TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]
@@ -87,7 +80,6 @@ class Tribal_TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     list_display = [
         'RecordType',
         'RPT_MONTH_YEAR',
-        'CASE_NUMBER',
         'datafile',
         'stt_code',
     ]

From 0c783987e78bb1a5ee43aaa6cd07db19bee378db Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 08:06:19 -0600
Subject: [PATCH 107/127] - Update based on ux feedback

---
 tdrs-backend/tdpservice/parsers/test/test_parse.py      | 3 ++-
 tdrs-backend/tdpservice/parsers/test/test_validators.py | 3 ++-
 tdrs-backend/tdpservice/parsers/validators.py           | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py
index 240e74419..f0593abe6 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_parse.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py
@@ -1697,6 +1697,7 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs):
 
     assert parser_errors.count() == 3
 
-    parser_errors.first().error_message == "T7: 2020  is an invalid value for the CALENDAR_QUARTER field."
+    parser_errors.first().error_message == "T7: 2020  is invalid. Calendar Quarter must be a numeric" + \
+        "representing the Calendar Year and Quarter formatted as YYYYQ"
 
     Tribal_TANF_T7.objects.count() == 0
diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py
index 350cde40f..e80d35f65 100644
--- a/tdrs-backend/tdpservice/parsers/test/test_validators.py
+++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py
@@ -494,7 +494,8 @@ def test_calendarQuarterIsValid_returns_invalid(value):
     is_valid, error_msg = val(value, RowSchema(), "friendly_name", "item_no")
 
     assert is_valid is False
-    assert error_msg == f"T1: {value[2:7]} is an invalid value for the CALENDAR_QUARTER field."
+    assert error_msg == f"T1: {value[2:7]} is invalid. Calendar Quarter must be a numeric " + \
+        "representing the Calendar Year and Quarter formatted as YYYYQ"
 
 
 @pytest.mark.parametrize("value", ["T720201", "T720202", "T720203", "T720204"])
diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py
index 463895e63..99c8c9db4 100644
--- a/tdrs-backend/tdpservice/parsers/validators.py
+++ b/tdrs-backend/tdpservice/parsers/validators.py
@@ -378,7 +378,8 @@ def calendarQuarterIsValid(start=0, end=None):
         lambda value,
         row_schema,
         friendly_name,
-        item_num: f"{row_schema.record_type}: {value[start:end]} is an invalid value for the CALENDAR_QUARTER field.",
+        item_num: f"{row_schema.record_type}: {value[start:end]} is invalid. Calendar Quarter must be a numeric "
+        "representing the Calendar Year and Quarter formatted as YYYYQ",
     )
 
 

From 626e2187a142ee90ea7641f0a38b58583c8f5d32 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 08:19:10 -0600
Subject: [PATCH 108/127] - Update to go from 2021 to current year

---
 tdrs-backend/tdpservice/search_indexes/admin/filters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index d5205bd55..ea97a6a45 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -116,7 +116,7 @@ def lookups(self, request, model_admin):
         current_year = datetime.date.today().year
         quarters = [1, 2, 3, 4]
         months = ["(Oct - Dec)", "(Jan - Mar)", "(Apr - Jun)", "(Jul - Sep)"]
-        years = [year for year in range(current_year - 5, current_year + 1)]
+        years = [year for year in range(current_year - 3, current_year + 1)]
         options = [(None, _('All'))]
 
         for year in years:

From 8376973669c4ecf3e525d8e9d12541e2c6b45829 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 08:56:48 -0600
Subject: [PATCH 109/127] - testing a sleep

---
 scripts/cf-runtask.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/cf-runtask.sh b/scripts/cf-runtask.sh
index c61569951..c08325a94 100755
--- a/scripts/cf-runtask.sh
+++ b/scripts/cf-runtask.sh
@@ -15,7 +15,7 @@ ZAP_FRONTEND_FAIL_COUNT=$9
 
 CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
 echo .
-echo Executing command: $CMD
+sleep 120
 echo .
 echo START!
 APP=tdp-backend-$TARGET

From 3de74937f479285878826db2f487aeef5502f28c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 09:09:51 -0600
Subject: [PATCH 110/127] - running command

---
 scripts/cf-runtask.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/cf-runtask.sh b/scripts/cf-runtask.sh
index c08325a94..9578062f0 100755
--- a/scripts/cf-runtask.sh
+++ b/scripts/cf-runtask.sh
@@ -17,7 +17,6 @@ CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count
 echo .
 sleep 120
 echo .
-echo START!
+echo Executing command: $CMD
 APP=tdp-backend-$TARGET
-cf run-task "$APP" --wait --command "python -V" --name "nightly-owasp-scan"
-echo DONE!
+cf run-task "$APP" --wait --command "$CMD" --name "nightly-owasp-scan"

From 273adbcce8eff4cfa17e1026721b3a2e58f4be73 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 09:30:27 -0600
Subject: [PATCH 111/127] - try both scans

---
 .circleci/owasp/jobs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 69e021ae1..351911be8 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -71,10 +71,10 @@
           cf-space: <<parameters.cf_space>>
           cf-org: <<parameters.cf_org>>
           cf-username: <<parameters.cf_username>>
-      # - run-owasp-scan:
-      #     environment: nightly
-      #     target: backend
-      #     target_env: <<parameters.target_env>>
+      - run-owasp-scan:
+          environment: nightly
+          target: backend
+          target_env: <<parameters.target_env>>
       - run-owasp-scan:
           environment: nightly
           target: frontend

From cde3936eaf70bf0b7fa7ca678a7fe465e8ca4f77 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 09:44:45 -0600
Subject: [PATCH 112/127] - extra long sleep

---
 scripts/cf-runtask.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/cf-runtask.sh b/scripts/cf-runtask.sh
index 9578062f0..1cec24d9c 100755
--- a/scripts/cf-runtask.sh
+++ b/scripts/cf-runtask.sh
@@ -15,7 +15,7 @@ ZAP_FRONTEND_FAIL_COUNT=$9
 
 CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
 echo .
-sleep 120
+sleep 300
 echo .
 echo Executing command: $CMD
 APP=tdp-backend-$TARGET

From bb4fd293e3dff5311b02439d9b7c945f268d8466 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 10:09:21 -0600
Subject: [PATCH 113/127] - back to 10 min

---
 scripts/zap-scanner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 4f81b027a..145787ca8 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -137,7 +137,7 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(21\).enabled=true \
   -config spider.postform=true"
 # How long ZAP will crawl the app with the spider process
-ZAP_SPIDER_MINS=2
+ZAP_SPIDER_MINS=10
 
 ZAP_ARGS=(-t "$APP_URL" -m "$ZAP_SPIDER_MINS" -r "$REPORT_NAME" -z "$ZAP_CLI_OPTIONS")
 if [ -z ${CONFIG_FILE+x} ]; then

From dcffc436bf392e62ceeff561773546677ad03159 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 10:45:17 -0600
Subject: [PATCH 114/127] - Fix display bug with newest filter - Remove
 erroneous filter options - Update fiscal period query generation to match
 frontend generation exactly

---
 .../tdpservice/search_indexes/admin/filters.py   | 16 ++++++++++++++--
 .../tdpservice/search_indexes/admin/ssp.py       | 11 -----------
 .../tdpservice/search_indexes/admin/tanf.py      | 11 -----------
 .../tdpservice/search_indexes/admin/tribal.py    | 11 -----------
 4 files changed, 14 insertions(+), 35 deletions(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index ea97a6a45..6a6555423 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -72,6 +72,17 @@ def lookups(self, request, model_admin):
             ('all', _('All')),
         )
 
+    def choices(self, cl):
+        """Update query string based on selection."""
+        for lookup, title in self.lookup_choices:
+            yield {
+                'selected': self.value() == lookup,
+                'query_string': cl.get_query_string({
+                    self.parameter_name: lookup,
+                }, []),
+                'display': title,
+            }
+
     def queryset(self, request, queryset):
         """Sort queryset to show latest records."""
         if self.value() is None and queryset.exists():
@@ -113,10 +124,11 @@ class FiscalPeriodFilter(SimpleListFilter):
 
     def lookups(self, request, model_admin):
         """Available options in dropdown."""
-        current_year = datetime.date.today().year
+        today = datetime.date.today()
+        current_year =  today.year + 1 if datetime.date.today().month > 8 else today.year
         quarters = [1, 2, 3, 4]
         months = ["(Oct - Dec)", "(Jan - Mar)", "(Apr - Jun)", "(Jul - Sep)"]
-        years = [year for year in range(current_year - 3, current_year + 1)]
+        years = [year for year in range(current_year, 2020, -1)]
         options = [(None, _('All'))]
 
         for year in years:
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
index e5f1b0f89..f8b210691 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py
@@ -17,9 +17,6 @@ class SSP_M1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
-        'ZIP_CODE',
-        'STRATUM',
     ]
 
 
@@ -37,7 +34,6 @@ class SSP_M2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 
@@ -55,7 +51,6 @@ class SSP_M3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 class SSP_M4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
@@ -72,7 +67,6 @@ class SSP_M4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 class SSP_M5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
@@ -89,7 +83,6 @@ class SSP_M5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 class SSP_M6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
@@ -104,11 +97,9 @@ class SSP_M6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     ]
 
     list_filter = [
-        'CALENDAR_QUARTER',
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR'
     ]
 
 class SSP_M7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
@@ -126,9 +117,7 @@ class SSP_M7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     ]
 
     list_filter = [
-        'CALENDAR_QUARTER',
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
index b9fe84bf9..203652323 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py
@@ -17,9 +17,6 @@ class TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
-        'ZIP_CODE',
-        'STRATUM',
     ]
 
 
@@ -37,7 +34,6 @@ class TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 
@@ -55,7 +51,6 @@ class TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 
@@ -73,7 +68,6 @@ class TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 
@@ -91,7 +85,6 @@ class TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 
@@ -106,11 +99,9 @@ class TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     ]
 
     list_filter = [
-        'CALENDAR_QUARTER',
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR'
     ]
 
 
@@ -129,9 +120,7 @@ class TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     ]
 
     list_filter = [
-        'CALENDAR_QUARTER',
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
index e4ca82f06..704b07893 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/tribal.py
@@ -17,9 +17,6 @@ class Tribal_TANF_T1Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
-        'ZIP_CODE',
-        'STRATUM',
     ]
 
 
@@ -37,7 +34,6 @@ class Tribal_TANF_T2Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 
@@ -55,7 +51,6 @@ class Tribal_TANF_T3Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 class Tribal_TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
@@ -72,7 +67,6 @@ class Tribal_TANF_T4Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 class Tribal_TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     """ModelAdmin class for parsed Tribal_T5 data files."""
@@ -88,7 +82,6 @@ class Tribal_TANF_T5Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]
 
 class Tribal_TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
@@ -103,11 +96,9 @@ class Tribal_TANF_T6Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     ]
 
     list_filter = [
-        'CALENDAR_QUARTER',
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR'
     ]
 
 class Tribal_TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
@@ -125,9 +116,7 @@ class Tribal_TANF_T7Admin(ReadOnlyAdminMixin, CsvExportAdminMixin):
     ]
 
     list_filter = [
-        'CALENDAR_QUARTER',
         FiscalPeriodFilter,
         CreationDateFilter,
         STTFilter,
-        'RPT_MONTH_YEAR',
     ]

From bd56a09ad72017577ad7294f26cf32fee2f093b9 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 10:55:33 -0600
Subject: [PATCH 115/127] - fix lint

---
 tdrs-backend/tdpservice/search_indexes/admin/filters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index 6a6555423..1aed793d4 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -125,7 +125,7 @@ class FiscalPeriodFilter(SimpleListFilter):
     def lookups(self, request, model_admin):
         """Available options in dropdown."""
         today = datetime.date.today()
-        current_year =  today.year + 1 if datetime.date.today().month > 8 else today.year
+        current_year = today.year + 1 if datetime.date.today().month > 8 else today.year
         quarters = [1, 2, 3, 4]
         months = ["(Oct - Dec)", "(Jan - Mar)", "(Apr - Jun)", "(Jul - Sep)"]
         years = [year for year in range(current_year, 2020, -1)]

From 58ff9cebeef83460bdc06a4b63380cdd3736ccf7 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 10:55:54 -0600
Subject: [PATCH 116/127] - logic fix

---
 tdrs-backend/tdpservice/search_indexes/admin/filters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filters.py b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
index 1aed793d4..1777d4dab 100644
--- a/tdrs-backend/tdpservice/search_indexes/admin/filters.py
+++ b/tdrs-backend/tdpservice/search_indexes/admin/filters.py
@@ -125,7 +125,7 @@ class FiscalPeriodFilter(SimpleListFilter):
     def lookups(self, request, model_admin):
         """Available options in dropdown."""
         today = datetime.date.today()
-        current_year = today.year + 1 if datetime.date.today().month > 8 else today.year
+        current_year = today.year + 1 if today.month > 8 else today.year
         quarters = [1, 2, 3, 4]
         months = ["(Oct - Dec)", "(Jan - Mar)", "(Apr - Jun)", "(Jul - Sep)"]
         years = [year for year in range(current_year, 2020, -1)]

From 4e62b48eaa915754fda8229798b9eddf9e5d04e8 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 10:56:49 -0600
Subject: [PATCH 117/127] - testing develop scan

---
 .circleci/build-and-test/workflows.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml
index 1ecb5e9ae..de60699fd 100644
--- a/.circleci/build-and-test/workflows.yml
+++ b/.circleci/build-and-test/workflows.yml
@@ -5,7 +5,7 @@
       - secrets-check
       # TODO: Revert back to tests after review
       - nightly-owasp-scan:
-          target_env: staging
+          target_env: develop
 
   ci-build-and-test-all:
     jobs:

From 9d47eb690cdc4f85bfba50d42bebc2f800acc846 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 11:17:54 -0600
Subject: [PATCH 118/127] - recomment exit code

---
 scripts/zap-scanner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh
index 145787ca8..ce351cb00 100755
--- a/scripts/zap-scanner.sh
+++ b/scripts/zap-scanner.sh
@@ -46,7 +46,7 @@ fi
 # Ensure the APP_URL is reachable from the zaproxy container
 if ! docker-compose run --rm zaproxy curl -Is "$APP_URL" > /dev/null 2>&1; then
   echo "Target application at $APP_URL is unreachable by ZAP scanner"
-  # exit 3
+  exit 3
 fi
 
 echo "================== OWASP ZAP tests =================="

From 351b7065f1f72bf869c5d862133fe1cdd7033b8f Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 12:18:15 -0600
Subject: [PATCH 119/127] - move everything back to yaml

---
 .circleci/owasp/jobs.yml | 21 ++++++++++++++++++++-
 scripts/cf-runtask.sh    | 22 ----------------------
 2 files changed, 20 insertions(+), 23 deletions(-)
 delete mode 100755 scripts/cf-runtask.sh

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 351911be8..94c76a24b 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -84,4 +84,23 @@
           command: |
             # Construct the project slug from the current branch name and user
             PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-            ./scripts/cf-runtask.sh "$PROJECT_SLUG" << parameters.target_env >> "$CIRCLE_BUILD_NUM" "${ZAP_BACKEND_PASS_COUNT:-0}" "${ZAP_BACKEND_WARN_COUNT:-0}" "${ZAP_BACKEND_FAIL_COUNT:-0}" "${ZAP_FRONTEND_PASS_COUNT:-0}" "${ZAP_FRONTEND_WARN_COUNT:-0}" "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+            # These environment variables are exported to Circle CI's BASH_ENV
+            # by the zap-scanner.sh script for each respective app target.
+            CMD_ARGS=(
+              "$CIRCLE_BUILD_NUM"
+              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
+              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
+              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
+              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
+              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
+              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+              --project-slug "$PROJECT_SLUG"
+            )
+            # Evaluate the full command before passing it in so it doesn't
+            # get improperly interpolated by Cloud.gov.
+            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+            # Submit a CF Task for execution that will run the necessary command
+            sleep 300
+            cf run-task tdp-backend-<< parameters.target_env >> \
+              --command "$CMD" \
+              --name nightly-owasp-scan
diff --git a/scripts/cf-runtask.sh b/scripts/cf-runtask.sh
deleted file mode 100755
index 1cec24d9c..000000000
--- a/scripts/cf-runtask.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# pipefail is needed to correctly carry over the exit code from zap-full-scan.py
-set -uo pipefail
-
-PROJECT_SLUG=$1
-TARGET=$2
-BUILD_NO=$3
-ZAP_BACKEND_PASS_COUNT=$4
-ZAP_BACKEND_WARN_COUNT=$5
-ZAP_BACKEND_FAIL_COUNT=$6
-ZAP_FRONTEND_PASS_COUNT=$7
-ZAP_FRONTEND_WARN_COUNT=$8
-ZAP_FRONTEND_FAIL_COUNT=$9
-
-CMD="python manage.py process_owasp_scan $CIRCLE_BUILD_NUM --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} --project-slug $PROJECT_SLUG"
-echo .
-sleep 300
-echo .
-echo Executing command: $CMD
-APP=tdp-backend-$TARGET
-cf run-task "$APP" --wait --command "$CMD" --name "nightly-owasp-scan"

From b8ddd9ace866a7c0e065167609c4de2c8cbebda5 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 12:49:06 -0600
Subject: [PATCH 120/127] - testing different sleep time

---
 .circleci/owasp/jobs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 94c76a24b..d623e47a2 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -100,7 +100,7 @@
             # get improperly interpolated by Cloud.gov.
             CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
             # Submit a CF Task for execution that will run the necessary command
-            sleep 300
+            sleep 180
             cf run-task tdp-backend-<< parameters.target_env >> \
               --command "$CMD" \
               --name nightly-owasp-scan

From 8bd5e2e093b773b9001747a0d602c37734cf8998 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 13:09:01 -0600
Subject: [PATCH 121/127] - 4 minutes

---
 .circleci/owasp/jobs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index d623e47a2..6b6446561 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -100,7 +100,7 @@
             # get improperly interpolated by Cloud.gov.
             CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
             # Submit a CF Task for execution that will run the necessary command
-            sleep 180
+            sleep 240
             cf run-task tdp-backend-<< parameters.target_env >> \
               --command "$CMD" \
               --name nightly-owasp-scan

From 23601bc08d8de66c531418f3d1b998f6d8c6c3e9 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Mon, 15 Apr 2024 13:29:33 -0600
Subject: [PATCH 122/127] - comment update

---
 .circleci/owasp/jobs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml
index 6b6446561..225758ef5 100644
--- a/.circleci/owasp/jobs.yml
+++ b/.circleci/owasp/jobs.yml
@@ -99,7 +99,7 @@
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
             CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
-            # Submit a CF Task for execution that will run the necessary command
+            # Submit a CF Task for execution after a 4 minute sleep to ensure all of the scan's previous state has been closed.
             sleep 240
             cf run-task tdp-backend-<< parameters.target_env >> \
               --command "$CMD" \

From 3f33f4637dfaf2dae48b4124b32067c35f3b615c Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 17 Apr 2024 09:13:16 -0600
Subject: [PATCH 123/127] - Revert workflow back to OG

---
 .circleci/build-and-test/workflows.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml
index de60699fd..b822f1cdc 100644
--- a/.circleci/build-and-test/workflows.yml
+++ b/.circleci/build-and-test/workflows.yml
@@ -3,9 +3,15 @@
     when: << pipeline.parameters.build_and_test_all >>
     jobs:
       - secrets-check
-      # TODO: Revert back to tests after review
-      - nightly-owasp-scan:
-          target_env: develop
+      - test-backend:
+          requires:
+            - secrets-check
+      - test-frontend:
+          requires:
+            - secrets-check
+      - test-e2e:
+          requires:
+            - secrets-check
 
   ci-build-and-test-all:
     jobs:

From 9e3b703de41b236179ad8195ed956432f9b30d5d Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Wed, 17 Apr 2024 12:44:47 -0600
Subject: [PATCH 124/127] - removing old artifacts

---
 .circleci/build-and-test/jobs.yml      | 4 ++--
 .circleci/util/commands.yml            | 6 ------
 docs/Technical-Documentation/clamav.md | 2 +-
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml
index f9b602adf..a40d1568f 100644
--- a/.circleci/build-and-test/jobs.yml
+++ b/.circleci/build-and-test/jobs.yml
@@ -3,7 +3,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
           command: |
@@ -46,7 +46,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-backend
       - docker-compose-up-frontend
       - install-nodejs-machine
       - disable-npm-audit
diff --git a/.circleci/util/commands.yml b/.circleci/util/commands.yml
index 09d175b69..ebbdfb7e1 100644
--- a/.circleci/util/commands.yml
+++ b/.circleci/util/commands.yml
@@ -11,12 +11,6 @@
           name: Build and spin-up Django API service
           command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
 
-  docker-compose-up-with-elastic-backend:
-    steps:
-      - run:
-          name: Build and spin-up Django API service
-          command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build
-
   cf-check:
     steps:
       - run:
diff --git a/docs/Technical-Documentation/clamav.md b/docs/Technical-Documentation/clamav.md
index a8c37aaa8..894853348 100644
--- a/docs/Technical-Documentation/clamav.md
+++ b/docs/Technical-Documentation/clamav.md
@@ -23,7 +23,7 @@ The instance name then will be set as an environment variable to redirect each i
 First, set the environment variable __AV_SCAN_URL__ as follows:
 ```
 Environment variable name: AV_SCAN_URL
-Environment variable value: http://{nginx_instance}.apps.internal:9000/scan/
+Environment variable value: http://{nginx_instance}.apps.internal:9000/scan
 ```
 
 ### Add network policy from _{tdp-clamav-nginx}_ to clamav in prod

From a0c0b196224f57ae6b94107e88765bb046c7f729 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 18 Apr 2024 09:25:27 -0600
Subject: [PATCH 125/127] - recomment dev var

---
 tdrs-frontend/.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tdrs-frontend/.env b/tdrs-frontend/.env
index 0a6401a63..11902ac81 100644
--- a/tdrs-frontend/.env
+++ b/tdrs-frontend/.env
@@ -37,7 +37,7 @@ REACT_APP_DEBOUNCE_TIME=30000
 REACT_APP_EVENT_THROTTLE_TIME=60000
 
 # Enable the Kibana tab for dev purposes.
-REACT_APP_DEV_KIBANA=true
+# REACT_APP_DEV_KIBANA=true
 
 # Setup SCSS:
 # The following makes it possible to import SASS modules

From 4f725d9857a2b6e82055df173a09d801184d1421 Mon Sep 17 00:00:00 2001
From: Eric Lipe <elipe@teamraft.com>
Date: Thu, 18 Apr 2024 10:35:27 -0600
Subject: [PATCH 126/127] - added docs for dynamic locations

---
 tdrs-frontend/nginx/README.md | 40 +++++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/tdrs-frontend/nginx/README.md b/tdrs-frontend/nginx/README.md
index e6400eb37..d5beefc6b 100644
--- a/tdrs-frontend/nginx/README.md
+++ b/tdrs-frontend/nginx/README.md
@@ -2,7 +2,7 @@
 
 ## 1. Installation
 
-The Nginx uses a master process and mutiple workers to efficiently manage requests. The master process reads and evaluates config file and manages the workers. 
+The Nginx uses a master process and mutiple workers to efficiently manage requests. The master process reads and evaluates config file and manages the workers.
 
 In TDP, all config and associated files with it are placed in ```/tdrs-frontend/nginx``` folder. There are various differences between the local and deployed versions. Due to the fact that cloud.gov uses buildpacks and local version uses containerized docker instance, there are two separate directories for local and deployed versions.
 
@@ -35,7 +35,35 @@ location ^~ /v1/ {
 
 The Nginx configuration files are located in: ```*'/frontend/nginx/'```*
 
-## Backend
+### Dynamic locations
+In cloud environments, DNS information for applications is in a state of constant flux. To avoid having to restart Nginx each time an app's DNS information changes, Nginx provides features to resolve a host after a TTL has expired. The http block and location block below demonstrate the config updates to support dynamic locations. Note the `...` in the blocks below represents other config options in the blocks that we don't need to worry about.
+
+```
+http {
+    include mime.types;
+
+    resolver 127.0.0.11 ipv6=off valid=5s;
+    ...
+}
+```
+
+```
+location ~* ^/kibana/(.*)$ {
+    auth_request /kibana_auth_check;
+    auth_request_set $auth_status $upstream_status;
+
+    set $kibana http://${KIBANA}:5601/;
+    proxy_pass $kibana$1$is_args$args;
+    ...
+}
+```
+
+In the `http` block we need to include the `resolver` directive. This tells Nginx where our desired nameserver is and any other options we want to configure, such as the TTL option.
+The configuration above indicates the nameserver is at `127.0.0.11`, ipv6 is disabled, and the TTL for a hostname is five seconds. However, for Nginx to re-resolve a host
+we have to update the location, i.e. the `proxy_pass` in the location to use variable resolution. When Nginx resolves the variable given to the `proxy_pass` directive it will also determine if the TTL has expired for the host that the variable resolves to and will then resolve the DNS info if it has expired. If `proxy_pass` is not given a variable, Nginx will never resolve the host given to the `proxy_pass` directive no matter how short the TTL option is.
+
+
+## 3. Backend
 
 ### Nginx
 
@@ -43,7 +71,7 @@ The frontend then sends processing requests to the *backend* Django server, whic
 
 ### Gunicorn
 
-Gunicorn is WSGI HTTP server based on python and uses **worker [Worker Processes](https://docs.gunicorn.org/en/stable/design.html#choosing-a-worker-type)**. The number of workers is relative to server request load. 
+Gunicorn is WSGI HTTP server based on python and uses **worker [Worker Processes](https://docs.gunicorn.org/en/stable/design.html#choosing-a-worker-type)**. The number of workers is relative to server request load.
 
 With having workers responding to requests in Gunicorn, it is importnat to have Nginx in front to handle requests first, otherwise DDOS attacks would consume the server (See Nginx config).
 
@@ -55,7 +83,7 @@ workers = 2
 
 There are two config files: one for development and one for deployed applications. Gunicorn is started from ```gunicorn_start.sh``` which starts either version of config files based on environment.
 
-## Security
+## 4. Security
 
 ### Whitelist IPs
 A list of IP addresses has been added to ```ip_whitelist.conf```. This means any request from an ip address not in the subnets included in this file will be rejected. This list is created manually and needs to be maintained to whitelist and include user IP subnets.
@@ -63,9 +91,9 @@ A list of IP addresses has been added to ```ip_whitelist.conf```. This means any
 ### Security Headers
 All security headers are following best practices from [Mozilla](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers) and [OWASP](https://owasp.org/www-project-secure-headers/) and are added with comments on the config files.
 
-### CORS 
+### CORS
 
-Cross-Origin Resource Sharing (CORS) header allows a server to indicate any origin such as domain or port other than its own from which a browser can load resources. By adding HTTP headers that let server know which origins are permitted to read that information from the web browser. It should be noted that request might be passed without implications on CORS, this includes most form requests.  
+Cross-Origin Resource Sharing (CORS) header allows a server to indicate any origin such as domain or port other than its own from which a browser can load resources. By adding HTTP headers that let server know which origins are permitted to read that information from the web browser. It should be noted that request might be passed without implications on CORS, this includes most form requests.
 
 Since the frontend has to send requests to the backend server, the security headers are being set when serving the frontend requests as well as when Nginx acts as proxy server for the backend API. This is to ensure the sercurity headers cannot be tampered with and are always set to the correct values.
 

From c588373977fa161493e5a35ba7441ceb774a3c07 Mon Sep 17 00:00:00 2001
From: robgendron <163159602+robgendron@users.noreply.github.com>
Date: Tue, 30 Apr 2024 11:24:33 -0400
Subject: [PATCH 127/127] Create sprint-96-summary.md (#2958)

---
 docs/Sprint-Review/sprint-96-summary.md | 85 +++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 docs/Sprint-Review/sprint-96-summary.md

diff --git a/docs/Sprint-Review/sprint-96-summary.md b/docs/Sprint-Review/sprint-96-summary.md
new file mode 100644
index 000000000..aff7fabcf
--- /dev/null
+++ b/docs/Sprint-Review/sprint-96-summary.md
@@ -0,0 +1,85 @@
+# Sprint 96 Summary
+
+03-27-2024 - 04-09-2024
+
+
+## Sprint Goal
+* Dev:
+*Support closing out (14) tickets in review & continue progress on Cat4, and walk on more parsing enhancement tickets*
+    * Category 4 Validators #2842
+    * Further validation enhancements #2757, #2807, #2818, ... 
+
+* DevOps:
+*Successful deployments across environments and pipeline stability investments*
+    * ES re-indexing automation #2870
+    * OWASP scan fix #2768 
+
+* Design: 
+*QASP review Tribal friendly names and continue on to TANF & SSP, Continue KC work*
+    * Error Categories GitHub Documentation
+    * Friendly name fixes (#2801) 
+    * Knowledge Center Content (#2847, #2846)
+        * Submission History
+        * Error reports / data file structure
+   
+
+
+## Tickets
+### Completed/Merged
+* [#1441 As OFA tech lead, I need a new permissions group for OFA data analysts](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1441)
+* [#2845 [Design Deliverable] GitHub Error Categories Guide](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2845)
+* [#2871 As tech lead I need file transfer bug resolved](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2871)
+* [#2877 Move ES AWS routing docker image to RAFT hub](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2877)
+* [#2886 [bug] SSP feedback reports and files not downloadable from submission history tab](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2886)
+* [#2887 as tech lead, I need SSP Section 2 item # 18A (REC_OASDI_INSURANCE) schema def updated as not a required field](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2887) 
+* [#2673 Cat 1 Errors Audit Fixes](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2673)
+
+
+### Ready to Merge
+
+
+
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2908 As a `DIGIT Team` user, I need the ability to export parsed data from DAC to csv#2908](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2908)
+* [#2846 [Design Deliverable] Submission History Knowledge Center Explainer#2846](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2846)
+* [#2801 Friendly name cleanup](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2801)
+* [#2757 Generate preparser errors when multi-record rows are the wrong length or are missing space-filled second records](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/27570)
+
+
+
+
+
+### Closed (not merged)
+* [#2840 DIGIT Kibana Access](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2840)
+
+
+---
+
+## Moved to Next Sprint (In Progress, Blocked, Raft Review)
+### In Progress
+*	[#2509 As a data analyst I need to know when my data has been processed with or w/o errors](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2509)
+*	[#2847 Report Knowledge Center Explainer](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2847)
+* [2870 Spike: As tech lead, I need elastic re-indexing to be automated](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2870)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+*	[#2796 Add `STT_CODE` (or location information) to tanf/tribal t1-t7 and m1-m7 records in the django admin](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2796)
+*	[#2797 [Knowledge Center a11y] The purpose of a link must be descriptive](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2798)
+*	[#2814 Aggregate Cloud.gov ES instances](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2814)
+*	[#2818 [As tech lead, I need TDP to reject files that do not have an update indicator == DD]](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2818)
+*	[#2688 TANF Section 2 validation clean-up](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2688)
+* [#2749 As tech lead, I need validation checks to be consistent with FTANF validation checks](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2749)
+* [#2884 Generate preparser errors when multi-record rows are the wrong length or are missing space-filled second records - M3 and Tribal T3](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2884)
+
+### Blocked
+* [#2883 Pre-Made Reporting Dashboards on Kibana](https://github.com/raft-tech/TANF-app/issues/2883)
+* [#2870 Spike: As tech lead, I need elastic re-indexing to be automated](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2870)
+
+### Raft Review
+* [#1349 Terraform: Automate Deployment of Elasticsearch#1349](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1349)
+* [#2839 Spike - Cloud.gov Kibana Reschedule Route](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2839)
+* [#2826 As tech lead, I need some record types that currently require trailing spaces to be parsed](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2826)
+* [#2846 [Design Deliverable] Submission History Knowledge Center Explainer](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2846)