From 210b75c35ffada4479126f3239c0869aada87668 Mon Sep 17 00:00:00 2001 From: Andrew Jameson Date: Mon, 3 Apr 2023 10:55:10 -0400 Subject: [PATCH 001/120] saving state real quick --- .../migrations/0002_datafilesummary.py | 24 ++++++++ tdrs-backend/tdpservice/parsers/models.py | 54 ++++++++++++++++++ .../tdpservice/parsers/test/factories.py | 56 ++++++++++++++++++- .../tdpservice/parsers/test/test_summary.py | 22 ++++++++ 4 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 tdrs-backend/tdpservice/parsers/migrations/0002_datafilesummary.py create mode 100644 tdrs-backend/tdpservice/parsers/test/test_summary.py diff --git a/tdrs-backend/tdpservice/parsers/migrations/0002_datafilesummary.py b/tdrs-backend/tdpservice/parsers/migrations/0002_datafilesummary.py new file mode 100644 index 000000000..4f401770f --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/migrations/0002_datafilesummary.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.15 on 2023-03-22 18:04 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_files', '0012_datafile_s3_versioning_id'), + ('parsers', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='DataFileSummary', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('Pending', 'Pending'), ('Accepted', 'Accepted'), ('Accepted with Errors', 'Accepted With Errors'), ('Rejected', 'Rejected')], default='Pending', max_length=50)), + ('case_aggregates', models.JSONField(null=True)), + ('datafile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data_files.datafile')), + ], + ), + ] diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 98a722c69..f5492f3c1 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -4,6 +4,7 @@ from django.db import models from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType +from tdpservice.data_files.models import DataFile class ParserError(models.Model): """Model representing a parser error.""" @@ -54,3 +55,56 @@ def __str__(self): def _get_error_message(self): """Return the error message.""" return self.error_message + +class DataFileSummary(models.Model): + """Aggregates information about a parsed file.""" + + class Status(models.TextChoices): + """Enum for status of parsed file.""" + + PENDING = "Pending" # file has been uploaded, but not validated + ACCEPTED = "Accepted" + ACCEPTED_WITH_ERRORS = "Accepted with Errors" + REJECTED = "Rejected" + + status = models.CharField( + max_length=50, + choices=Status.choices, + default=Status.PENDING, + ) + + datafile = models.ForeignKey(DataFile, on_delete=models.CASCADE) + + # eventually needs a breakdown of cases (accepted, rejected, total) per month + # elif qtr2 jan-mar + # elif qtr3 apr-jun + # elif qtr4 jul-sept + + case_aggregates = models.JSONField(null=True, blank=False) + """ + # Do these queries only once, save result during creation of this model + # or do we grab this data during parsing and bubble it up during create call? + { + "Jan": { + "accepted": 100, + "rejected": 10, + "total": 110 + }, + "Feb": { + "accepted": 100, + "rejected": 10, + "total": 110 + }, + "Mar": { + "accepted": 100, + "rejected": 10, + "total": 110 + } + """ + + def set_status(self, datafile): + """Set and return the status field based on errors and models associated with datafile.""" + # to set rejected, we would need to have raised an exception during (pre)parsing + # else if there are errors, we can set to accepted with errors + # else we can set to accepted (default) + pass diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 48195aa38..df642157c 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -1,6 +1,60 @@ """Factories for generating test data for parsers.""" import factory +from tdpservice.search_indexes.parsers.models import DataFileSummary from tdpservice.data_files.test.factories import DataFileFactory +from tdpservice.users.test.factories import UserFactory +from tdpservice.stts.test.factories import STTFactory + +class ParsingFileFactory(factory.django.DjangoModelFactory): + """Generate test data for data files.""" + + class Meta: + """Hardcoded meta data for data files.""" + + model = "data_files.DataFile" + + original_filename = "data_file.txt" + slug = "data_file-txt-slug" + extension = "txt" + section = "Active Case Data" + quarter = "Q1" + year = "2020" + version = 1 + user = factory.SubFactory(UserFactory) + stt = factory.SubFactory(STTFactory) + file = factory.django.FileField(data=b'test', filename='my_data_file.txt') + s3_versioning_id = 0 + +class DataFileSummaryFactory(factory.django.DjangoModelFactory): + """Generate test data for data files.""" + + class Meta: + """Hardcoded meta data for data files.""" + + model = DataFileSummary + + status = DataFileSummary.Status.ACCEPTED + + case_aggregates = { + "Jan": { + "accepted": 100, + "rejected": 10, + "total": 110 + }, + "Feb": { + "accepted": 100, + "rejected": 10, + "total": 110 + }, + "Mar": { + "accepted": 100, + "rejected": 10, + "total": 110 + } + } + + datafile = factory.SubFactory(DataFileFactory) + class ParserErrorFactory(factory.django.DjangoModelFactory): """Generate test data for parser errors.""" @@ -8,7 +62,7 @@ class ParserErrorFactory(factory.django.DjangoModelFactory): class Meta: """Hardcoded meta data for parser errors.""" - model = "parsers.ParserError" + model = "search_indexes.ParserError" file = factory.SubFactory(DataFileFactory) row_number = 1 diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py new file mode 100644 index 000000000..02e9b2d3c --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -0,0 +1,22 @@ +"""Test the new model for DataFileSummary.""" + +import pytest +from tdpservice.search_indexes.parsers import tanf_parser, preparser, util +from tdpservice.search_indexes.models import T1 +from tdpservice.search_indexes.parsers.models import DataFileSummary +from .factories import DataFileSummaryFactory +from .test_data_parsing import bad_file_missing_header +import logging +logger = logging.getLogger(__name__) + +@pytest.mark.django_db +def test_data_file_summary_model(): + """Test that the model is created and populated correctly.""" + dfs = DataFileSummaryFactory() + + assert dfs.case_aggregates['Jan']['accepted'] == 100 + +def test_dfs_rejected(bad_file_missing_header): + """Ensure that an invalid file generates a rejected status.""" + dfs = preparser.preparse(bad_file_missing_header, 'TANF', 'Active Case Data') + assert dfs == DataFileSummary.Status.REJECTED \ No newline at end of file From 2c20472d472c6189c1662527456755e3e333f6c9 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 12 Apr 2023 13:55:41 -0400 Subject: [PATCH 002/120] finishing merge with latest --- tdrs-backend/tdpservice/parsers/preparser.py | 235 ------------------ .../parsers/schema_defs/universal.py | 59 ----- 2 files changed, 294 deletions(-) delete mode 100644 tdrs-backend/tdpservice/parsers/preparser.py delete mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/universal.py diff --git a/tdrs-backend/tdpservice/parsers/preparser.py b/tdrs-backend/tdpservice/parsers/preparser.py deleted file mode 100644 index d00ab5fea..000000000 --- a/tdrs-backend/tdpservice/parsers/preparser.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Converts data files into a model that can be indexed by Elasticsearch.""" - -import os -import logging -import argparse -from cerberus import Validator -from .schema_defs.universal import get_header_schema, get_trailer_schema -from .util import get_record_type -from . import tanf_parser - -from io import BufferedReader -# from .models import ParserLog -from tdpservice.data_files.models import DataFile - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.DEBUG) - - -def validate_header(line, data_type, given_section): - """Validate the header (first line) of the datafile.""" - logger.debug('Validating header line.') - section_map = { - 'A': 'Active Case Data', - 'C': 'Closed Case Data', - 'G': 'Aggregate Data', - 'S': 'Stratum Data', - } - - try: - header = { - 'title': line[0:6], - 'year': line[6:10], - 'quarter': line[10:11], - 'type': line[11:12], - 'state_fips': line[12:14], - 'tribe_code': line[14:17], - 'program_type': line[17:20], - 'edit': line[20:21], - 'encryption': line[21:22], - 'update': line[22:23], - } - - for key, value in header.items(): - logger.debug('Header key %s: "%s"' % (key, value)) - - # TODO: Will need to be saved in parserLog, #1354 - - try: - section_type = section_map[header['type']] - logger.debug("Given section: '%s'\t Header section: '%s'", given_section, section_type) - program_type = header['program_type'] - logger.debug("Given program type: '%s'\t Header program type: '%s'", data_type, header['program_type']) - - if given_section != section_map[header['type']]: - raise ValueError('Given section does not match header section.') - - if (data_type == 'TANF' and program_type != 'TAN')\ - or (data_type == 'SSP' and program_type != 'SSP')\ - or (data_type not in ['TANF', 'SSP']): - raise ValueError("Given data type does not match header program type.") - except KeyError as e: - logger.error('Ran into issue with header type: %s', e) - - validator = Validator(get_header_schema()) - is_valid = validator.validate(header) - logger.debug(validator.errors) - - return is_valid, validator - - except KeyError as e: - logger.error('Exception splicing header line or referencing it, please see line and error.') - logger.error(line) - logger.error(e) - return False, e - -def validate_trailer(line): - """Validate the trailer (last) line in a datafile.""" - logger.info('Validating trailer line.') - validator = Validator(get_trailer_schema()) - - trailer = { - 'title': line[0:7], - 'record_count': line[7:14], - 'blank': line[14:23], - } - - is_valid = validator.validate(trailer) - - logger.debug("Trailer title: '%s'", trailer['title']) - logger.debug("Trailer record count: '%s'", trailer['record_count']) - logger.debug("Trailer blank: '%s'", trailer['blank']) - logger.debug("Trailer errors: '%s'", validator.errors) - - return is_valid, validator - -def get_header_line(datafile): - """Alters header line into string.""" - # intentionally only reading first line of file - datafile.seek(0) - header = datafile.readline() - datafile.seek(0) # reset file pointer to beginning of file - - # datafile when passed via redis/db is a FileField which returns bytes - if isinstance(header, bytes): - header = header.decode() - - header = header.strip() - - if get_record_type(header) != 'HE': - return False, {'preparsing': 'First line in file is not recognized as a valid header.'} - elif len(header) != 23: - logger.debug("line: '%s' len: %d", header, len(header)) - return False, {'preparsing': 'Header length incorrect.'} - - return True, header - -def get_trailer_line(datafile): - """Alters the trailer line into usable string.""" - # certify/transform input line to be correct form/type - - # Don't want to read whole file, just last line, only possible with binary - # Credit: https://openwritings.net/pg/python/python-read-last-line-file - # https://stackoverflow.com/questions/46258499/ - try: - datafile.seek(-2, os.SEEK_END) # Jump to the second last byte. - while datafile.read(1) != b'\n': # Check if new line. - datafile.seek(-2, os.SEEK_CUR) # Jump two bytes back - except OSError: # Either file is empty or contains one line. - datafile.seek(0) - return False, {'preparsing': 'File too short or missing trailer.'} - - # Having set the file pointer to the last line, read it in. - line = datafile.readline().decode() - datafile.seek(0) # Reset file pointer to beginning of file. - - logger.info("Trailer line: '%s'", line) - if get_record_type(line) != 'TR': - raise ValueError('Last line is not recognized as a trailer line.') - elif len(line) < 13: - # Previously, we checked to ensure exact length was 24. At the business level, we - # don't care about the exact length, just that it's long enough to contain the - # trailer information (>=14) and can ignore the trailing spaces. - logger.debug("line: '%s' len: %d", line, len(line)) - return False, {'preparsing': 'Trailer length incorrect.'} - line = line.strip('\n') - - return True, line - -def check_plural_count(datafile): - """Ensure only one header and one trailer exist in file.""" - header_count = 0 - trailer_count = 0 - line_no = 0 - - for line in datafile: - line_no += 1 - if get_record_type(line) == 'HE': - header_count += 1 - if header_count > 1: - return False, {'preparsing': 'Multiple header lines found on ' + str(line_no) + '.'} - elif get_record_type(line) == 'TR': - trailer_count += 1 - if trailer_count > 1: - return False, {'preparsing': 'Multiple trailer lines found on ' + str(line_no) + '.'} - return True, None - -def preparse(data_file, data_type, section): - """Validate metadata then dispatches file to appropriate parser.""" - if isinstance(data_file, DataFile): - logger.debug("Beginning preparsing on '%s'", data_file.file.name) - datafile = data_file.file - elif isinstance(data_file, BufferedReader): - datafile = data_file - else: - logger.error("Unexpected datafile type %s", type(data_file)) - raise TypeError("Unexpected datafile type.") - - unique_header_footer, line = check_plural_count(datafile) - if unique_header_footer is False: - raise ValueError("Preparsing error: %s" % line['preparsing']) - - header_preparsed, line = get_header_line(datafile) - if header_preparsed is False: - raise ValueError("Header invalid, error: %s" % line['preparsing']) - # logger.debug("Header: %s", line) - - header_is_valid, header_validator = validate_header(line, data_type, section) - if isinstance(header_validator, Exception): - raise header_validator - - trailer_preparsed, line = get_trailer_line(datafile) - if trailer_preparsed is False: - raise ValueError("Trailer invalid, error: %s" % line['preparsing']) - trailer_is_valid, trailer_validator = validate_trailer(line) - if isinstance(trailer_validator, Exception): - raise trailer_validator - - errors = {'header': header_validator.errors, 'trailer': trailer_validator.errors} - - if header_is_valid and trailer_is_valid: - logger.info("Preparsing succeeded.") - else: - # TODO: should we end here or let parser run to collect more errors? - logger.error("Preparse failed: %s", errors) - return False, errors - # return ParserLog.objects.create( - # data_file=args.file, - # errors=errors, - # status=ParserLog.Status.REJECTED, - # ) - - logger.debug("Data type: '%s'", data_type) - if data_type == 'TANF': - logger.info("Dispatching to TANF parser.") - tanf_parser.parse(datafile) - # elif data_type == 'SSP': - # ssp_parser.parse(datafile) - # elif data_type == 'Tribal TANF': - # tribal_tanf_parser.parse(datafile) - else: - raise ValueError('Preparser given invalid data_type parameter.') - - return True - - -if __name__ == '__main__': - """Take in command-line arguments and run the parser.""" - parser = argparse.ArgumentParser(description='Parse TANF active cases data.') - parser.add_argument('--file', type=argparse.FileType('r'), help='The file to parse.') - parser.add_argument('--data_type', type=str, default='TANF', help='The type of data to parse.') - parser.add_argument('--section', type=str, default='Active Case Data', help='The section submitted.') - - args = parser.parse_args() - logger.debug("Arguments: %s", args) - preparse(args.file, data_type="TANF", section="Active Case Data") diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/universal.py b/tdrs-backend/tdpservice/parsers/schema_defs/universal.py deleted file mode 100644 index f80f47363..000000000 --- a/tdrs-backend/tdpservice/parsers/schema_defs/universal.py +++ /dev/null @@ -1,59 +0,0 @@ -"""A commonly-shared set of definitions for line schemas in datafiles.""" - -def get_header_schema(): - """Return the schema dictionary for the header record.""" - """DEFINITION SOURCE: - https://www.acf.hhs.gov/sites/default/files/documents/ofa/transmission_file_header_trailer_record.pdf - - DESCRIPTION LENGTH FROM TO COMMENT - Title 6 1 6 Value = HEADER - YYYYQ 5 7 11 Value = YYYYQ - Type 1 12 12 A=Active; C=Closed; G=Aggregate, S=Stratum - State Fips 2 13 14 "2 digit state code 000 a tribe" - Tribe Code 3 15 17 "3 digit tribe code 000 a state" - Program Type 3 18 20 Value = TAN (TANF) or Value = SSP (SSP-MOE) - Edit Indicator 1 21 21 1=Return Fatal & Warning Edits 2=Return Fatal Edits only - Encryption 1 22 22 E=SSN is encrypted Blank = SSN is not encrypted - Update 1 23 23 N = New data D = Delete existing data U - QUARTERS: - Q=1 (Jan-Mar) - Q=2 (Apr-Jun) - Q=3 (Jul-Sep) - Q=4 (Oct-Dec) - Example: - HEADERYYYYQTFIPSSP1EN - """ - header_schema = { - 'title': {'type': 'string', 'required': True, 'allowed': ['HEADER']}, - 'year': {'type': 'string', 'required': True, 'regex': '^20[0-9]{2}$'}, # 4 digits, starts with 20 - 'quarter': {'type': 'string', 'required': True, 'allowed': ['1', '2', '3', '4']}, - 'type': {'type': 'string', 'required': True, 'allowed': ['A', 'C', 'G', 'S']}, - 'state_fips': {'type': 'string', 'required': True, 'regex': '^[0-9]{2}$'}, # 2 digits - 'tribe_code': {'type': 'string', 'required': False, 'regex': '^([0-9]{3}|[ ]{3})$'}, # digits or spaces - 'program_type': {'type': 'string', 'required': True, 'allowed': ['TAN', 'SSP']}, - 'edit': {'type': 'string', 'required': True, 'allowed': ['1', '2']}, - 'encryption': {'type': 'string', 'required': True, 'allowed': ['E', ' ']}, - 'update': {'type': 'string', 'required': True, 'allowed': ['N', 'D', 'U']}, - } - # we're limited to string for all values otherwise we have to cast to int in header and it fails - # with a raised ValueError instead of in the validator.errors - return header_schema - -def get_trailer_schema(): - """Return the schema dictionary for the trailer record.""" - """DEFINITION SOURCE: - https://www.acf.hhs.gov/sites/default/files/documents/ofa/transmission_file_header_trailer_record.pdf - length of 24 - DESCRIPTION LENGTH FROM TO COMMENT - Title 7 1 7 Value = TRAILER - Record Count7 8 14 Right Adjusted - Blank 9 15 23 Value = spaces - Example: - 'TRAILER0000001 ' - """ - trailer_schema = { - 'title': {'type': 'string', 'required': True, 'allowed': ['TRAILER']}, - 'record_count': {'type': 'string', 'required': True, 'regex': '^[0-9]{7}$'}, - 'blank': {'type': 'string', 'required': True, 'regex': '^[ ]{9}$'}, - } - return trailer_schema From 9c0b25d89fc37888ddffba94a201c346f425fdd1 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 12 Apr 2023 13:57:13 -0400 Subject: [PATCH 003/120] Missed old test script --- .../parsers/test/test_data_parsing.py | 265 ------------------ 1 file changed, 265 deletions(-) delete mode 100644 tdrs-backend/tdpservice/parsers/test/test_data_parsing.py diff --git a/tdrs-backend/tdpservice/parsers/test/test_data_parsing.py b/tdrs-backend/tdpservice/parsers/test/test_data_parsing.py deleted file mode 100644 index 0d430d8ef..000000000 --- a/tdrs-backend/tdpservice/parsers/test/test_data_parsing.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Test preparser functions and tanf_parser.""" -import pytest -from functools import reduce -from pathlib import Path - -from tdpservice.parsers import tanf_parser, preparser, util -from tdpservice.search_indexes.models import T1 - -import logging -logger = logging.getLogger(__name__) - -# TODO: ORM mock for true data file factories -# https://stackoverflow.com/questions/1533861/testing-django-models-with-filefield - -@pytest.fixture -def test_file(): - """Open file pointer to test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/small_correct_file" - yield open(test_filename, 'rb') - -@pytest.fixture -def test_big_file(): - """Open file pointer to test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/ADS.E2J.FTP1.TS06" - yield open(test_filename, 'rb') - -@pytest.fixture -def bad_test_file(): - """Open file pointer to bad test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/bad_TANF_S2.txt" - yield open(test_filename, 'rb') - -@pytest.fixture -def bad_file_missing_header(): - """Open file pointer to bad test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/bad_missing_header.txt" - yield open(test_filename, 'rb') - -@pytest.fixture -def bad_file_multiple_headers(): - """Open file pointer to bad test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/bad_two_headers.txt" - yield open(test_filename, 'rb') - -@pytest.fixture -def big_bad_test_file(): - """Open file pointer to bad test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/bad_TANF_S1.txt" - yield open(test_filename, 'rb') - -@pytest.fixture -def bad_trailer_file(): - """Open file pointer to bad test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/bad_trailer_1.txt" - yield open(test_filename, 'rb') - -@pytest.fixture -def bad_trailer_file_2(): - """Open file pointer to bad test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/bad_trailer_2.txt" - yield open(test_filename, 'rb') - -@pytest.fixture -def empty_file(): - """Open file pointer to bad test file.""" - test_filepath = str(Path(__file__).parent.joinpath('data')) - test_filename = test_filepath + "/empty_file" - yield open(test_filename, 'rb') - -def test_preparser_header(test_file, bad_test_file): - """Test header preparser.""" - logger.info("test_file type: %s", type(test_file)) - test_line = test_file.readline().decode() - is_valid, validator = preparser.validate_header(test_line, 'TANF', 'Active Case Data') - - logger.info("is_valid: %s", is_valid) - logger.info("errors: %s", validator.errors) - assert is_valid is True - assert validator.errors == {} - assert validator.document['state_fips'] == '06' - - # negative case - bad_line = bad_test_file.readline().decode() - not_valid, not_validator = preparser.validate_header(bad_line, 'TANF', 'Active Case Data') - assert not_valid is False - assert not_validator.errors != {} - - # Inserting a bad section type - with pytest.raises(ValueError) as e_info: - preparser.validate_header(test_line, 'TANF', 'Active Casexs') - assert str(e_info.value) == "Given section does not match header section." - - # Inserting a bad program type - with pytest.raises(ValueError) as e_info: - preparser.validate_header(test_line, 'GARBAGE', 'Active Case Data') - assert str(e_info.value) == "Given data type does not match header program type." - -def test_preparser_trailer(test_file): - """Test trailer preparser.""" - for line in test_file: - line = line.decode() - if util.get_record_type(line) == 'TR': - trailer_line = line - break - is_valid, validator = preparser.validate_trailer(trailer_line) - assert is_valid - assert validator.errors == {} - - logger.debug("validator: %s", validator) - logger.debug("validator.document: %s", validator.document) - assert validator.document['record_count'] == '0000001' - -def test_preparser_trailer_bad(bad_trailer_file, bad_trailer_file_2, empty_file): - """Test trailer preparser with malformed trailers.""" - status, err = preparser.get_trailer_line(empty_file) - assert status is False - assert err['preparsing'] == 'File too short or missing trailer.' - - status, err = preparser.get_trailer_line(bad_trailer_file) - assert status is False - assert err['preparsing'] == 'Trailer length incorrect.' - - with pytest.raises(ValueError) as e_info: - logger.debug(preparser.get_trailer_line(bad_trailer_file_2)) - assert str(e_info.value) == 'Last line is not recognized as a trailer line.' - -def spy_count_check(spies, expected_counts): - """Run reduce against two lists, returning True if all functions were called the expected number of times.""" - return reduce(lambda carry, expected: carry == all(expected), - [zip([spy.call_count for spy in spies], expected_counts)], - True) - # logger.debug("reduceVal: %s", reduceVal) - # for spy, expected in zip(spies, expected_counts): - # logger.debug("%s: spy called %s times\texpected\t%s", spy, spy.call_count, expected) - # assert spy.call_count == expected - -@pytest.mark.django_db -def test_preparser_body(test_file, mocker): - """Test that preparse correctly calls lower parser functions...or doesn't.""" - spy_preparse = mocker.spy(preparser, 'preparse') - spy_head = mocker.spy(preparser, 'validate_header') - spy_tail = mocker.spy(preparser, 'validate_trailer') - spy_parse = mocker.spy(tanf_parser, 'parse') - spy_t1 = mocker.spy(tanf_parser, 'active_t1_parser') - - spies = [spy_preparse, spy_head, spy_tail, spy_parse, spy_t1] - assert preparser.preparse(test_file, 'TANF', 'Active Case Data') - - assert spy_count_check(spies, [1, 1, 1, 1, 1]) - -@pytest.mark.django_db -def test_preparser_big_file(test_big_file, mocker): - """Test the preparse correctly handles a large, correct file.""" - spy_preparse = mocker.spy(preparser, 'preparse') - spy_head = mocker.spy(preparser, 'validate_header') - spy_tail = mocker.spy(preparser, 'validate_trailer') - spy_parse = mocker.spy(tanf_parser, 'parse') - spy_t1 = mocker.spy(tanf_parser, 'active_t1_parser') - - spies = [spy_preparse, spy_head, spy_tail, spy_parse, spy_t1] - assert preparser.preparse(test_big_file, 'TANF', 'Active Case Data') - - assert spy_count_check(spies, [1, 1, 1, 1, 815]) - -@pytest.mark.django_db -def test_preparser_bad_file(bad_test_file, bad_file_missing_header, bad_file_multiple_headers, mocker): - """Test that preparse correctly catches issues in a bad file.""" - spy_preparse = mocker.spy(preparser, 'preparse') - spy_head = mocker.spy(preparser, 'validate_header') - spy_tail = mocker.spy(preparser, 'validate_trailer') - spy_parse = mocker.spy(tanf_parser, 'parse') - spy_t1 = mocker.spy(tanf_parser, 'active_t1_parser') - - spies = [spy_preparse, spy_head, spy_tail, spy_parse, spy_t1] - with pytest.raises(ValueError) as e_info: - is_valid, preparser_errors = preparser.preparse(bad_test_file, 'TANF', 'Active Case Data') - assert str(e_info.value) == 'Header invalid, error: Header length incorrect.' - - assert spy_count_check(spies, [1, 0, 0, 0, 0]) - - with pytest.raises(ValueError) as e_info: - preparser.preparse(bad_file_missing_header, 'TANF', 'Active Case Data') - assert str(e_info.value) == 'Header invalid, error: First line in file is not recognized as a valid header.' - - with pytest.raises(ValueError) as e_info: - preparser.preparse(bad_file_multiple_headers, 'TANF', 'Active Case Data') - assert str(e_info.value).startswith('Preparsing error: Multiple header lines found') - -@pytest.mark.django_db -def test_preparser_bad_params(test_file, mocker): - """Test that preparse correctly catches bad parameters.""" - spy_preparse = mocker.spy(preparser, 'preparse') - spy_head = mocker.spy(preparser, 'validate_header') - spy_tail = mocker.spy(preparser, 'validate_trailer') - spy_parse = mocker.spy(tanf_parser, 'parse') - spy_t1 = mocker.spy(tanf_parser, 'active_t1_parser') - - spies = [spy_preparse, spy_head, spy_tail, spy_parse, spy_t1] - - with pytest.raises(ValueError) as e_info: - preparser.preparse(test_file, 'TANF', 'Garbage Cases') - assert str(e_info.value) == 'Given section does not match header section.' - logger.debug("test_preparser_bad_params::garbage section value:") - for spy in spies: - logger.debug("Spy: %s\tCount: %s", spy, spy.call_count) - assert spy_count_check(spies, [1, 1, 0, 0, 0]) - - with pytest.raises(ValueError) as e_info: - preparser.preparse(test_file, 'GARBAGE', 'Active Case Data') - assert str(e_info.value) == "Given data type does not match header program type." - logger.debug("test_preparser_bad_params::wrong program_type value:") - for spy in spies: - logger.debug("Spy: %s\tCount: %s", spy, spy.call_count) - assert spy_count_check(spies, [2, 2, 0, 0, 0]) - - with pytest.raises(ValueError) as e_info: - preparser.preparse(test_file, 1234, 'Active Case Data') - assert str(e_info.value) == 'Given data type does not match header program type.' - logger.debug("test_preparser_bad_params::wrong program_type type:") - for spy in spies: - logger.debug("Spy: %s\tCount: %s", spy, spy.call_count) - assert spy_count_check(spies, [3, 3, 0, 0, 0]) - - -@pytest.mark.django_db -def test_parsing_tanf_t1_active(test_file): - """Test tanf_parser.active_t1_parser.""" - t1_count_before = T1.objects.count() - assert t1_count_before == 0 - tanf_parser.parse(test_file) - t1_count_after = T1.objects.count() - assert t1_count_after == (t1_count_before + 1) - - # define expected values - # we get back a parser log object for 1354 - # should we create a FK between parserlog and t1 model? - -@pytest.mark.django_db -def test_parsing_tanf_t1_bad(bad_test_file, big_bad_test_file): - """Test tanf_parser.active_case_data with bad data.""" - t1_count_before = T1.objects.count() - logger.info("t1_count_before: %s", t1_count_before) - - tanf_parser.parse(bad_test_file) - - t1_count_after = T1.objects.count() - logger.info("t1_count_after: %s", t1_count_after) - assert t1_count_after == t1_count_before - - t1_count_before = T1.objects.count() - logger.info("t1_count_before: %s", t1_count_before) - - tanf_parser.parse(big_bad_test_file) - - t1_count_after = T1.objects.count() - logger.info("t1_count_after: %s", t1_count_after) - assert t1_count_after == t1_count_before From fba7f437afb9b428bbe33fd9b1a83be1cff5699d Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 18 Apr 2023 09:53:50 -0400 Subject: [PATCH 004/120] Added new test, more cleanup --- tdrs-backend/tdpservice/parsers/models.py | 13 ++- tdrs-backend/tdpservice/parsers/parse.py | 7 +- .../tdpservice/parsers/tanf_parser.py | 97 ------------------- .../tdpservice/parsers/test/factories.py | 3 +- .../tdpservice/parsers/test/test_summary.py | 28 ++++-- tdrs-backend/tdpservice/parsers/util.py | 4 +- .../tdpservice/scheduling/parser_task.py | 4 +- 7 files changed, 42 insertions(+), 114 deletions(-) delete mode 100644 tdrs-backend/tdpservice/parsers/tanf_parser.py diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index f5492f3c1..ce5ad5a99 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -102,9 +102,16 @@ class Status(models.TextChoices): } """ - def set_status(self, datafile): + def set_status(self, errors): """Set and return the status field based on errors and models associated with datafile.""" # to set rejected, we would need to have raised an exception during (pre)parsing - # else if there are errors, we can set to accepted with errors + # else if there are errors, we can set to accepted with errors # else we can set to accepted (default) - pass + if errors == {}: # This feels better than running len() on `errors` + self.status = self.Status.ACCEPTED + elif errors: + self.status = self.Status.ACCEPTED_WITH_ERRORS + elif errors == {'document': ['No headers found.']}: # we need a signal for unacceptable errors. Another func? + self.status = self.Status.REJECTED + + return self.status diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index a1d58ca2d..812047e95 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -5,6 +5,7 @@ from . import schema_defs from . import validators from tdpservice.data_files.models import DataFile +from .models import DataFileSummary def parse_datafile(datafile): @@ -80,7 +81,11 @@ def parse_datafile(datafile): if not record_is_valid: errors[line_number] = record_errors - return errors + summary = DataFileSummary(datafile=datafile) + summary.set_status(errors) + summary.save() + + return summary, errors def parse_datafile_line(line, schema): diff --git a/tdrs-backend/tdpservice/parsers/tanf_parser.py b/tdrs-backend/tdpservice/parsers/tanf_parser.py deleted file mode 100644 index 27d498c05..000000000 --- a/tdrs-backend/tdpservice/parsers/tanf_parser.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Transforms a TANF datafile into an search_index model.""" - -import logging -from tdpservice.search_indexes.models import T1 # , T2, T3, T4, T5, T6, T7, ParserLog -# from django.core.exceptions import ValidationError -from .util import get_record_type -from .schema_defs.tanf import t1_schema - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -def active_t1_parser(line, line_number): - """Parse line in datafile as active case data, T1 only.""" - family_case_schema = t1_schema() - # create search_index model - t1 = T1() - content_is_valid = True - - min_line_length = 118 # we will need to adjust for other types - actual_line_length = len(line) - if actual_line_length < min_line_length: - logger.error('Expected minimum line length of %s, got: %s', min_line_length, actual_line_length) - return - - for field in family_case_schema.get_all_fields(): - if field.name == 'blank': - # We are discarding this data. - break - content = line[field.start-1:field.end] # descriptor pdfs were off by one, could also adjust start values - - # check if content is type string or integer - if field.type == 'Numeric': - try: - content = int(content) - except ValueError: - logger.warn('[LineNo:%d, col%d] Expected field "%s" to be numeric, got: "%s"', - line_number, field.start-1, field.name, content) - content_is_valid = False - continue - elif field.type == 'Alphanumeric': - pass # maybe we can regex check some of these later - # The below is extremely spammy, turn on selectively. - # logger.debug('field: %s\t::content: "%s"\t::end: %s', field.name, content, field.end) - - if content_is_valid: - setattr(t1, field.name, content) - - if content_is_valid is False: - logger.warn('Content is not valid, skipping model creation.') - return - - # try: - # t1.full_clean() - t1.save() - ''' - # holdovers for 1354 - ParserLog.objects.create( - data_file=datafile, - status=ParserLog.Status.ACCEPTED, - ) - - except ValidationError as e: - return ParserLog.objects.create( - data_file=datafile, - status=ParserLog.Status.ACCEPTED_WITH_ERRORS, - errors=e.message - ) - ''' - -# TODO: def closed_case_data(datafile): - -# TODO: def aggregate_data(datafile): - -# TODO: def stratum_data(datafile): - -def parse(datafile): - """Parse the datafile into the search_index model.""" - logger.info('Parsing TANF datafile: %s', datafile) - - datafile.seek(0) # ensure we are at the beginning of the file - line_number = 0 - for raw_line in datafile: - line_number += 1 - if isinstance(raw_line, bytes): - raw_line = raw_line.decode() - line = raw_line.strip('\r\n') - - record_type = get_record_type(line) - - if record_type == 'HE' or record_type == 'TR': - # Header/trailers do not differ between types, this is part of preparsing. - continue - elif record_type == 'T1': - active_t1_parser(line, line_number) - else: - logger.warn("Parsing for type %s not yet implemented", record_type) - continue diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 78e46fdda..6b387da2b 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -1,7 +1,7 @@ """Factories for generating test data for parsers.""" import factory -from tdpservice.search_indexes.parsers.models import DataFileSummary +from tdpservice.parsers.models import DataFileSummary from tdpservice.data_files.test.factories import DataFileFactory from tdpservice.users.test.factories import UserFactory from tdpservice.stts.test.factories import STTFactory @@ -62,6 +62,7 @@ class ParserErrorFactory(factory.django.DjangoModelFactory): class Meta: """Hardcoded meta data for parser errors.""" + model = "parsers.ParserError" file = factory.SubFactory(DataFileFactory) diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 02e9b2d3c..489b35f15 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -1,22 +1,36 @@ """Test the new model for DataFileSummary.""" import pytest -from tdpservice.search_indexes.parsers import tanf_parser, preparser, util -from tdpservice.search_indexes.models import T1 -from tdpservice.search_indexes.parsers.models import DataFileSummary +from tdpservice.parsers import parse +from tdpservice.parsers.models import DataFileSummary from .factories import DataFileSummaryFactory -from .test_data_parsing import bad_file_missing_header +from .test_parse import bad_file_missing_header + import logging logger = logging.getLogger(__name__) @pytest.mark.django_db -def test_data_file_summary_model(): +def test_dfs_model(): """Test that the model is created and populated correctly.""" dfs = DataFileSummaryFactory() assert dfs.case_aggregates['Jan']['accepted'] == 100 +@pytest.mark.django_db def test_dfs_rejected(bad_file_missing_header): """Ensure that an invalid file generates a rejected status.""" - dfs = preparser.preparse(bad_file_missing_header, 'TANF', 'Active Case Data') - assert dfs == DataFileSummary.Status.REJECTED \ No newline at end of file + dfs = DataFileSummaryFactory() + dfs.set_status(parse.parse_datafile(bad_file_missing_header)) + assert dfs.status == DataFileSummary.Status.REJECTED + +@pytest.mark.django_db +def test_dfs_set_status(): + """Test that the status is set correctly.""" + dfs = DataFileSummaryFactory() + assert dfs.status == DataFileSummary.Status.PENDING + dfs.set_status(errors={}) + assert dfs.status == DataFileSummary.Status.ACCEPTED + dfs.set_status(errors=[1, 2, 3]) + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + dfs.set_status(errors={'document': ['No headers found.']}) + assert dfs.status == DataFileSummary.Status.REJECTED diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 912b4852d..7a1bcc2f6 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -3,10 +3,9 @@ logger = logging.getLogger(__name__) - - class Field: """Provides a mapping between a field name and its position.""" + def __init__(self, name, type, startIndex, endIndex, required=True, validators=[]): self.name = name self.type = type @@ -59,7 +58,6 @@ def get_all_fields(self): """Get all fields from the schema.""" return self.fields - def parse_and_validate(self, line): """Run all validation steps in order, and parse the given line into a record.""" errors = [] diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index 649ae0723..b1133d1f1 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -17,5 +17,5 @@ def parse(data_file_id): data_file = DataFile.objects.get(id=data_file_id) logger.info(f"DataFile parsing started for file {data_file.filename}") - errors = parse_datafile(data_file) - logger.info(f"DataFile parsing finished with {len(errors)} errors: {errors}") + status, errors = parse_datafile(data_file) + logger.info(f"DataFile parsing finished with status {status} and {len(errors)} errors: {errors}") From b948a5b8fa078009c41082b6e26eb7849e873ebe Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 26 Apr 2023 14:29:49 -0400 Subject: [PATCH 005/120] Updating unit tests in DFS, preparing for 1610 --- tdrs-backend/tdpservice/parsers/models.py | 14 +++++-- .../tdpservice/parsers/test/factories.py | 2 +- .../tdpservice/parsers/test/test_summary.py | 38 ++++++++++++++----- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index ce5ad5a99..a71eea022 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -107,11 +107,19 @@ def set_status(self, errors): # to set rejected, we would need to have raised an exception during (pre)parsing # else if there are errors, we can set to accepted with errors # else we can set to accepted (default) - if errors == {}: # This feels better than running len() on `errors` + if errors == {}: # This feels better than running len() on `errors`...but is it a dict vs list? self.status = self.Status.ACCEPTED + elif check_for_preparsing(errors): + self.status = self.Status.REJECTED elif errors: self.status = self.Status.ACCEPTED_WITH_ERRORS - elif errors == {'document': ['No headers found.']}: # we need a signal for unacceptable errors. Another func? - self.status = self.Status.REJECTED return self.status + +def check_for_preparsing(errors): + """Check for pre-parsing errors.""" + for error in errors['document']: + print(error) + if error.error_type == "pre-parsing": + return True + return False \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 6b387da2b..9858a5eed 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -34,7 +34,7 @@ class Meta: model = DataFileSummary - status = DataFileSummary.Status.ACCEPTED + status = DataFileSummary.Status.PENDING case_aggregates = { "Jan": { diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 489b35f15..dd0619bd1 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -3,24 +3,31 @@ import pytest from tdpservice.parsers import parse from tdpservice.parsers.models import DataFileSummary -from .factories import DataFileSummaryFactory -from .test_parse import bad_file_missing_header +from .factories import DataFileSummaryFactory, ParserErrorFactory +from .test_parse import bad_file_missing_header, test_datafile import logging logger = logging.getLogger(__name__) +@pytest.fixture +def dfs(): + """Fixture for DataFileSummary.""" + return DataFileSummaryFactory.create() + @pytest.mark.django_db -def test_dfs_model(): +def test_dfs_model(dfs): """Test that the model is created and populated correctly.""" dfs = DataFileSummaryFactory() assert dfs.case_aggregates['Jan']['accepted'] == 100 @pytest.mark.django_db -def test_dfs_rejected(bad_file_missing_header): +def test_dfs_rejected(test_datafile, dfs): """Ensure that an invalid file generates a rejected status.""" - dfs = DataFileSummaryFactory() - dfs.set_status(parse.parse_datafile(bad_file_missing_header)) + dfs = dfs + test_datafile.section = 'Closed Case Data' + test_datafile.save() + dfs.set_status(parse.parse_datafile(test_datafile)) assert dfs.status == DataFileSummary.Status.REJECTED @pytest.mark.django_db @@ -30,7 +37,20 @@ def test_dfs_set_status(): assert dfs.status == DataFileSummary.Status.PENDING dfs.set_status(errors={}) assert dfs.status == DataFileSummary.Status.ACCEPTED - dfs.set_status(errors=[1, 2, 3]) + + # create category 1 ParserError list to prompt rejected status + parser_errors = [] + + # this feels precarious for tests. the defaults in the factory could cause issues should logic change + # resulting in failures if we stop keying off category and instead go to content or msg + for i in range(1, 4): + parser_errors.append(ParserErrorFactory(row_number=(i), category=i)) + + dfs.set_status(errors={'document': parser_errors}) + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - dfs.set_status(errors={'document': ['No headers found.']}) - assert dfs.status == DataFileSummary.Status.REJECTED + + parser_errors.append(ParserErrorFactory(row_number=5, category=5, error_type='pre-parsing')) + dfs.set_status(errors={'document': parser_errors}) + + assert dfs.status == DataFileSummary.Status.REJECTED \ No newline at end of file From c5796da69d0e9a6d356057550378d536e2be5f8b Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 26 Apr 2023 14:33:28 -0400 Subject: [PATCH 006/120] Merging in Jan's 1610 code for parserError useful-ness --- tdrs-backend/tdpservice/parsers/admin.py | 16 ++ tdrs-backend/tdpservice/parsers/parse.py | 97 +++++++- .../tdpservice/parsers/schema_defs/header.py | 20 +- .../tdpservice/parsers/schema_defs/ssp/m1.py | 108 +++++++++ .../tdpservice/parsers/schema_defs/ssp/m2.py | 151 ++++++++++++ .../tdpservice/parsers/schema_defs/ssp/m3.py | 218 ++++++++++++++++++ .../tdpservice/parsers/schema_defs/tanf/t1.py | 92 ++++---- .../tdpservice/parsers/schema_defs/trailer.py | 6 +- .../tdpservice/parsers/test/test_util.py | 185 +++++++++++++-- tdrs-backend/tdpservice/parsers/util.py | 153 ++++++++++-- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 11 files changed, 942 insertions(+), 106 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py create mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py create mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py diff --git a/tdrs-backend/tdpservice/parsers/admin.py b/tdrs-backend/tdpservice/parsers/admin.py index 3d785191b..3ac3b444b 100644 --- a/tdrs-backend/tdpservice/parsers/admin.py +++ b/tdrs-backend/tdpservice/parsers/admin.py @@ -1,3 +1,19 @@ """Django admin customizations for the parser models.""" +from django.contrib import admin +from . import models + # Register your models here. +class ParserErrorAdmin(admin.ModelAdmin): + """ModelAdmin class for parsed M1 data files.""" + + list_display = [ + 'row_number', + 'field_name', + 'category', + 'error_type', + 'error_message', + ] + + +admin.site.register(models.ParserError, ParserErrorAdmin) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 812047e95..afaec32e8 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -2,8 +2,13 @@ import os +<<<<<<< HEAD from . import schema_defs from . import validators +======= +from . import schema_defs, validators, util +# from .models import ParserError +>>>>>>> 6772b8e6 (wip parser error generator) from tdpservice.data_files.models import DataFile from .models import DataFileSummary @@ -13,7 +18,10 @@ def parse_datafile(datafile): rawfile = datafile.file errors = {} - document_is_valid, document_error = validators.validate_single_header_trailer(rawfile) + document_is_valid, document_error = validators.validate_single_header_trailer( + rawfile, + util.make_generate_parser_error(datafile, 1) + ) if not document_is_valid: errors['document'] = [document_error] return errors @@ -31,12 +39,18 @@ def parse_datafile(datafile): trailer_line = rawfile.readline().decode().strip('\n') # parse header, trailer - header, header_is_valid, header_errors = schema_defs.header.parse_and_validate(header_line) + header, header_is_valid, header_errors = schema_defs.header.parse_and_validate( + header_line, + util.make_generate_parser_error(datafile, 1) + ) if not header_is_valid: errors['header'] = header_errors return errors - trailer, trailer_is_valid, trailer_errors = schema_defs.trailer.parse_and_validate(trailer_line) + trailer, trailer_is_valid, trailer_errors = schema_defs.trailer.parse_and_validate( + trailer_line, + util.make_generate_parser_error(datafile, -1) + ) if not trailer_is_valid: errors['trailer'] = trailer_errors @@ -60,6 +74,7 @@ def parse_datafile(datafile): section = header['type'] if datafile.section != section_names.get(program_type, {}).get(section): + # error_func call errors['document'] = ['Section does not match.'] return errors @@ -78,6 +93,7 @@ def parse_datafile(datafile): schema = get_schema(line, section, schema_options) record_is_valid, record_errors = parse_datafile_line(line, schema) +<<<<<<< HEAD if not record_is_valid: errors[line_number] = record_errors @@ -86,17 +102,88 @@ def parse_datafile(datafile): summary.save() return summary, errors +======= + if isinstance(schema, util.MultiRecordRowSchema): + records = parse_multi_record_line( + line, + schema, + util.make_generate_parser_error(datafile, line_number) + ) + + n = 0 + for r in records: + n += 1 + record, record_is_valid, record_errors = r + if not record_is_valid: + line_errors = errors.get(line_number, {}) + line_errors[n] = record_errors + errors[line_number] = line_errors + else: + record_is_valid, record_errors = parse_datafile_line( + line, + schema, + util.make_generate_parser_error(datafile, line_number) + ) + if not record_is_valid: + errors[line_number] = record_errors + + return errors + +def parse_multi_record_line(line, schema, error_func): + if schema: + records = schema.parse_and_validate(line, error_func) + + for r in records: + record, record_is_valid, record_errors = r + + if record: + record.save() + + # for error_msg in record_errors: + # error_obj = ParserError.objects.create( + # file=None, + # row_number=None, + # column_number=None, + # field_name=None, + # category=None, + # case_number=getattr(record, 'CASE_NUMBER', None), + # error_message=error_msg, + # error_type=None, + # content_type=schema.model, + # object_id=record.pk, + # fields_json=None + # ) + return records -def parse_datafile_line(line, schema): + return [(None, False, ['No schema selected.'])] +>>>>>>> 6772b8e6 (wip parser error generator) + + +def parse_datafile_line(line, schema, error_func): """Parse and validate a datafile line and save any errors to the model.""" if schema: - record, record_is_valid, record_errors = schema.parse_and_validate(line) + record, record_is_valid, record_errors = schema.parse_and_validate(line, error_func) if record: record.errors = record_errors record.save() + # for error_msg in record_errors: + # error_obj = ParserError.objects.create( + # file=None, + # row_number=None, + # column_number=None, + # field_name=None, + # category=None, + # case_number=None, + # error_message=error_msg, + # error_type=None, + # content_type=schema.model, + # object_id=record.pk, + # fields_json=None + # ) + return record_is_valid, record_errors return (False, ['No schema selected.']) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index b6af3e713..c2b0b55c0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -13,34 +13,34 @@ ], postparsing_validators=[], fields=[ - Field(name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ + Field(item=1, name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ validators.matches('HEADER'), ]), - Field(name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ + Field(item=2, name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ validators.between(2000, 2099) ]), - Field(name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ + Field(item=3, name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ validators.oneOf(['1', '2', '3', '4']) ]), - Field(name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ + Field(item=4, name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ validators.oneOf(['A', 'C', 'G', 'S']) ]), - Field(name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ + Field(item=5, name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ validators.between(0, 99) ]), - Field(name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ + Field(item=6, name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ validators.between(0, 999) ]), - Field(name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ + Field(item=7, name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ validators.oneOf(['TAN', 'SSP']) ]), - Field(name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ + Field(item=8, name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ validators.oneOf(['1', '2']) ]), - Field(name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ + Field(item=9, name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ validators.matches('E') ]), - Field(name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ + Field(item=10, name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ validators.oneOf(['N', 'D', 'U']) ]), ], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py new file mode 100644 index 000000000..390e1e20d --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -0,0 +1,108 @@ +"""Schema for SSP M1 record type.""" + + +from ...util import RowSchema, Field +from ... import validators +from tdpservice.search_indexes.models.ssp import SSP_M1 + + +m1 = RowSchema( + model=SSP_M1, + preparsing_validators=[ + validators.hasLength(150), + ], + postparsing_validators=[], + fields=[ + Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ + ]), + Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ + ]), + Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ + ]), + Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[ + ]), + Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[ + ]), + Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ + ]), + Field(item=7, name='DISPOSITION', type='number', startIndex=29, endIndex=30, required=True, validators=[ + ]), + # Field(itemNumbe=1, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[ + # ]), + Field(item=8, name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, required=True, validators=[ + ]), + Field(item=9, name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33, required=True, validators=[ + ]), + Field(item=10, name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, required=True, validators=[ + ]), + Field(item=11, name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, required=True, validators=[ + ]), + Field(item=12, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, required=True, validators=[ + ]), + Field(item=13, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, required=True, validators=[ + ]), + Field(item=14, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, required=True, validators=[ + ]), + Field(item=15, name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, required=True, validators=[ + ]), + Field(item=16, name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, required=True, validators=[ + ]), + Field(item=17, name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, required=True, validators=[ + ]), + Field(item=18, name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, required=True, validators=[ + ]), + Field(item=19, name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, required=True, validators=[ + ]), + Field(item=20, name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, required=True, validators=[ + ]), + Field(item=21, name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, required=True, validators=[ + ]), + Field(item=22, name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, required=True, validators=[ + ]), + Field(item=23, name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, required=True, validators=[ + ]), + Field(item=24, name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, required=True, validators=[ + ]), + Field(item=25, name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, required=True, validators=[ + ]), + Field(item=26, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, required=True, validators=[ + ]), + Field(item=27, name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, required=True, validators=[ + ]), + Field(item=28, name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, required=True, validators=[ + ]), + Field(item=29, name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, required=True, validators=[ + ]), + Field(item=30, name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, required=True, validators=[ + ]), + Field(item=31, name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, required=True, validators=[ + ]), + Field(item=32, name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, required=True, validators=[ + ]), + Field(item=33, name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, required=True, validators=[ + ]), + Field(item=34, name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, required=True, validators=[ + ]), + Field(item=35, name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, required=True, validators=[ + ]), + Field(item=36, name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, required=True, validators=[ + ]), + Field(item=37, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, required=True, validators=[ + ]), + Field(item=38, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, required=True, validators=[ + ]), + Field(item=39, name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, required=True, validators=[ + ]), + Field(item=40, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, required=True, validators=[ + ]), + Field(item=41, name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, required=True, validators=[ + ]), + Field(item=42, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, required=True, validators=[ + ]), + # Field(item=1, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ + # ]), + # Field(item=1, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ + # ]), + Field(item=43, name='BLANK', type='string', startIndex=113, endIndex=150, required=False, validators=[]), + ], +) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py new file mode 100644 index 000000000..8870d140d --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -0,0 +1,151 @@ +"""Schema for SSP M1 record type.""" + + +from ...util import RowSchema, Field +from ... import validators +from tdpservice.search_indexes.models.ssp import SSP_M2 + + +m2 = RowSchema( + model=SSP_M2, + preparsing_validators=[ + validators.hasLength(150), + ], + postparsing_validators=[], + fields=[ + Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ + ]), + Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ + ]), + Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ + ]), + # Field(item=1, name='FIPS_CODE', type='string', startIndex=8, endIndex=19, required=True, validators=[ + # ]), + Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[ + ]), + Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[ + ]), + Field(item=6, name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, required=True, validators=[ + ]), + Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[ + ]), + Field(item=8, name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, validators=[ + ]), + Field(item=9, name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[ + ]), + Field(item=10, name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, validators=[ + ]), + Field(item=11, name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, validators=[ + ]), + Field(item=12, name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, validators=[ + ]), + Field(item=13, name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, validators=[ + ]), + Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[ + ]), + Field(item=15, name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[ + ]), + Field(item=16, name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[ + ]), + Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, required=True, validators=[ + ]), + Field(item=18, name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, validators=[ + ]), + Field(item=19, name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[ + ]), + Field(item=20, name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, validators=[ + ]), + Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[ + ]), + Field(item=22, name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, validators=[ + ]), + Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, validators=[ + ]), + Field(item=24, name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, required=True, validators=[ + ]), + Field(item=25, name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, validators=[ + ]), + Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, validators=[ + ]), + Field(item=27, name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, required=True, validators=[ + ]), + Field(item=28, name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, required=True, validators=[ + ]), + Field(item=29, name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, required=True, validators=[ + ]), + Field(item=30, name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, required=True, validators=[ + ]), + Field(item=31, name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, required=True, validators=[ + ]), + Field(item=32, name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, required=True, validators=[ + ]), + Field(item=33, name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, required=True, validators=[ + ]), + Field(item=34, name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, required=True, validators=[ + ]), + Field(item=35, name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, required=True, validators=[ + ]), + Field(item=36, name='OJT', type='number', startIndex=76, endIndex=78, required=True, validators=[ + ]), + Field(item=37, name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, required=True, validators=[ + ]), + Field(item=38, name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, required=True, validators=[ + ]), + Field(item=39, name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, required=True, validators=[ + ]), + Field(item=40, name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, required=True, validators=[ + ]), + Field(item=41, name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, required=True, validators=[ + ]), + Field(item=42, name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, required=True, validators=[ + ]), + Field(item=43, name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, required=True, validators=[ + ]), + Field(item=44, name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, required=True, validators=[ + ]), + Field(item=45, name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, required=True, validators=[ + ]), + Field(item=46, name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, required=True, validators=[ + ]), + Field(item=47, name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, required=True, validators=[ + ]), + Field(item=48, name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, required=True, validators=[ + ]), + Field(item=49, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, required=True, validators=[ + ]), + Field(item=50, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, required=True, validators=[ + ]), + Field(item=51, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, required=True, validators=[ + ]), + Field(item=52, name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, required=True, validators=[ + ]), + Field(item=53, name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, required=True, validators=[ + ]), + Field(item=54, name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, required=True, validators=[ + ]), + Field(item=55, name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, required=True, validators=[ + ]), + Field(item=56, name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, required=True, validators=[ + ]), + Field(item=57, name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, required=True, validators=[ + ]), + Field(item=58, name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, required=True, validators=[ + ]), + Field(item=59, name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, required=True, validators=[ + ]), + Field(item=60, name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, required=True, validators=[ + ]), + Field(item=61, name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, required=True, validators=[ + ]), + Field(item=62, name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, required=True, validators=[ + ]), + Field(item=63, name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, required=True, validators=[ + ]), + Field(item=64, name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, required=True, validators=[ + ]), + Field(item=65, name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, required=True, validators=[ + ]), + Field(item=66, name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, required=True, validators=[ + ]), + ], +) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py new file mode 100644 index 000000000..7ab02db76 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -0,0 +1,218 @@ +"""Schema for SSP M1 record type.""" + + +from ...util import MultiRecordRowSchema, RowSchema, Field +from ... import validators +from tdpservice.search_indexes.models.ssp import SSP_M3 + +first_part_schema = RowSchema( + model=SSP_M3, + preparsing_validators=[ + # validators.hasLength(150), # unreliable. + validators.notEmpty(start=19, end=60), + ], + postparsing_validators=[], + fields=[ + Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ + ]), + Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ + ]), + Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ + ]), + # Field(item=1, name='FIPS_CODE', type='string', startIndex=8, endIndex=19, required=True, validators=[ + # ]), + Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[ + ]), + Field(item=5, name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, required=True, validators=[ + ]), + Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[ + ]), + Field(item=7, name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[ + ]), + Field(item=8, name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[ + ]), + Field(item=9, name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[ + ]), + Field(item=10, name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[ + ]), + Field(item=11, name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[ + ]), + Field(item=12, name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[ + ]), + Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ + ]), + Field(item=14, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, required=True, validators=[ + ]), + Field(item=15, name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, required=True, validators=[ + ]), + Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[ + ]), + Field(item=17, name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[ + ]), + Field(item=18, name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, required=True, validators=[ + ]), + Field(item=19, name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[ + ]), + Field(item=20, name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, required=True, validators=[ + ]), + Field(item=21, name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, required=True, validators=[ + ]) + ] +) + +second_part_schema = RowSchema( + model=SSP_M3, + quiet_preparser_errors=True, + preparsing_validators=[ + # validators.hasLength(150), # unreliable. + validators.notEmpty(start=60, end=101), + ], + postparsing_validators=[], + fields=[ + Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ + ]), + Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ + ]), + Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ + ]), + # Field(item=1, name='FIPS_CODE', type='string', startIndex=8, endIndex=19, required=True, validators=[ + # ]), + Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[ + ]), + Field(item=23, name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, required=True, validators=[ + ]), + Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[ + ]), + Field(item=25, name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[ + ]), + Field(item=26, name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[ + ]), + Field(item=27, name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[ + ]), + Field(item=28, name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[ + ]), + Field(item=29, name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[ + ]), + Field(item=30, name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[ + ]), + Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ + ]), + Field(item=32, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, required=True, validators=[ + ]), + Field(item=33, name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, required=True, validators=[ + ]), + Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[ + ]), + Field(item=35, name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[ + ]), + Field(item=36, name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, required=True, validators=[ + ]), + Field(item=37, name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[ + ]), + Field(item=38, name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, required=True, validators=[ + ]), + Field(item=39, name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, required=True, validators=[ + ]) + ] +) + +m3 = MultiRecordRowSchema( + schemas=[ + first_part_schema, + second_part_schema + ] +) + + +# m3 = RowSchema( +# model=SSP_M3, +# preparsing_validators=[ +# validators.hasLength(150), # ?? the format document shows double fields/length? +# ], +# postparsing_validators=[], +# fields=[ +# Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ +# ]), +# Field(item=1, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ +# ]), +# Field(item=1, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ +# ]), +# # Field(item=1, name='FIPS_CODE', type='string', startIndex=8, endIndex=19, required=True, validators=[ +# # ]), +# Field(item=1, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[ +# ]), +# Field(item=1, name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, required=True, validators=[ +# ]), +# Field(item=1, name='SSN', type='number', startIndex=28, endIndex=37, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[ +# ]), +# Field(item=1, name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ +# ]), +# Field(item=1, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, required=True, validators=[ +# ]), +# Field(item=1, name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, required=True, validators=[ +# ]), +# Field(item=1, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[ +# ]), +# Field(item=1, name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[ +# ]), +# Field(item=1, name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, required=True, validators=[ +# ]), +# Field(item=1, name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[ +# ]), +# Field(item=1, name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, required=True, validators=[ +# ]), +# Field(item=1, name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, required=True, validators=[ +# ]), + +# # ?? +# Field(item=1, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[ +# ]), +# Field(item=1, name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, required=True, validators=[ +# ]), +# Field(item=1, name='SSN', type='number', startIndex=69, endIndex=78, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[ +# ]), +# Field(item=1, name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[ +# ]), +# Field(item=1, name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ +# ]), +# Field(item=1, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, required=True, validators=[ +# ]), +# Field(item=1, name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, required=True, validators=[ +# ]), +# Field(item=1, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[ +# ]), +# Field(item=1, name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[ +# ]), +# Field(item=1, name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, required=True, validators=[ +# ]), +# Field(item=1, name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[ +# ]), +# Field(item=1, name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, required=True, validators=[ +# ]), +# Field(item=1, name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, required=True, validators=[ +# ]), +# # Field(item=1, name='BLANK', type='string', startIndex=101, endIndex=150, required=False, validators=[]), +# ], +# ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index cea1ce3d2..e02930dc4 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -13,96 +13,96 @@ ], postparsing_validators=[], fields=[ - Field(name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ + Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ ]), - Field(name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ + Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ ]), - Field(name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ + Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ ]), - Field(name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[ + Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[ ]), - Field(name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[ + Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[ ]), - Field(name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ + Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ ]), - Field(name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[ + Field(item=7, name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[ ]), - Field(name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[ + Field(item=8, name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[ ]), - Field(name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[ + Field(item=9, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[ ]), - Field(name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[ + Field(item=10, name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[ ]), - Field(name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[ + Field(item=11, name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[ ]), - Field(name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[ + Field(item=12, name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[ ]), - Field(name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[ + Field(item=13, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[ ]), - Field(name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[ + Field(item=14, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[ ]), - Field(name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ + Field(item=15, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ ]), - Field(name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[ + Field(item=16, name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[ ]), - Field(name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ + Field(item=17, name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ ]), - Field(name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[ + Field(item=18, name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[ ]), - Field(name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[ + Field(item=19, name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[ ]), - Field(name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[ + Field(item=20, name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[ ]), - Field(name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[ + Field(item=21, name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[ ]), - Field(name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[ + Field(item=22, name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[ ]), - Field(name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[ + Field(item=23, name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[ ]), - Field(name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[ + Field(item=24, name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[ ]), - Field(name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[ + Field(item=25, name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[ ]), - Field(name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[ + Field(item=26, name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[ ]), - Field(name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[ + Field(item=27, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[ ]), - Field(name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[ + Field(item=28, name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[ ]), - Field(name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[ + Field(item=29, name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[ ]), - Field(name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[ + Field(item=30, name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[ ]), - Field(name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ + Field(item=31, name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ ]), - Field(name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[ + Field(item=32, name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[ ]), - Field(name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[ + Field(item=33, name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[ ]), - Field(name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ + Field(item=34, name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ ]), - Field(name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[ + Field(item=35, name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[ ]), - Field(name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[ + Field(item=36, name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[ ]), - Field(name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[ + Field(item=37, name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[ ]), - Field(name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[ + Field(item=38, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[ ]), - Field(name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[ + Field(item=39, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[ ]), - Field(name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[ + Field(item=40, name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[ ]), - Field(name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[ + Field(item=41, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[ ]), - Field(name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[ + Field(item=42, name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[ ]), - Field(name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[ + Field(item=43, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[ ]), - Field(name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ + Field(item=44, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ ]), - Field(name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ + Field(item=45, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ ]), - Field(name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), + Field(item=46, name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index b1b912c0e..625b2042e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -13,13 +13,13 @@ ], postparsing_validators=[], fields=[ - Field(name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ + Field(item=1, name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ validators.matches('TRAILER') ]), - Field(name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ + Field(item=2, name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ validators.between(0, 9999999) ]), - Field(name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ + Field(item=3, name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ validators.matches(' ') ]), ], diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index a5ee1cc06..075f3eb52 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -14,6 +14,10 @@ def failing_validator(): return lambda _: (False, 'Value is not valid.') +def error_func(schema, error_category, error_message, record, field): + return error_message + + def test_run_preparsing_validators_returns_valid(): """Test run_preparsing_validators executes all preparsing_validators provided in schema.""" line = '12345' @@ -23,7 +27,7 @@ def test_run_preparsing_validators_returns_valid(): ] ) - is_valid, errors = schema.run_preparsing_validators(line) + is_valid, errors = schema.run_preparsing_validators(line, error_func) assert is_valid is True assert errors == [] @@ -38,7 +42,7 @@ def test_run_preparsing_validators_returns_invalid_and_errors(): ] ) - is_valid, errors = schema.run_preparsing_validators(line) + is_valid, errors = schema.run_preparsing_validators(line, error_func) assert is_valid is False assert errors == ['Value is not valid.'] @@ -49,9 +53,17 @@ def test_parse_line_parses_line_from_schema_to_dict(): schema = RowSchema( model=dict, fields=[ +<<<<<<< HEAD Field(name='first', type='string', startIndex=0, endIndex=3), Field(name='second', type='string', startIndex=3, endIndex=4), Field(name='third', type='string', startIndex=4, endIndex=5), +======= + Field(item=1, name='first', type='string', startIndex=0, endIndex=3), + Field(item=2, name='second', type='string', startIndex=3, endIndex=4), + Field(item=3, name='third', type='string', startIndex=4, endIndex=5), + Field(item=4, name='fourth', type='number', startIndex=5, endIndex=7), + Field(item=5, name='fifth', type='number', startIndex=7, endIndex=8), +>>>>>>> 6772b8e6 (wip parser error generator) ] ) @@ -73,9 +85,17 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ +<<<<<<< HEAD Field(name='first', type='string', startIndex=0, endIndex=3), Field(name='second', type='string', startIndex=3, endIndex=4), Field(name='third', type='string', startIndex=4, endIndex=5), +======= + Field(item=1, name='first', type='string', startIndex=0, endIndex=3), + Field(item=2, name='second', type='string', startIndex=3, endIndex=4), + Field(item=3, name='third', type='string', startIndex=4, endIndex=5), + Field(item=4, name='fourth', type='number', startIndex=5, endIndex=7), + Field(item=5, name='fifth', type='number', startIndex=7, endIndex=8), +>>>>>>> 6772b8e6 (wip parser error generator) ] ) @@ -96,19 +116,19 @@ def test_run_field_validators_returns_valid_with_dict(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator() ]), - Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is True assert errors == [] @@ -128,19 +148,19 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator() ]), - Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is True assert errors == [] @@ -155,20 +175,20 @@ def test_run_field_validators_returns_invalid_with_dict(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator(), failing_validator() ]), - Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is False assert errors == ['Value is not valid.'] @@ -188,20 +208,20 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator(), failing_validator() ]), - Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is False assert errors == ['Value is not valid.'] @@ -215,16 +235,16 @@ def test_field_validators_blank_and_required_returns_error(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=1, required=True, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=1, required=True, validators=[ passing_validator(), ]), - Field(name='second', type='string', startIndex=1, endIndex=3, required=True, validators=[ + Field(item=2, name='second', type='string', startIndex=1, endIndex=3, required=True, validators=[ passing_validator(), ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is False assert errors == [ 'first is required but a value was not provided.', @@ -240,14 +260,14 @@ def test_field_validators_blank_and_not_required_returns_valid(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[ passing_validator(), failing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is True assert errors == [] @@ -261,7 +281,7 @@ def test_run_postparsing_validators_returns_valid(): ] ) - is_valid, errors = schema.run_postparsing_validators(instance) + is_valid, errors = schema.run_postparsing_validators(instance, error_func) assert is_valid is True assert errors == [] @@ -276,6 +296,121 @@ def test_run_postparsing_validators_returns_invalid_and_errors(): ] ) - is_valid, errors = schema.run_postparsing_validators(instance) + is_valid, errors = schema.run_postparsing_validators(instance, error_func) assert is_valid is False assert errors == ['Value is not valid.'] +<<<<<<< HEAD +======= + + +@pytest.mark.parametrize("value,length", [ + (None, 0), + (None, 10), + (' ', 5), + ('###', 3) +]) +def test_value_is_empty_returns_true(value, length): + result = value_is_empty(value, length) + assert result is True + + +@pytest.mark.parametrize("value,length", [ + (0, 1), + (1, 1), + (10, 2), + ('0', 1), + ('0000', 4), + ('1 ', 5), + ('##3', 3) +]) +def test_value_is_empty_returns_false(value, length): + result = value_is_empty(value, length) + assert result is False + + +def test_multi_record_schema_parses_and_validates(): + line = '12345' + schema = MultiRecordRowSchema( + schemas=[ + RowSchema( + model=dict, + preparsing_validators=[ + passing_validator() + ], + postparsing_validators=[ + failing_validator() + ], + fields=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ + passing_validator() + ]), + ] + ), + RowSchema( + model=dict, + preparsing_validators=[ + passing_validator() + ], + postparsing_validators=[ + passing_validator() + ], + fields=[ + Field(item=2, name='second', type='string', startIndex=2, endIndex=4, validators=[ + passing_validator() + ]), + ] + ), + RowSchema( + model=dict, + preparsing_validators=[ + failing_validator() + ], + postparsing_validators=[ + passing_validator() + ], + fields=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ + passing_validator() + ]), + ] + ), + RowSchema( + model=dict, + preparsing_validators=[ + passing_validator() + ], + postparsing_validators=[ + passing_validator() + ], + fields=[ + Field(item=4, name='fourth', type='string', startIndex=4, endIndex=5, validators=[ + failing_validator() + ]), + ] + ) + ] + ) + + rs = schema.parse_and_validate(line, error_func) + + r0_record, r0_is_valid, r0_errors = rs[0] + r1_record, r1_is_valid, r1_errors = rs[1] + r2_record, r2_is_valid, r2_errors = rs[2] + r3_record, r3_is_valid, r3_errors = rs[3] + + assert r0_record == {'first': '123'} + assert r0_is_valid is False + assert r0_errors == ['Value is not valid.'] + + assert r1_record == {'second': '34'} + assert r1_is_valid is True + assert r1_errors == [] + + assert r2_record is None + assert r2_is_valid is False + assert r2_errors == ['Value is not valid.'] + + assert r3_record == {'fourth': '5'} + assert r3_is_valid is False + assert r3_errors == ['Value is not valid.'] +>>>>>>> 6772b8e6 (wip parser error generator) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 7a1bcc2f6..c125eed07 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -1,12 +1,77 @@ """Utility file for functions shared between all parsers even preparser.""" +<<<<<<< HEAD import logging logger = logging.getLogger(__name__) +======= +from .models import ParserError +from django.contrib.contenttypes.models import ContentType +from tdpservice.search_indexes.models.ssp import SSP_M1 + +def value_is_empty(value, length): + """Handle 'empty' values as field inputs.""" + + empty_values = [ + ' '*length, # ' ' + '#'*length, # '#####' + ] + + return value is None or value in empty_values + +>>>>>>> 6772b8e6 (wip parser error generator) + +error_types = { + 1: 'File pre-check', + 2: 'Record value invalid', + 3: 'Record value consistency', + 4: 'Case consistency', + 5: 'Section consistency', + 6: 'Historical consistency' +} + +# def generate_parser_error(datafile, schema, record, line_number, field, error_category, error_msg): +# return ParserError.objects.create( +# file=datafile, +# row_number=line_number, +# column_number=field.item_number, +# field_name=field.name, +# category=error_category, +# case_number=getattr(record, 'CASE_NUMBER', None), +# error_message=error_msg, +# error_type=error_types.get(error_category, None), +# content_type=schema.model, +# object_id=record.pk, +# fields_json=None +# ) + + +def make_generate_parser_error(datafile, line_number): + def generate_parser_error(schema, error_category, error_message, record=None, field=None): + model = schema.model if schema else None + return ParserError.objects.create( + file=datafile, + row_number=line_number, + column_number=getattr(field, 'item', 0), + item_number=getattr(field, 'item', 0), + field_name=getattr(field, 'name', 'none'), + category=error_category, + case_number=getattr(record, 'CASE_NUMBER', None), + error_message=error_message, + error_type=error_types.get(error_category, None), + content_type=ContentType.objects.get( + model=model if record and not isinstance(record, dict) else 'ssp_m1' + ), + object_id=getattr(record, 'pk', 0) if record and not isinstance(record, dict) else 0, + fields_json=None + ) + + return generate_parser_error class Field: """Provides a mapping between a field name and its position.""" - def __init__(self, name, type, startIndex, endIndex, required=True, validators=[]): + def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]): + self.item = item self.name = name self.type = type self.startIndex = startIndex @@ -14,9 +79,9 @@ def __init__(self, name, type, startIndex, endIndex, required=True, validators=[ self.required = required self.validators = validators - def create(self, name, length, start, end, type): + def create(self, item, name, length, start, end, type): """Create a new field.""" - return Field(name, type, length, start, end) + return Field(item, name, type, length, start, end) def __repr__(self): """Return a string representation of the field.""" @@ -43,10 +108,10 @@ def __init__(self, model=dict, preparsing_validators=[], postparsing_validators= self.fields = fields # self.section = section # intended for future use with multiple section objects - def _add_field(self, name, length, start, end, type): + def _add_field(self, item, name, length, start, end, type): """Add a field to the schema.""" self.fields.append( - Field(name, type, start, end) + Field(item, name, type, start, end) ) def add_fields(self, fields: list): @@ -58,12 +123,12 @@ def get_all_fields(self): """Get all fields from the schema.""" return self.fields - def parse_and_validate(self, line): + def parse_and_validate(self, line, error_func): """Run all validation steps in order, and parse the given line into a record.""" errors = [] # run preparsing validators - preparsing_is_valid, preparsing_errors = self.run_preparsing_validators(line) + preparsing_is_valid, preparsing_errors = self.run_preparsing_validators(line, error_func) if not preparsing_is_valid: return None, False, preparsing_errors @@ -72,17 +137,17 @@ def parse_and_validate(self, line): record = self.parse_line(line) # run field validators - fields_are_valid, field_errors = self.run_field_validators(record) + fields_are_valid, field_errors = self.run_field_validators(record, error_func) # run postparsing validators - postparsing_is_valid, postparsing_errors = self.run_postparsing_validators(record) + postparsing_is_valid, postparsing_errors = self.run_postparsing_validators(record, error_func) is_valid = fields_are_valid and postparsing_is_valid errors = field_errors + postparsing_errors return record, is_valid, errors - def run_preparsing_validators(self, line): + def run_preparsing_validators(self, line, error_func): """Run each of the `preparsing_validator` functions in the schema against the un-parsed line.""" is_valid = True errors = [] @@ -91,7 +156,15 @@ def run_preparsing_validators(self, line): validator_is_valid, validator_error = validator(line) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append(validator_error) + errors.append( + error_func( + schema=self, + error_category=1, + error_message=validator_error, + record=None, + field=None + ) + ) return is_valid, errors @@ -107,9 +180,12 @@ def parse_line(self, line): else: setattr(record, field.name, value) + # if not isinstance(record, dict): + # record.save() + return record - def run_field_validators(self, instance): + def run_field_validators(self, instance, error_func): """Run all validators for each field in the parsed model.""" is_valid = True errors = [] @@ -126,14 +202,30 @@ def run_field_validators(self, instance): validator_is_valid, validator_error = validator(value) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append(validator_error) + errors.append( + error_func( + schema=self, + error_category=2, + error_message=validator_error, + record=instance, + field=field + ) + ) elif field.required: is_valid = False - errors.append(f"{field.name} is required but a value was not provided.") + errors.append( + error_func( + schema=self, + error_category=2, + error_message=f"{field.name} is required but a value was not provided.", + record=instance, + field=field + ) + ) return is_valid, errors - def run_postparsing_validators(self, instance): + def run_postparsing_validators(self, instance, error_func): """Run each of the `postparsing_validator` functions against the parsed model.""" is_valid = True errors = [] @@ -142,6 +234,35 @@ def run_postparsing_validators(self, instance): validator_is_valid, validator_error = validator(instance) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append(validator_error) + errors.append( + error_func( + schema=self, + error_category=3, + error_message=validator_error, + record=instance, + field=None + ) + ) return is_valid, errors +<<<<<<< HEAD +======= + + +class MultiRecordRowSchema: + """Maps a line to multiple `RowSchema`s and runs all parsers and validators.""" + + def __init__(self, schemas): + # self.common_fields = None + self.schemas = schemas + + def parse_and_validate(self, line, error_func): + """Run `parse_and_validate` for each schema provided and bubble up errors.""" + records = [] + + for schema in self.schemas: + r = schema.parse_and_validate(line, error_func) + records.append(r) + + return records +>>>>>>> 6772b8e6 (wip parser error generator) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index cedaf50b0..7b7d41b1d 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -60,7 +60,7 @@ def startsWith(substring): # custom validators -def validate_single_header_trailer(file): +def validate_single_header_trailer(file, error_func): """Validate that a raw datafile has one trailer and one footer.""" headers = 0 trailers = 0 From e6087fd90ae5483b088024774c7222d32953bd4a Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 26 Apr 2023 14:37:21 -0400 Subject: [PATCH 007/120] Revert "Merging in Jan's 1610 code for parserError useful-ness" This reverts commit c5796da69d0e9a6d356057550378d536e2be5f8b. --- tdrs-backend/tdpservice/parsers/admin.py | 16 -- tdrs-backend/tdpservice/parsers/parse.py | 97 +------- .../tdpservice/parsers/schema_defs/header.py | 20 +- .../tdpservice/parsers/schema_defs/ssp/m1.py | 108 --------- .../tdpservice/parsers/schema_defs/ssp/m2.py | 151 ------------ .../tdpservice/parsers/schema_defs/ssp/m3.py | 218 ------------------ .../tdpservice/parsers/schema_defs/tanf/t1.py | 92 ++++---- .../tdpservice/parsers/schema_defs/trailer.py | 6 +- .../tdpservice/parsers/test/test_util.py | 185 ++------------- tdrs-backend/tdpservice/parsers/util.py | 153 ++---------- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 11 files changed, 106 insertions(+), 942 deletions(-) delete mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py delete mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py delete mode 100644 tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py diff --git a/tdrs-backend/tdpservice/parsers/admin.py b/tdrs-backend/tdpservice/parsers/admin.py index 3ac3b444b..3d785191b 100644 --- a/tdrs-backend/tdpservice/parsers/admin.py +++ b/tdrs-backend/tdpservice/parsers/admin.py @@ -1,19 +1,3 @@ """Django admin customizations for the parser models.""" -from django.contrib import admin -from . import models - # Register your models here. -class ParserErrorAdmin(admin.ModelAdmin): - """ModelAdmin class for parsed M1 data files.""" - - list_display = [ - 'row_number', - 'field_name', - 'category', - 'error_type', - 'error_message', - ] - - -admin.site.register(models.ParserError, ParserErrorAdmin) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index afaec32e8..812047e95 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -2,13 +2,8 @@ import os -<<<<<<< HEAD from . import schema_defs from . import validators -======= -from . import schema_defs, validators, util -# from .models import ParserError ->>>>>>> 6772b8e6 (wip parser error generator) from tdpservice.data_files.models import DataFile from .models import DataFileSummary @@ -18,10 +13,7 @@ def parse_datafile(datafile): rawfile = datafile.file errors = {} - document_is_valid, document_error = validators.validate_single_header_trailer( - rawfile, - util.make_generate_parser_error(datafile, 1) - ) + document_is_valid, document_error = validators.validate_single_header_trailer(rawfile) if not document_is_valid: errors['document'] = [document_error] return errors @@ -39,18 +31,12 @@ def parse_datafile(datafile): trailer_line = rawfile.readline().decode().strip('\n') # parse header, trailer - header, header_is_valid, header_errors = schema_defs.header.parse_and_validate( - header_line, - util.make_generate_parser_error(datafile, 1) - ) + header, header_is_valid, header_errors = schema_defs.header.parse_and_validate(header_line) if not header_is_valid: errors['header'] = header_errors return errors - trailer, trailer_is_valid, trailer_errors = schema_defs.trailer.parse_and_validate( - trailer_line, - util.make_generate_parser_error(datafile, -1) - ) + trailer, trailer_is_valid, trailer_errors = schema_defs.trailer.parse_and_validate(trailer_line) if not trailer_is_valid: errors['trailer'] = trailer_errors @@ -74,7 +60,6 @@ def parse_datafile(datafile): section = header['type'] if datafile.section != section_names.get(program_type, {}).get(section): - # error_func call errors['document'] = ['Section does not match.'] return errors @@ -93,7 +78,6 @@ def parse_datafile(datafile): schema = get_schema(line, section, schema_options) record_is_valid, record_errors = parse_datafile_line(line, schema) -<<<<<<< HEAD if not record_is_valid: errors[line_number] = record_errors @@ -102,88 +86,17 @@ def parse_datafile(datafile): summary.save() return summary, errors -======= - if isinstance(schema, util.MultiRecordRowSchema): - records = parse_multi_record_line( - line, - schema, - util.make_generate_parser_error(datafile, line_number) - ) - - n = 0 - for r in records: - n += 1 - record, record_is_valid, record_errors = r - if not record_is_valid: - line_errors = errors.get(line_number, {}) - line_errors[n] = record_errors - errors[line_number] = line_errors - else: - record_is_valid, record_errors = parse_datafile_line( - line, - schema, - util.make_generate_parser_error(datafile, line_number) - ) - if not record_is_valid: - errors[line_number] = record_errors - - return errors - -def parse_multi_record_line(line, schema, error_func): - if schema: - records = schema.parse_and_validate(line, error_func) - - for r in records: - record, record_is_valid, record_errors = r - - if record: - record.save() - - # for error_msg in record_errors: - # error_obj = ParserError.objects.create( - # file=None, - # row_number=None, - # column_number=None, - # field_name=None, - # category=None, - # case_number=getattr(record, 'CASE_NUMBER', None), - # error_message=error_msg, - # error_type=None, - # content_type=schema.model, - # object_id=record.pk, - # fields_json=None - # ) - return records - return [(None, False, ['No schema selected.'])] ->>>>>>> 6772b8e6 (wip parser error generator) - - -def parse_datafile_line(line, schema, error_func): +def parse_datafile_line(line, schema): """Parse and validate a datafile line and save any errors to the model.""" if schema: - record, record_is_valid, record_errors = schema.parse_and_validate(line, error_func) + record, record_is_valid, record_errors = schema.parse_and_validate(line) if record: record.errors = record_errors record.save() - # for error_msg in record_errors: - # error_obj = ParserError.objects.create( - # file=None, - # row_number=None, - # column_number=None, - # field_name=None, - # category=None, - # case_number=None, - # error_message=error_msg, - # error_type=None, - # content_type=schema.model, - # object_id=record.pk, - # fields_json=None - # ) - return record_is_valid, record_errors return (False, ['No schema selected.']) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index c2b0b55c0..b6af3e713 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -13,34 +13,34 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ + Field(name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ validators.matches('HEADER'), ]), - Field(item=2, name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ + Field(name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ validators.between(2000, 2099) ]), - Field(item=3, name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ + Field(name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ validators.oneOf(['1', '2', '3', '4']) ]), - Field(item=4, name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ + Field(name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ validators.oneOf(['A', 'C', 'G', 'S']) ]), - Field(item=5, name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ + Field(name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ validators.between(0, 99) ]), - Field(item=6, name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ + Field(name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ validators.between(0, 999) ]), - Field(item=7, name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ + Field(name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ validators.oneOf(['TAN', 'SSP']) ]), - Field(item=8, name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ + Field(name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ validators.oneOf(['1', '2']) ]), - Field(item=9, name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ + Field(name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ validators.matches('E') ]), - Field(item=10, name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ + Field(name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ validators.oneOf(['N', 'D', 'U']) ]), ], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py deleted file mode 100644 index 390e1e20d..000000000 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Schema for SSP M1 record type.""" - - -from ...util import RowSchema, Field -from ... import validators -from tdpservice.search_indexes.models.ssp import SSP_M1 - - -m1 = RowSchema( - model=SSP_M1, - preparsing_validators=[ - validators.hasLength(150), - ], - postparsing_validators=[], - fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ - ]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ - ]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ - ]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[ - ]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[ - ]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ - ]), - Field(item=7, name='DISPOSITION', type='number', startIndex=29, endIndex=30, required=True, validators=[ - ]), - # Field(itemNumbe=1, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[ - # ]), - Field(item=8, name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, required=True, validators=[ - ]), - Field(item=9, name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33, required=True, validators=[ - ]), - Field(item=10, name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, required=True, validators=[ - ]), - Field(item=11, name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, required=True, validators=[ - ]), - Field(item=12, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, required=True, validators=[ - ]), - Field(item=13, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, required=True, validators=[ - ]), - Field(item=14, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, required=True, validators=[ - ]), - Field(item=15, name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, required=True, validators=[ - ]), - Field(item=16, name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, required=True, validators=[ - ]), - Field(item=17, name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, required=True, validators=[ - ]), - Field(item=18, name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, required=True, validators=[ - ]), - Field(item=19, name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, required=True, validators=[ - ]), - Field(item=20, name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, required=True, validators=[ - ]), - Field(item=21, name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, required=True, validators=[ - ]), - Field(item=22, name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, required=True, validators=[ - ]), - Field(item=23, name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, required=True, validators=[ - ]), - Field(item=24, name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, required=True, validators=[ - ]), - Field(item=25, name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, required=True, validators=[ - ]), - Field(item=26, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, required=True, validators=[ - ]), - Field(item=27, name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, required=True, validators=[ - ]), - Field(item=28, name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, required=True, validators=[ - ]), - Field(item=29, name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, required=True, validators=[ - ]), - Field(item=30, name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, required=True, validators=[ - ]), - Field(item=31, name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, required=True, validators=[ - ]), - Field(item=32, name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, required=True, validators=[ - ]), - Field(item=33, name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, required=True, validators=[ - ]), - Field(item=34, name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, required=True, validators=[ - ]), - Field(item=35, name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, required=True, validators=[ - ]), - Field(item=36, name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, required=True, validators=[ - ]), - Field(item=37, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, required=True, validators=[ - ]), - Field(item=38, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, required=True, validators=[ - ]), - Field(item=39, name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, required=True, validators=[ - ]), - Field(item=40, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, required=True, validators=[ - ]), - Field(item=41, name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, required=True, validators=[ - ]), - Field(item=42, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, required=True, validators=[ - ]), - # Field(item=1, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ - # ]), - # Field(item=1, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ - # ]), - Field(item=43, name='BLANK', type='string', startIndex=113, endIndex=150, required=False, validators=[]), - ], -) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py deleted file mode 100644 index 8870d140d..000000000 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Schema for SSP M1 record type.""" - - -from ...util import RowSchema, Field -from ... import validators -from tdpservice.search_indexes.models.ssp import SSP_M2 - - -m2 = RowSchema( - model=SSP_M2, - preparsing_validators=[ - validators.hasLength(150), - ], - postparsing_validators=[], - fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ - ]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ - ]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ - ]), - # Field(item=1, name='FIPS_CODE', type='string', startIndex=8, endIndex=19, required=True, validators=[ - # ]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[ - ]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[ - ]), - Field(item=6, name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, required=True, validators=[ - ]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[ - ]), - Field(item=8, name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, validators=[ - ]), - Field(item=9, name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[ - ]), - Field(item=10, name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, validators=[ - ]), - Field(item=11, name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, validators=[ - ]), - Field(item=12, name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, validators=[ - ]), - Field(item=13, name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, validators=[ - ]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[ - ]), - Field(item=15, name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[ - ]), - Field(item=16, name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[ - ]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, required=True, validators=[ - ]), - Field(item=18, name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, validators=[ - ]), - Field(item=19, name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[ - ]), - Field(item=20, name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, validators=[ - ]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[ - ]), - Field(item=22, name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, validators=[ - ]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, validators=[ - ]), - Field(item=24, name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, required=True, validators=[ - ]), - Field(item=25, name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, validators=[ - ]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, validators=[ - ]), - Field(item=27, name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, required=True, validators=[ - ]), - Field(item=28, name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, required=True, validators=[ - ]), - Field(item=29, name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, required=True, validators=[ - ]), - Field(item=30, name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, required=True, validators=[ - ]), - Field(item=31, name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, required=True, validators=[ - ]), - Field(item=32, name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, required=True, validators=[ - ]), - Field(item=33, name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, required=True, validators=[ - ]), - Field(item=34, name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, required=True, validators=[ - ]), - Field(item=35, name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, required=True, validators=[ - ]), - Field(item=36, name='OJT', type='number', startIndex=76, endIndex=78, required=True, validators=[ - ]), - Field(item=37, name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, required=True, validators=[ - ]), - Field(item=38, name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, required=True, validators=[ - ]), - Field(item=39, name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, required=True, validators=[ - ]), - Field(item=40, name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, required=True, validators=[ - ]), - Field(item=41, name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, required=True, validators=[ - ]), - Field(item=42, name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, required=True, validators=[ - ]), - Field(item=43, name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, required=True, validators=[ - ]), - Field(item=44, name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, required=True, validators=[ - ]), - Field(item=45, name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, required=True, validators=[ - ]), - Field(item=46, name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, required=True, validators=[ - ]), - Field(item=47, name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, required=True, validators=[ - ]), - Field(item=48, name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, required=True, validators=[ - ]), - Field(item=49, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, required=True, validators=[ - ]), - Field(item=50, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, required=True, validators=[ - ]), - Field(item=51, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, required=True, validators=[ - ]), - Field(item=52, name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, required=True, validators=[ - ]), - Field(item=53, name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, required=True, validators=[ - ]), - Field(item=54, name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, required=True, validators=[ - ]), - Field(item=55, name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, required=True, validators=[ - ]), - Field(item=56, name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, required=True, validators=[ - ]), - Field(item=57, name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, required=True, validators=[ - ]), - Field(item=58, name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, required=True, validators=[ - ]), - Field(item=59, name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, required=True, validators=[ - ]), - Field(item=60, name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, required=True, validators=[ - ]), - Field(item=61, name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, required=True, validators=[ - ]), - Field(item=62, name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, required=True, validators=[ - ]), - Field(item=63, name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, required=True, validators=[ - ]), - Field(item=64, name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, required=True, validators=[ - ]), - Field(item=65, name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, required=True, validators=[ - ]), - Field(item=66, name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, required=True, validators=[ - ]), - ], -) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py deleted file mode 100644 index 7ab02db76..000000000 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Schema for SSP M1 record type.""" - - -from ...util import MultiRecordRowSchema, RowSchema, Field -from ... import validators -from tdpservice.search_indexes.models.ssp import SSP_M3 - -first_part_schema = RowSchema( - model=SSP_M3, - preparsing_validators=[ - # validators.hasLength(150), # unreliable. - validators.notEmpty(start=19, end=60), - ], - postparsing_validators=[], - fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ - ]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ - ]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ - ]), - # Field(item=1, name='FIPS_CODE', type='string', startIndex=8, endIndex=19, required=True, validators=[ - # ]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[ - ]), - Field(item=5, name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, required=True, validators=[ - ]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[ - ]), - Field(item=7, name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[ - ]), - Field(item=8, name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[ - ]), - Field(item=9, name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[ - ]), - Field(item=10, name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[ - ]), - Field(item=11, name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[ - ]), - Field(item=12, name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[ - ]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ - ]), - Field(item=14, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, required=True, validators=[ - ]), - Field(item=15, name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, required=True, validators=[ - ]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[ - ]), - Field(item=17, name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[ - ]), - Field(item=18, name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, required=True, validators=[ - ]), - Field(item=19, name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[ - ]), - Field(item=20, name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, required=True, validators=[ - ]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, required=True, validators=[ - ]) - ] -) - -second_part_schema = RowSchema( - model=SSP_M3, - quiet_preparser_errors=True, - preparsing_validators=[ - # validators.hasLength(150), # unreliable. - validators.notEmpty(start=60, end=101), - ], - postparsing_validators=[], - fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ - ]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ - ]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ - ]), - # Field(item=1, name='FIPS_CODE', type='string', startIndex=8, endIndex=19, required=True, validators=[ - # ]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[ - ]), - Field(item=23, name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, required=True, validators=[ - ]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[ - ]), - Field(item=25, name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[ - ]), - Field(item=26, name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[ - ]), - Field(item=27, name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[ - ]), - Field(item=28, name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[ - ]), - Field(item=29, name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[ - ]), - Field(item=30, name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[ - ]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ - ]), - Field(item=32, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, required=True, validators=[ - ]), - Field(item=33, name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, required=True, validators=[ - ]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[ - ]), - Field(item=35, name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[ - ]), - Field(item=36, name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, required=True, validators=[ - ]), - Field(item=37, name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[ - ]), - Field(item=38, name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, required=True, validators=[ - ]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, required=True, validators=[ - ]) - ] -) - -m3 = MultiRecordRowSchema( - schemas=[ - first_part_schema, - second_part_schema - ] -) - - -# m3 = RowSchema( -# model=SSP_M3, -# preparsing_validators=[ -# validators.hasLength(150), # ?? the format document shows double fields/length? -# ], -# postparsing_validators=[], -# fields=[ -# Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ -# ]), -# Field(item=1, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ -# ]), -# Field(item=1, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ -# ]), -# # Field(item=1, name='FIPS_CODE', type='string', startIndex=8, endIndex=19, required=True, validators=[ -# # ]), -# Field(item=1, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[ -# ]), -# Field(item=1, name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, required=True, validators=[ -# ]), -# Field(item=1, name='SSN', type='number', startIndex=28, endIndex=37, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[ -# ]), -# Field(item=1, name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[ -# ]), -# Field(item=1, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, required=True, validators=[ -# ]), -# Field(item=1, name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, required=True, validators=[ -# ]), -# Field(item=1, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[ -# ]), -# Field(item=1, name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[ -# ]), -# Field(item=1, name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, required=True, validators=[ -# ]), -# Field(item=1, name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[ -# ]), -# Field(item=1, name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, required=True, validators=[ -# ]), -# Field(item=1, name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, required=True, validators=[ -# ]), - -# # ?? -# Field(item=1, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[ -# ]), -# Field(item=1, name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, required=True, validators=[ -# ]), -# Field(item=1, name='SSN', type='number', startIndex=69, endIndex=78, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[ -# ]), -# Field(item=1, name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[ -# ]), -# Field(item=1, name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[ -# ]), -# Field(item=1, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, required=True, validators=[ -# ]), -# Field(item=1, name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, required=True, validators=[ -# ]), -# Field(item=1, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[ -# ]), -# Field(item=1, name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[ -# ]), -# Field(item=1, name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, required=True, validators=[ -# ]), -# Field(item=1, name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[ -# ]), -# Field(item=1, name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, required=True, validators=[ -# ]), -# Field(item=1, name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, required=True, validators=[ -# ]), -# # Field(item=1, name='BLANK', type='string', startIndex=101, endIndex=150, required=False, validators=[]), -# ], -# ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index e02930dc4..cea1ce3d2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -13,96 +13,96 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ + Field(name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ ]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ + Field(name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ ]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ + Field(name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ ]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[ + Field(name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[ ]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[ + Field(name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[ ]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ + Field(name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ ]), - Field(item=7, name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[ + Field(name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[ ]), - Field(item=8, name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[ + Field(name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[ ]), - Field(item=9, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[ + Field(name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[ ]), - Field(item=10, name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[ + Field(name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[ ]), - Field(item=11, name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[ + Field(name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[ ]), - Field(item=12, name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[ + Field(name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[ ]), - Field(item=13, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[ + Field(name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[ ]), - Field(item=14, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[ + Field(name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[ ]), - Field(item=15, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ + Field(name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ ]), - Field(item=16, name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[ + Field(name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[ ]), - Field(item=17, name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ + Field(name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ ]), - Field(item=18, name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[ + Field(name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[ ]), - Field(item=19, name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[ + Field(name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[ ]), - Field(item=20, name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[ + Field(name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[ ]), - Field(item=21, name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[ + Field(name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[ ]), - Field(item=22, name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[ + Field(name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[ ]), - Field(item=23, name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[ + Field(name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[ ]), - Field(item=24, name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[ + Field(name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[ ]), - Field(item=25, name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[ + Field(name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[ ]), - Field(item=26, name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[ + Field(name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[ ]), - Field(item=27, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[ + Field(name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[ ]), - Field(item=28, name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[ + Field(name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[ ]), - Field(item=29, name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[ + Field(name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[ ]), - Field(item=30, name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[ + Field(name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[ ]), - Field(item=31, name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ + Field(name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ ]), - Field(item=32, name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[ + Field(name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[ ]), - Field(item=33, name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[ + Field(name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[ ]), - Field(item=34, name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ + Field(name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ ]), - Field(item=35, name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[ + Field(name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[ ]), - Field(item=36, name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[ + Field(name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[ ]), - Field(item=37, name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[ + Field(name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[ ]), - Field(item=38, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[ + Field(name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[ ]), - Field(item=39, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[ + Field(name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[ ]), - Field(item=40, name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[ + Field(name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[ ]), - Field(item=41, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[ + Field(name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[ ]), - Field(item=42, name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[ + Field(name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[ ]), - Field(item=43, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[ + Field(name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[ ]), - Field(item=44, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ + Field(name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ ]), - Field(item=45, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ + Field(name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ ]), - Field(item=46, name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), + Field(name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 625b2042e..b1b912c0e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -13,13 +13,13 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ + Field(name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ validators.matches('TRAILER') ]), - Field(item=2, name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ + Field(name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ validators.between(0, 9999999) ]), - Field(item=3, name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ + Field(name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ validators.matches(' ') ]), ], diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index 075f3eb52..a5ee1cc06 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -14,10 +14,6 @@ def failing_validator(): return lambda _: (False, 'Value is not valid.') -def error_func(schema, error_category, error_message, record, field): - return error_message - - def test_run_preparsing_validators_returns_valid(): """Test run_preparsing_validators executes all preparsing_validators provided in schema.""" line = '12345' @@ -27,7 +23,7 @@ def test_run_preparsing_validators_returns_valid(): ] ) - is_valid, errors = schema.run_preparsing_validators(line, error_func) + is_valid, errors = schema.run_preparsing_validators(line) assert is_valid is True assert errors == [] @@ -42,7 +38,7 @@ def test_run_preparsing_validators_returns_invalid_and_errors(): ] ) - is_valid, errors = schema.run_preparsing_validators(line, error_func) + is_valid, errors = schema.run_preparsing_validators(line) assert is_valid is False assert errors == ['Value is not valid.'] @@ -53,17 +49,9 @@ def test_parse_line_parses_line_from_schema_to_dict(): schema = RowSchema( model=dict, fields=[ -<<<<<<< HEAD Field(name='first', type='string', startIndex=0, endIndex=3), Field(name='second', type='string', startIndex=3, endIndex=4), Field(name='third', type='string', startIndex=4, endIndex=5), -======= - Field(item=1, name='first', type='string', startIndex=0, endIndex=3), - Field(item=2, name='second', type='string', startIndex=3, endIndex=4), - Field(item=3, name='third', type='string', startIndex=4, endIndex=5), - Field(item=4, name='fourth', type='number', startIndex=5, endIndex=7), - Field(item=5, name='fifth', type='number', startIndex=7, endIndex=8), ->>>>>>> 6772b8e6 (wip parser error generator) ] ) @@ -85,17 +73,9 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ -<<<<<<< HEAD Field(name='first', type='string', startIndex=0, endIndex=3), Field(name='second', type='string', startIndex=3, endIndex=4), Field(name='third', type='string', startIndex=4, endIndex=5), -======= - Field(item=1, name='first', type='string', startIndex=0, endIndex=3), - Field(item=2, name='second', type='string', startIndex=3, endIndex=4), - Field(item=3, name='third', type='string', startIndex=4, endIndex=5), - Field(item=4, name='fourth', type='number', startIndex=5, endIndex=7), - Field(item=5, name='fifth', type='number', startIndex=7, endIndex=8), ->>>>>>> 6772b8e6 (wip parser error generator) ] ) @@ -116,19 +96,19 @@ def test_run_field_validators_returns_valid_with_dict(): schema = RowSchema( model=dict, fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator() ]), - Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance, error_func) + is_valid, errors = schema.run_field_validators(instance) assert is_valid is True assert errors == [] @@ -148,19 +128,19 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator() ]), - Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance, error_func) + is_valid, errors = schema.run_field_validators(instance) assert is_valid is True assert errors == [] @@ -175,20 +155,20 @@ def test_run_field_validators_returns_invalid_with_dict(): schema = RowSchema( model=dict, fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator(), failing_validator() ]), - Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance, error_func) + is_valid, errors = schema.run_field_validators(instance) assert is_valid is False assert errors == ['Value is not valid.'] @@ -208,20 +188,20 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator(), failing_validator() ]), - Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance, error_func) + is_valid, errors = schema.run_field_validators(instance) assert is_valid is False assert errors == ['Value is not valid.'] @@ -235,16 +215,16 @@ def test_field_validators_blank_and_required_returns_error(): schema = RowSchema( model=dict, fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=1, required=True, validators=[ + Field(name='first', type='string', startIndex=0, endIndex=1, required=True, validators=[ passing_validator(), ]), - Field(item=2, name='second', type='string', startIndex=1, endIndex=3, required=True, validators=[ + Field(name='second', type='string', startIndex=1, endIndex=3, required=True, validators=[ passing_validator(), ]), ] ) - is_valid, errors = schema.run_field_validators(instance, error_func) + is_valid, errors = schema.run_field_validators(instance) assert is_valid is False assert errors == [ 'first is required but a value was not provided.', @@ -260,14 +240,14 @@ def test_field_validators_blank_and_not_required_returns_valid(): schema = RowSchema( model=dict, fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[ + Field(name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[ passing_validator(), failing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance, error_func) + is_valid, errors = schema.run_field_validators(instance) assert is_valid is True assert errors == [] @@ -281,7 +261,7 @@ def test_run_postparsing_validators_returns_valid(): ] ) - is_valid, errors = schema.run_postparsing_validators(instance, error_func) + is_valid, errors = schema.run_postparsing_validators(instance) assert is_valid is True assert errors == [] @@ -296,121 +276,6 @@ def test_run_postparsing_validators_returns_invalid_and_errors(): ] ) - is_valid, errors = schema.run_postparsing_validators(instance, error_func) + is_valid, errors = schema.run_postparsing_validators(instance) assert is_valid is False assert errors == ['Value is not valid.'] -<<<<<<< HEAD -======= - - -@pytest.mark.parametrize("value,length", [ - (None, 0), - (None, 10), - (' ', 5), - ('###', 3) -]) -def test_value_is_empty_returns_true(value, length): - result = value_is_empty(value, length) - assert result is True - - -@pytest.mark.parametrize("value,length", [ - (0, 1), - (1, 1), - (10, 2), - ('0', 1), - ('0000', 4), - ('1 ', 5), - ('##3', 3) -]) -def test_value_is_empty_returns_false(value, length): - result = value_is_empty(value, length) - assert result is False - - -def test_multi_record_schema_parses_and_validates(): - line = '12345' - schema = MultiRecordRowSchema( - schemas=[ - RowSchema( - model=dict, - preparsing_validators=[ - passing_validator() - ], - postparsing_validators=[ - failing_validator() - ], - fields=[ - Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ - passing_validator() - ]), - ] - ), - RowSchema( - model=dict, - preparsing_validators=[ - passing_validator() - ], - postparsing_validators=[ - passing_validator() - ], - fields=[ - Field(item=2, name='second', type='string', startIndex=2, endIndex=4, validators=[ - passing_validator() - ]), - ] - ), - RowSchema( - model=dict, - preparsing_validators=[ - failing_validator() - ], - postparsing_validators=[ - passing_validator() - ], - fields=[ - Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ - passing_validator() - ]), - ] - ), - RowSchema( - model=dict, - preparsing_validators=[ - passing_validator() - ], - postparsing_validators=[ - passing_validator() - ], - fields=[ - Field(item=4, name='fourth', type='string', startIndex=4, endIndex=5, validators=[ - failing_validator() - ]), - ] - ) - ] - ) - - rs = schema.parse_and_validate(line, error_func) - - r0_record, r0_is_valid, r0_errors = rs[0] - r1_record, r1_is_valid, r1_errors = rs[1] - r2_record, r2_is_valid, r2_errors = rs[2] - r3_record, r3_is_valid, r3_errors = rs[3] - - assert r0_record == {'first': '123'} - assert r0_is_valid is False - assert r0_errors == ['Value is not valid.'] - - assert r1_record == {'second': '34'} - assert r1_is_valid is True - assert r1_errors == [] - - assert r2_record is None - assert r2_is_valid is False - assert r2_errors == ['Value is not valid.'] - - assert r3_record == {'fourth': '5'} - assert r3_is_valid is False - assert r3_errors == ['Value is not valid.'] ->>>>>>> 6772b8e6 (wip parser error generator) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index c125eed07..7a1bcc2f6 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -1,77 +1,12 @@ """Utility file for functions shared between all parsers even preparser.""" -<<<<<<< HEAD import logging logger = logging.getLogger(__name__) -======= -from .models import ParserError -from django.contrib.contenttypes.models import ContentType -from tdpservice.search_indexes.models.ssp import SSP_M1 - -def value_is_empty(value, length): - """Handle 'empty' values as field inputs.""" - - empty_values = [ - ' '*length, # ' ' - '#'*length, # '#####' - ] - - return value is None or value in empty_values - ->>>>>>> 6772b8e6 (wip parser error generator) - -error_types = { - 1: 'File pre-check', - 2: 'Record value invalid', - 3: 'Record value consistency', - 4: 'Case consistency', - 5: 'Section consistency', - 6: 'Historical consistency' -} - -# def generate_parser_error(datafile, schema, record, line_number, field, error_category, error_msg): -# return ParserError.objects.create( -# file=datafile, -# row_number=line_number, -# column_number=field.item_number, -# field_name=field.name, -# category=error_category, -# case_number=getattr(record, 'CASE_NUMBER', None), -# error_message=error_msg, -# error_type=error_types.get(error_category, None), -# content_type=schema.model, -# object_id=record.pk, -# fields_json=None -# ) - - -def make_generate_parser_error(datafile, line_number): - def generate_parser_error(schema, error_category, error_message, record=None, field=None): - model = schema.model if schema else None - return ParserError.objects.create( - file=datafile, - row_number=line_number, - column_number=getattr(field, 'item', 0), - item_number=getattr(field, 'item', 0), - field_name=getattr(field, 'name', 'none'), - category=error_category, - case_number=getattr(record, 'CASE_NUMBER', None), - error_message=error_message, - error_type=error_types.get(error_category, None), - content_type=ContentType.objects.get( - model=model if record and not isinstance(record, dict) else 'ssp_m1' - ), - object_id=getattr(record, 'pk', 0) if record and not isinstance(record, dict) else 0, - fields_json=None - ) - - return generate_parser_error class Field: """Provides a mapping between a field name and its position.""" - def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]): - self.item = item + def __init__(self, name, type, startIndex, endIndex, required=True, validators=[]): self.name = name self.type = type self.startIndex = startIndex @@ -79,9 +14,9 @@ def __init__(self, item, name, type, startIndex, endIndex, required=True, valida self.required = required self.validators = validators - def create(self, item, name, length, start, end, type): + def create(self, name, length, start, end, type): """Create a new field.""" - return Field(item, name, type, length, start, end) + return Field(name, type, length, start, end) def __repr__(self): """Return a string representation of the field.""" @@ -108,10 +43,10 @@ def __init__(self, model=dict, preparsing_validators=[], postparsing_validators= self.fields = fields # self.section = section # intended for future use with multiple section objects - def _add_field(self, item, name, length, start, end, type): + def _add_field(self, name, length, start, end, type): """Add a field to the schema.""" self.fields.append( - Field(item, name, type, start, end) + Field(name, type, start, end) ) def add_fields(self, fields: list): @@ -123,12 +58,12 @@ def get_all_fields(self): """Get all fields from the schema.""" return self.fields - def parse_and_validate(self, line, error_func): + def parse_and_validate(self, line): """Run all validation steps in order, and parse the given line into a record.""" errors = [] # run preparsing validators - preparsing_is_valid, preparsing_errors = self.run_preparsing_validators(line, error_func) + preparsing_is_valid, preparsing_errors = self.run_preparsing_validators(line) if not preparsing_is_valid: return None, False, preparsing_errors @@ -137,17 +72,17 @@ def parse_and_validate(self, line, error_func): record = self.parse_line(line) # run field validators - fields_are_valid, field_errors = self.run_field_validators(record, error_func) + fields_are_valid, field_errors = self.run_field_validators(record) # run postparsing validators - postparsing_is_valid, postparsing_errors = self.run_postparsing_validators(record, error_func) + postparsing_is_valid, postparsing_errors = self.run_postparsing_validators(record) is_valid = fields_are_valid and postparsing_is_valid errors = field_errors + postparsing_errors return record, is_valid, errors - def run_preparsing_validators(self, line, error_func): + def run_preparsing_validators(self, line): """Run each of the `preparsing_validator` functions in the schema against the un-parsed line.""" is_valid = True errors = [] @@ -156,15 +91,7 @@ def run_preparsing_validators(self, line, error_func): validator_is_valid, validator_error = validator(line) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append( - error_func( - schema=self, - error_category=1, - error_message=validator_error, - record=None, - field=None - ) - ) + errors.append(validator_error) return is_valid, errors @@ -180,12 +107,9 @@ def parse_line(self, line): else: setattr(record, field.name, value) - # if not isinstance(record, dict): - # record.save() - return record - def run_field_validators(self, instance, error_func): + def run_field_validators(self, instance): """Run all validators for each field in the parsed model.""" is_valid = True errors = [] @@ -202,30 +126,14 @@ def run_field_validators(self, instance, error_func): validator_is_valid, validator_error = validator(value) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append( - error_func( - schema=self, - error_category=2, - error_message=validator_error, - record=instance, - field=field - ) - ) + errors.append(validator_error) elif field.required: is_valid = False - errors.append( - error_func( - schema=self, - error_category=2, - error_message=f"{field.name} is required but a value was not provided.", - record=instance, - field=field - ) - ) + errors.append(f"{field.name} is required but a value was not provided.") return is_valid, errors - def run_postparsing_validators(self, instance, error_func): + def run_postparsing_validators(self, instance): """Run each of the `postparsing_validator` functions against the parsed model.""" is_valid = True errors = [] @@ -234,35 +142,6 @@ def run_postparsing_validators(self, instance, error_func): validator_is_valid, validator_error = validator(instance) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append( - error_func( - schema=self, - error_category=3, - error_message=validator_error, - record=instance, - field=None - ) - ) + errors.append(validator_error) return is_valid, errors -<<<<<<< HEAD -======= - - -class MultiRecordRowSchema: - """Maps a line to multiple `RowSchema`s and runs all parsers and validators.""" - - def __init__(self, schemas): - # self.common_fields = None - self.schemas = schemas - - def parse_and_validate(self, line, error_func): - """Run `parse_and_validate` for each schema provided and bubble up errors.""" - records = [] - - for schema in self.schemas: - r = schema.parse_and_validate(line, error_func) - records.append(r) - - return records ->>>>>>> 6772b8e6 (wip parser error generator) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 7b7d41b1d..cedaf50b0 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -60,7 +60,7 @@ def startsWith(substring): # custom validators -def validate_single_header_trailer(file, error_func): +def validate_single_header_trailer(file): """Validate that a raw datafile has one trailer and one footer.""" headers = 0 trailers = 0 From 744ba4d3bc4ee3c7706693e35ac7c46c7efb4e63 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 26 Apr 2023 14:39:54 -0400 Subject: [PATCH 008/120] update to test to use dfs fixture --- tdrs-backend/tdpservice/parsers/test/test_summary.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index dd0619bd1..4033c4c4e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -17,7 +17,7 @@ def dfs(): @pytest.mark.django_db def test_dfs_model(dfs): """Test that the model is created and populated correctly.""" - dfs = DataFileSummaryFactory() + dfs = dfs assert dfs.case_aggregates['Jan']['accepted'] == 100 @@ -31,9 +31,9 @@ def test_dfs_rejected(test_datafile, dfs): assert dfs.status == DataFileSummary.Status.REJECTED @pytest.mark.django_db -def test_dfs_set_status(): +def test_dfs_set_status(dfs): """Test that the status is set correctly.""" - dfs = DataFileSummaryFactory() + dfs = dfs assert dfs.status == DataFileSummary.Status.PENDING dfs.set_status(errors={}) assert dfs.status == DataFileSummary.Status.ACCEPTED From 6bb20177ef128154e9fd088918ef77cd11e8cc43 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 27 Apr 2023 14:45:45 -0400 Subject: [PATCH 009/120] saving state before new 1610 merge --- tdrs-backend/tdpservice/parsers/admin.py | 16 ++ tdrs-backend/tdpservice/parsers/parse.py | 109 ++++++++++- .../tdpservice/parsers/schema_defs/header.py | 20 +- .../tdpservice/parsers/schema_defs/tanf/t1.py | 92 ++++----- .../tdpservice/parsers/schema_defs/trailer.py | 6 +- .../tdpservice/parsers/test/test_summary.py | 4 +- .../tdpservice/parsers/test/test_util.py | 184 +++++++++++++++--- tdrs-backend/tdpservice/parsers/util.py | 151 ++++++++++++-- tdrs-backend/tdpservice/parsers/validators.py | 2 +- 9 files changed, 464 insertions(+), 120 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/admin.py b/tdrs-backend/tdpservice/parsers/admin.py index 3d785191b..3ac3b444b 100644 --- a/tdrs-backend/tdpservice/parsers/admin.py +++ b/tdrs-backend/tdpservice/parsers/admin.py @@ -1,3 +1,19 @@ """Django admin customizations for the parser models.""" +from django.contrib import admin +from . import models + # Register your models here. +class ParserErrorAdmin(admin.ModelAdmin): + """ModelAdmin class for parsed M1 data files.""" + + list_display = [ + 'row_number', + 'field_name', + 'category', + 'error_type', + 'error_message', + ] + + +admin.site.register(models.ParserError, ParserErrorAdmin) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 812047e95..a49ea5508 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -1,11 +1,11 @@ """Convert raw uploaded Datafile into a parsed model, and accumulate/return any errors.""" - import os from . import schema_defs from . import validators +from . import schema_defs, validators, util from tdpservice.data_files.models import DataFile -from .models import DataFileSummary +from .models import DataFileSummary, ParserError def parse_datafile(datafile): @@ -13,7 +13,10 @@ def parse_datafile(datafile): rawfile = datafile.file errors = {} - document_is_valid, document_error = validators.validate_single_header_trailer(rawfile) + document_is_valid, document_error = validators.validate_single_header_trailer( + rawfile, + util.make_generate_parser_error(datafile, 1) + ) if not document_is_valid: errors['document'] = [document_error] return errors @@ -31,12 +34,18 @@ def parse_datafile(datafile): trailer_line = rawfile.readline().decode().strip('\n') # parse header, trailer - header, header_is_valid, header_errors = schema_defs.header.parse_and_validate(header_line) + header, header_is_valid, header_errors = schema_defs.header.parse_and_validate( + header_line, + util.make_generate_parser_error(datafile, 1) + ) if not header_is_valid: errors['header'] = header_errors return errors - trailer, trailer_is_valid, trailer_errors = schema_defs.trailer.parse_and_validate(trailer_line) + trailer, trailer_is_valid, trailer_errors = schema_defs.trailer.parse_and_validate( + trailer_line, + util.make_generate_parser_error(datafile, -1) + ) if not trailer_is_valid: errors['trailer'] = trailer_errors @@ -60,7 +69,22 @@ def parse_datafile(datafile): section = header['type'] if datafile.section != section_names.get(program_type, {}).get(section): - errors['document'] = ['Section does not match.'] + # error_func call + errors['document'] = \ + util.generate_parser_error( + datafile=datafile, + row_number=1, + column_number=schema_defs.header.get_field('type').startIndex, + item_number=0, + field_name='type', + category=1, + error_type=util.error_types.get(1, None), + error_message=f"Section '{section}' does not match upload section '{datafile.section}'", + content_type=None, + object_id=None, + fields_json=None + ) + return errors # parse line with appropriate schema @@ -78,8 +102,29 @@ def parse_datafile(datafile): schema = get_schema(line, section, schema_options) record_is_valid, record_errors = parse_datafile_line(line, schema) - if not record_is_valid: - errors[line_number] = record_errors + if isinstance(schema, util.MultiRecordRowSchema): + records = parse_multi_record_line( + line, + schema, + util.make_generate_parser_error(datafile, line_number) + ) + + n = 0 + for r in records: + n += 1 + record, record_is_valid, record_errors = r + if not record_is_valid: + line_errors = errors.get(line_number, {}) + line_errors[n] = record_errors + errors[line_number] = line_errors + else: + record_is_valid, record_errors = parse_datafile_line( + line, + schema, + util.make_generate_parser_error(datafile, line_number) + ) + if not record_is_valid: + errors[line_number] = record_errors summary = DataFileSummary(datafile=datafile) summary.set_status(errors) @@ -87,16 +132,60 @@ def parse_datafile(datafile): return summary, errors +def parse_multi_record_line(line, schema, error_func): + if schema: + records = schema.parse_and_validate(line, error_func) + + for r in records: + record, record_is_valid, record_errors = r + + if record: + record.save() -def parse_datafile_line(line, schema): + # for error_msg in record_errors: + # error_obj = ParserError.objects.create( + # file=None, + # row_number=None, + # column_number=None, + # field_name=None, + # category=None, + # case_number=getattr(record, 'CASE_NUMBER', None), + # error_message=error_msg, + # error_type=None, + # content_type=schema.model, + # object_id=record.pk, + # fields_json=None + # ) + + return records + + return [(None, False, ['No schema selected.'])] + + +def parse_datafile_line(line, schema, error_func): """Parse and validate a datafile line and save any errors to the model.""" if schema: - record, record_is_valid, record_errors = schema.parse_and_validate(line) + record, record_is_valid, record_errors = schema.parse_and_validate(line, error_func) if record: record.errors = record_errors record.save() + # for error_msg in record_errors: + # error_obj = ParserError.objects.create( + # file=None, + # row_number=None, + # column_number=None, + # field_name=None, + # category=None, + # case_number=None, + # error_message=error_msg, + # error_type=None, + # content_type=schema.model, + # object_id=record.pk, + # fields_json=None + # ) + return record_is_valid, record_errors return (False, ['No schema selected.']) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index b6af3e713..c2b0b55c0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -13,34 +13,34 @@ ], postparsing_validators=[], fields=[ - Field(name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ + Field(item=1, name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ validators.matches('HEADER'), ]), - Field(name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ + Field(item=2, name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ validators.between(2000, 2099) ]), - Field(name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ + Field(item=3, name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ validators.oneOf(['1', '2', '3', '4']) ]), - Field(name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ + Field(item=4, name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ validators.oneOf(['A', 'C', 'G', 'S']) ]), - Field(name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ + Field(item=5, name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ validators.between(0, 99) ]), - Field(name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ + Field(item=6, name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ validators.between(0, 999) ]), - Field(name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ + Field(item=7, name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ validators.oneOf(['TAN', 'SSP']) ]), - Field(name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ + Field(item=8, name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ validators.oneOf(['1', '2']) ]), - Field(name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ + Field(item=9, name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ validators.matches('E') ]), - Field(name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ + Field(item=10, name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ validators.oneOf(['N', 'D', 'U']) ]), ], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index cea1ce3d2..e02930dc4 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -13,96 +13,96 @@ ], postparsing_validators=[], fields=[ - Field(name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ + Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[ ]), - Field(name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ + Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[ ]), - Field(name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ + Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[ ]), - Field(name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[ + Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[ ]), - Field(name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[ + Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[ ]), - Field(name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ + Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[ ]), - Field(name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[ + Field(item=7, name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[ ]), - Field(name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[ + Field(item=8, name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[ ]), - Field(name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[ + Field(item=9, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[ ]), - Field(name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[ + Field(item=10, name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[ ]), - Field(name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[ + Field(item=11, name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[ ]), - Field(name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[ + Field(item=12, name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[ ]), - Field(name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[ + Field(item=13, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[ ]), - Field(name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[ + Field(item=14, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[ ]), - Field(name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ + Field(item=15, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[ ]), - Field(name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[ + Field(item=16, name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[ ]), - Field(name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ + Field(item=17, name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[ ]), - Field(name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[ + Field(item=18, name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[ ]), - Field(name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[ + Field(item=19, name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[ ]), - Field(name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[ + Field(item=20, name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[ ]), - Field(name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[ + Field(item=21, name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[ ]), - Field(name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[ + Field(item=22, name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[ ]), - Field(name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[ + Field(item=23, name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[ ]), - Field(name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[ + Field(item=24, name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[ ]), - Field(name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[ + Field(item=25, name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[ ]), - Field(name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[ + Field(item=26, name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[ ]), - Field(name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[ + Field(item=27, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[ ]), - Field(name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[ + Field(item=28, name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[ ]), - Field(name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[ + Field(item=29, name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[ ]), - Field(name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[ + Field(item=30, name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[ ]), - Field(name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ + Field(item=31, name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[ ]), - Field(name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[ + Field(item=32, name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[ ]), - Field(name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[ + Field(item=33, name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[ ]), - Field(name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ + Field(item=34, name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[ ]), - Field(name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[ + Field(item=35, name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[ ]), - Field(name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[ + Field(item=36, name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[ ]), - Field(name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[ + Field(item=37, name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[ ]), - Field(name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[ + Field(item=38, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[ ]), - Field(name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[ + Field(item=39, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[ ]), - Field(name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[ + Field(item=40, name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[ ]), - Field(name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[ + Field(item=41, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[ ]), - Field(name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[ + Field(item=42, name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[ ]), - Field(name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[ + Field(item=43, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[ ]), - Field(name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ + Field(item=44, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[ ]), - Field(name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ + Field(item=45, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[ ]), - Field(name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), + Field(item=46, name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index b1b912c0e..625b2042e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -13,13 +13,13 @@ ], postparsing_validators=[], fields=[ - Field(name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ + Field(item=1, name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ validators.matches('TRAILER') ]), - Field(name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ + Field(item=2, name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ validators.between(0, 9999999) ]), - Field(name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ + Field(item=3, name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ validators.matches(' ') ]), ], diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 4033c4c4e..a22379849 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -12,7 +12,7 @@ @pytest.fixture def dfs(): """Fixture for DataFileSummary.""" - return DataFileSummaryFactory.create() + return DataFileSummaryFactory() @pytest.mark.django_db def test_dfs_model(dfs): @@ -21,7 +21,7 @@ def test_dfs_model(dfs): assert dfs.case_aggregates['Jan']['accepted'] == 100 -@pytest.mark.django_db +@pytest.mark.django_db(transaction=True) def test_dfs_rejected(test_datafile, dfs): """Ensure that an invalid file generates a rejected status.""" dfs = dfs diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index a5ee1cc06..c7e8c1f28 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -1,6 +1,6 @@ """Test the methods of RowSchema to ensure parsing and validation work in all individual cases.""" - +import pytest from ..util import RowSchema, Field @@ -14,6 +14,10 @@ def failing_validator(): return lambda _: (False, 'Value is not valid.') +def error_func(schema, error_category, error_message, record, field): + return error_message + + def test_run_preparsing_validators_returns_valid(): """Test run_preparsing_validators executes all preparsing_validators provided in schema.""" line = '12345' @@ -23,7 +27,7 @@ def test_run_preparsing_validators_returns_valid(): ] ) - is_valid, errors = schema.run_preparsing_validators(line) + is_valid, errors = schema.run_preparsing_validators(line, error_func) assert is_valid is True assert errors == [] @@ -38,7 +42,7 @@ def test_run_preparsing_validators_returns_invalid_and_errors(): ] ) - is_valid, errors = schema.run_preparsing_validators(line) + is_valid, errors = schema.run_preparsing_validators(line, error_func) assert is_valid is False assert errors == ['Value is not valid.'] @@ -49,9 +53,11 @@ def test_parse_line_parses_line_from_schema_to_dict(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3), - Field(name='second', type='string', startIndex=3, endIndex=4), - Field(name='third', type='string', startIndex=4, endIndex=5), + Field(item=1, name='first', type='string', startIndex=0, endIndex=3), + Field(item=2, name='second', type='string', startIndex=3, endIndex=4), + Field(item=3, name='third', type='string', startIndex=4, endIndex=5), + Field(item=4, name='fourth', type='number', startIndex=5, endIndex=7), + Field(item=5, name='fifth', type='number', startIndex=7, endIndex=8), ] ) @@ -73,9 +79,11 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3), - Field(name='second', type='string', startIndex=3, endIndex=4), - Field(name='third', type='string', startIndex=4, endIndex=5), + Field(item=1, name='first', type='string', startIndex=0, endIndex=3), + Field(item=2, name='second', type='string', startIndex=3, endIndex=4), + Field(item=3, name='third', type='string', startIndex=4, endIndex=5), + Field(item=4, name='fourth', type='number', startIndex=5, endIndex=7), + Field(item=5, name='fifth', type='number', startIndex=7, endIndex=8), ] ) @@ -96,19 +104,19 @@ def test_run_field_validators_returns_valid_with_dict(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator() ]), - Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is True assert errors == [] @@ -128,19 +136,19 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator() ]), - Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is True assert errors == [] @@ -155,20 +163,20 @@ def test_run_field_validators_returns_invalid_with_dict(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator(), failing_validator() ]), - Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is False assert errors == ['Value is not valid.'] @@ -188,20 +196,20 @@ class TestModel: schema = RowSchema( model=TestModel, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ passing_validator(), failing_validator() ]), - Field(name='second', type='string', startIndex=3, endIndex=4, validators=[ + Field(item=2, name='second', type='string', startIndex=3, endIndex=4, validators=[ passing_validator() ]), - Field(name='third', type='string', startIndex=4, endIndex=5, validators=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ passing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is False assert errors == ['Value is not valid.'] @@ -215,16 +223,16 @@ def test_field_validators_blank_and_required_returns_error(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=1, required=True, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=1, required=True, validators=[ passing_validator(), ]), - Field(name='second', type='string', startIndex=1, endIndex=3, required=True, validators=[ + Field(item=2, name='second', type='string', startIndex=1, endIndex=3, required=True, validators=[ passing_validator(), ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is False assert errors == [ 'first is required but a value was not provided.', @@ -240,14 +248,14 @@ def test_field_validators_blank_and_not_required_returns_valid(): schema = RowSchema( model=dict, fields=[ - Field(name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, required=False, validators=[ passing_validator(), failing_validator() ]), ] ) - is_valid, errors = schema.run_field_validators(instance) + is_valid, errors = schema.run_field_validators(instance, error_func) assert is_valid is True assert errors == [] @@ -261,7 +269,7 @@ def test_run_postparsing_validators_returns_valid(): ] ) - is_valid, errors = schema.run_postparsing_validators(instance) + is_valid, errors = schema.run_postparsing_validators(instance, error_func) assert is_valid is True assert errors == [] @@ -276,6 +284,118 @@ def test_run_postparsing_validators_returns_invalid_and_errors(): ] ) - is_valid, errors = schema.run_postparsing_validators(instance) + is_valid, errors = schema.run_postparsing_validators(instance, error_func) assert is_valid is False assert errors == ['Value is not valid.'] + +@pytest.mark.parametrize("value,length", [ + (None, 0), + (None, 10), + (' ', 5), + ('###', 3) +]) +def test_value_is_empty_returns_true(value, length): + result = value_is_empty(value, length) + assert result is True + + +@pytest.mark.parametrize("value,length", [ + (0, 1), + (1, 1), + (10, 2), + ('0', 1), + ('0000', 4), + ('1 ', 5), + ('##3', 3) +]) +def test_value_is_empty_returns_false(value, length): + result = value_is_empty(value, length) + assert result is False + + +def test_multi_record_schema_parses_and_validates(): + line = '12345' + schema = MultiRecordRowSchema( + schemas=[ + RowSchema( + model=dict, + preparsing_validators=[ + passing_validator() + ], + postparsing_validators=[ + failing_validator() + ], + fields=[ + Field(item=1, name='first', type='string', startIndex=0, endIndex=3, validators=[ + passing_validator() + ]), + ] + ), + RowSchema( + model=dict, + preparsing_validators=[ + passing_validator() + ], + postparsing_validators=[ + passing_validator() + ], + fields=[ + Field(item=2, name='second', type='string', startIndex=2, endIndex=4, validators=[ + passing_validator() + ]), + ] + ), + RowSchema( + model=dict, + preparsing_validators=[ + failing_validator() + ], + postparsing_validators=[ + passing_validator() + ], + fields=[ + Field(item=3, name='third', type='string', startIndex=4, endIndex=5, validators=[ + passing_validator() + ]), + ] + ), + RowSchema( + model=dict, + preparsing_validators=[ + passing_validator() + ], + postparsing_validators=[ + passing_validator() + ], + fields=[ + Field(item=4, name='fourth', type='string', startIndex=4, endIndex=5, validators=[ + failing_validator() + ]), + ] + ) + ] + ) + + rs = schema.parse_and_validate(line, error_func) + + r0_record, r0_is_valid, r0_errors = rs[0] + r1_record, r1_is_valid, r1_errors = rs[1] + r2_record, r2_is_valid, r2_errors = rs[2] + r3_record, r3_is_valid, r3_errors = rs[3] + + assert r0_record == {'first': '123'} + assert r0_is_valid is False + assert r0_errors == ['Value is not valid.'] + + assert r1_record == {'second': '34'} + assert r1_is_valid is True + assert r1_errors == [] + + assert r2_record is None + assert r2_is_valid is False + assert r2_errors == ['Value is not valid.'] + + assert r3_record == {'fourth': '5'} + assert r3_is_valid is False + assert r3_errors == ['Value is not valid.'] + diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 7a1bcc2f6..00cae8458 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -1,12 +1,72 @@ """Utility file for functions shared between all parsers even preparser.""" +from .models import ParserError +from django.contrib.contenttypes.models import ContentType import logging logger = logging.getLogger(__name__) +def value_is_empty(value, length): + """Handle 'empty' values as field inputs.""" + + empty_values = [ + ' '*length, # ' ' + '#'*length, # '#####' + ] + + return value is None or value in empty_values + +error_types = { + 1: 'File pre-check', + 2: 'Record value invalid', + 3: 'Record value consistency', + 4: 'Case consistency', + 5: 'Section consistency', + 6: 'Historical consistency' +} + +def generate_parser_error(datafile, schema, record, line_number, field, error_category, error_msg): + return ParserError.objects.create( + file=datafile, + row_number=line_number, + column_number=field.item_number, + field_name=field.name, + category=error_category, + case_number=getattr(record, 'CASE_NUMBER', None), + error_message=error_msg, + error_type=error_types.get(error_category, None), + content_type=schema.model, + object_id=record.pk, + fields_json=None + ) + + +def make_generate_parser_error(datafile, line_number): + def generate_parser_error(schema, error_category, error_message, record=None, field=None): + model = schema.model if schema else None + return ParserError.objects.create( + file=datafile, + row_number=line_number, + column_number=getattr(field, 'item', 0), + item_number=getattr(field, 'item', 0), + field_name=getattr(field, 'name', 'none'), + category=error_category, + case_number=getattr(record, 'CASE_NUMBER', None), + error_message=error_message, + error_type=error_types.get(error_category, None), + content_type=ContentType.objects.get( + model=model if record and not isinstance(record, dict) else 'ssp_m1' + ), + object_id=getattr(record, 'pk', 0) if record and not isinstance(record, dict) else 0, + fields_json=None + ) + + return generate_parser_error + class Field: """Provides a mapping between a field name and its position.""" - def __init__(self, name, type, startIndex, endIndex, required=True, validators=[]): + def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]): + self.item = item self.name = name self.type = type self.startIndex = startIndex @@ -14,9 +74,9 @@ def __init__(self, name, type, startIndex, endIndex, required=True, validators=[ self.required = required self.validators = validators - def create(self, name, length, start, end, type): + def create(self, item, name, length, start, end, type): """Create a new field.""" - return Field(name, type, length, start, end) + return Field(item, name, type, length, start, end) def __repr__(self): """Return a string representation of the field.""" @@ -43,10 +103,10 @@ def __init__(self, model=dict, preparsing_validators=[], postparsing_validators= self.fields = fields # self.section = section # intended for future use with multiple section objects - def _add_field(self, name, length, start, end, type): + def _add_field(self, item, name, length, start, end, type): """Add a field to the schema.""" self.fields.append( - Field(name, type, start, end) + Field(item, name, type, start, end) ) def add_fields(self, fields: list): @@ -57,13 +117,19 @@ def add_fields(self, fields: list): def get_all_fields(self): """Get all fields from the schema.""" return self.fields + + def get_field(self, name): + """Get a field from the schema by name.""" + for field in self.fields: + if field.name == name: + return field - def parse_and_validate(self, line): + def parse_and_validate(self, line, error_func): """Run all validation steps in order, and parse the given line into a record.""" errors = [] # run preparsing validators - preparsing_is_valid, preparsing_errors = self.run_preparsing_validators(line) + preparsing_is_valid, preparsing_errors = self.run_preparsing_validators(line, error_func) if not preparsing_is_valid: return None, False, preparsing_errors @@ -72,17 +138,17 @@ def parse_and_validate(self, line): record = self.parse_line(line) # run field validators - fields_are_valid, field_errors = self.run_field_validators(record) + fields_are_valid, field_errors = self.run_field_validators(record, error_func) # run postparsing validators - postparsing_is_valid, postparsing_errors = self.run_postparsing_validators(record) + postparsing_is_valid, postparsing_errors = self.run_postparsing_validators(record, error_func) is_valid = fields_are_valid and postparsing_is_valid errors = field_errors + postparsing_errors return record, is_valid, errors - def run_preparsing_validators(self, line): + def run_preparsing_validators(self, line, error_func): """Run each of the `preparsing_validator` functions in the schema against the un-parsed line.""" is_valid = True errors = [] @@ -91,7 +157,15 @@ def run_preparsing_validators(self, line): validator_is_valid, validator_error = validator(line) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append(validator_error) + errors.append( + error_func( + schema=self, + error_category=1, + error_message=validator_error, + record=None, + field=None + ) + ) return is_valid, errors @@ -107,9 +181,12 @@ def parse_line(self, line): else: setattr(record, field.name, value) + # if not isinstance(record, dict): + # record.save() + return record - def run_field_validators(self, instance): + def run_field_validators(self, instance, error_func): """Run all validators for each field in the parsed model.""" is_valid = True errors = [] @@ -126,14 +203,30 @@ def run_field_validators(self, instance): validator_is_valid, validator_error = validator(value) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append(validator_error) + errors.append( + error_func( + schema=self, + error_category=2, + error_message=validator_error, + record=instance, + field=field + ) + ) elif field.required: is_valid = False - errors.append(f"{field.name} is required but a value was not provided.") + errors.append( + error_func( + schema=self, + error_category=2, + error_message=f"{field.name} is required but a value was not provided.", + record=instance, + field=field + ) + ) return is_valid, errors - def run_postparsing_validators(self, instance): + def run_postparsing_validators(self, instance, error_func): """Run each of the `postparsing_validator` functions against the parsed model.""" is_valid = True errors = [] @@ -142,6 +235,32 @@ def run_postparsing_validators(self, instance): validator_is_valid, validator_error = validator(instance) is_valid = False if not validator_is_valid else is_valid if validator_error: - errors.append(validator_error) + errors.append( + error_func( + schema=self, + error_category=3, + error_message=validator_error, + record=instance, + field=None + ) + ) return is_valid, errors + +class MultiRecordRowSchema: + """Maps a line to multiple `RowSchema`s and runs all parsers and validators.""" + + def __init__(self, schemas): + # self.common_fields = None + self.schemas = schemas + + def parse_and_validate(self, line, error_func): + """Run `parse_and_validate` for each schema provided and bubble up errors.""" + records = [] + + for schema in self.schemas: + r = schema.parse_and_validate(line, error_func) + records.append(r) + + return records + diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index cedaf50b0..7b7d41b1d 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -60,7 +60,7 @@ def startsWith(substring): # custom validators -def validate_single_header_trailer(file): +def validate_single_header_trailer(file, error_func): """Validate that a raw datafile has one trailer and one footer.""" headers = 0 trailers = 0 From 1316a70ef90a5d7d9932070d12d894b77704ad54 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 27 Apr 2023 16:00:42 -0400 Subject: [PATCH 010/120] Resolving merge conflicts with 1610. --- .../0002_alter_parsererror_error_type.py | 9 ++++ .../migrations/0002_datafilesummary.py | 24 ---------- tdrs-backend/tdpservice/parsers/models.py | 6 +-- tdrs-backend/tdpservice/parsers/parse.py | 10 ++--- .../tdpservice/parsers/test/factories.py | 2 +- .../tdpservice/parsers/test/test_summary.py | 10 +++-- tdrs-backend/tdpservice/parsers/util.py | 44 +++++++++---------- tdrs-backend/tdpservice/parsers/validators.py | 3 +- 8 files changed, 46 insertions(+), 62 deletions(-) delete mode 100644 tdrs-backend/tdpservice/parsers/migrations/0002_datafilesummary.py diff --git a/tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py b/tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py index 5236b5c29..72a8224af 100644 --- a/tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py +++ b/tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py @@ -15,4 +15,13 @@ class Migration(migrations.Migration): name='error_type', field=models.TextField(choices=[('1', 'File pre-check'), ('2', 'Record value invalid'), ('3', 'Record value consistency'), ('4', 'Case consistency'), ('5', 'Section consistency'), ('6', 'Historical consistency')], max_length=128), ), + migrations.CreateModel( + name='DataFileSummary', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('Pending', 'Pending'), ('Accepted', 'Accepted'), ('Accepted with Errors', 'Accepted With Errors'), ('Rejected', 'Rejected')], default='Pending', max_length=50)), + ('case_aggregates', models.JSONField(null=True)), + ('datafile', models.ForeignKey(on_delete=models.deletion.CASCADE, to='data_files.datafile')), + ], + ), ] diff --git a/tdrs-backend/tdpservice/parsers/migrations/0002_datafilesummary.py b/tdrs-backend/tdpservice/parsers/migrations/0002_datafilesummary.py deleted file mode 100644 index 4f401770f..000000000 --- a/tdrs-backend/tdpservice/parsers/migrations/0002_datafilesummary.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.2.15 on 2023-03-22 18:04 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('data_files', '0012_datafile_s3_versioning_id'), - ('parsers', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='DataFileSummary', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('status', models.CharField(choices=[('Pending', 'Pending'), ('Accepted', 'Accepted'), ('Accepted with Errors', 'Accepted With Errors'), ('Rejected', 'Rejected')], default='Pending', max_length=50)), - ('case_aggregates', models.JSONField(null=True)), - ('datafile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data_files.datafile')), - ], - ), - ] diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 713ef7cb7..ffe8be4c7 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -7,7 +7,6 @@ from django.contrib.contenttypes.models import ContentType from tdpservice.data_files.models import DataFile - class ParserErrorCategoryChoices(models.TextChoices): """Enum of ParserError error_type.""" PRE_CHECK = "1", _("File pre-check") @@ -132,6 +131,7 @@ def check_for_preparsing(errors): """Check for pre-parsing errors.""" for error in errors['document']: print(error) - if error.error_type == "pre-parsing": + print(error.category) + if error.category == "1": return True - return False \ No newline at end of file + return False diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 527780429..3cb74acc2 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -1,11 +1,9 @@ """Convert raw uploaded Datafile into a parsed model, and accumulate/return any errors.""" import os -from . import schema_defs -from . import validators from . import schema_defs, validators, util from tdpservice.data_files.models import DataFile -from .models import DataFileSummary, ParserError +from .models import DataFileSummary def parse_datafile(datafile): @@ -111,14 +109,14 @@ def parse_datafile(datafile): util.make_generate_parser_error(datafile, line_number) ) - if not record_is_valid: - errors[line_number] = record_errors + if not record_is_valid: + errors[line_number] = record_errors summary = DataFileSummary(datafile=datafile) summary.set_status(errors) summary.save() - return summary, errors + return errors def parse_multi_record_line(line, schema, error_func): if schema: diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 9858a5eed..7bbcda455 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -70,7 +70,7 @@ class Meta: column_number = 1 item_number = 1 field_name = "test field name" - category = 1 + category = "1" error_message = "test error message" error_type = "out of range" diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index a22379849..90a736bf3 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -4,7 +4,7 @@ from tdpservice.parsers import parse from tdpservice.parsers.models import DataFileSummary from .factories import DataFileSummaryFactory, ParserErrorFactory -from .test_parse import bad_file_missing_header, test_datafile +from .test_parse import test_datafile import logging logger = logging.getLogger(__name__) @@ -27,6 +27,7 @@ def test_dfs_rejected(test_datafile, dfs): dfs = dfs test_datafile.section = 'Closed Case Data' test_datafile.save() + dfs.set_status(parse.parse_datafile(test_datafile)) assert dfs.status == DataFileSummary.Status.REJECTED @@ -43,14 +44,15 @@ def test_dfs_set_status(dfs): # this feels precarious for tests. the defaults in the factory could cause issues should logic change # resulting in failures if we stop keying off category and instead go to content or msg - for i in range(1, 4): - parser_errors.append(ParserErrorFactory(row_number=(i), category=i)) + for i in range(2, 4): + parser_errors.append(ParserErrorFactory(row_number=i, category=str(i))) dfs.set_status(errors={'document': parser_errors}) assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - parser_errors.append(ParserErrorFactory(row_number=5, category=5, error_type='pre-parsing')) + print("about to add a category 1 error") + parser_errors.append(ParserErrorFactory(row_number=5, category="1")) dfs.set_status(errors={'document': parser_errors}) assert dfs.status == DataFileSummary.Status.REJECTED \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index b3c5fac97..595651df4 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -1,8 +1,6 @@ """Utility file for functions shared between all parsers even preparser.""" from .models import ParserError from django.contrib.contenttypes.models import ContentType -from tdpservice.search_indexes.models.ssp import SSP_M1 -import logging def value_is_empty(value, length): """Handle 'empty' values as field inputs.""" @@ -24,28 +22,29 @@ def value_is_empty(value, length): } def generate_parser_error(datafile, line_number, schema, error_category, error_message, record=None, field=None): - model = schema.model if schema else None - - # make fields optional - return ParserError.objects.create( - file=datafile, - row_number=line_number, - column_number=getattr(field, 'item', 0), - item_number=getattr(field, 'item', 0), - field_name=getattr(field, 'name', 'none'), - category=error_category, - case_number=getattr(record, 'CASE_NUMBER', None), - error_message=error_message, - error_type=error_category, - content_type=ContentType.objects.get( - model=model if record and not isinstance(record, dict) else 'ssp_m1' - ), - object_id=getattr(record, 'pk', 0) if record and not isinstance(record, dict) else 0, - fields_json=None - ) + model = schema.model if schema else None + + # make fields optional + return ParserError.objects.create( + file=datafile, + row_number=line_number, + column_number=getattr(field, 'item', 0), + item_number=getattr(field, 'item', 0), + field_name=getattr(field, 'name', 'none'), + category=error_category, + case_number=getattr(record, 'CASE_NUMBER', None), + error_message=error_message, + error_type=error_category, + content_type=ContentType.objects.get( + model=model if record and not isinstance(record, dict) else 'ssp_m1' + ), + object_id=getattr(record, 'pk', 0) if record and not isinstance(record, dict) else 0, + fields_json=None + ) def make_generate_parser_error(datafile, line_number): + """Generate a parser error object for a given datafile and line number.""" def generate(schema, error_category, error_message, record=None, field=None): return generate_parser_error( datafile=datafile, @@ -128,14 +127,13 @@ def add_fields(self, fields: list): def get_all_fields(self): """Get all fields from the schema.""" return self.fields - + def get_field(self, name): """Get a field from the schema by name.""" for field in self.fields: if field.name == name: return field - def parse_and_validate(self, line, error_func): """Run all validation steps in order, and parse the given line into a record.""" errors = [] diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index c5e2a4a96..088389b3f 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -61,6 +61,7 @@ def startsWith(substring): def notEmpty(start=0, end=None): + """Validate that string value is not empty between start and end positions.""" return make_validator( lambda value: not value[start:end if end else len(value)].isspace(), lambda value: f'{value} contains blanks between positions {start} and {end if end else len(value)}.' @@ -116,6 +117,7 @@ def validate_single_header_trailer(datafile): def validate_header_section_matches_submission(datafile, section): + """Validate that the header section matches the submission section.""" is_valid = datafile.section == section error = None @@ -131,4 +133,3 @@ def validate_header_section_matches_submission(datafile, section): ) return is_valid, error - From e118ff3318a12dab535a5d3adf0a69b6f4fd6b99 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 27 Apr 2023 16:05:56 -0400 Subject: [PATCH 011/120] Linting changes and comparing to 1610 --- tdrs-backend/tdpservice/parsers/parse.py | 15 --------------- .../tdpservice/parsers/test/test_summary.py | 2 +- tdrs-backend/tdpservice/parsers/util.py | 6 ------ tdrs-backend/tdpservice/scheduling/parser_task.py | 2 +- 4 files changed, 2 insertions(+), 23 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 3cb74acc2..fa3924bb7 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -149,21 +149,6 @@ def parse_datafile_line(line, schema, error_func): if record: record.save() - # for error_msg in record_errors: - # error_obj = ParserError.objects.create( - # file=None, - # row_number=None, - # column_number=None, - # field_name=None, - # category=None, - # case_number=None, - # error_message=error_msg, - # error_type=None, - # content_type=schema.model, - # object_id=record.pk, - # fields_json=None - # ) - return record_is_valid, record_errors return (False, [ diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 90a736bf3..b797f508b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -55,4 +55,4 @@ def test_dfs_set_status(dfs): parser_errors.append(ParserErrorFactory(row_number=5, category="1")) dfs.set_status(errors={'document': parser_errors}) - assert dfs.status == DataFileSummary.Status.REJECTED \ No newline at end of file + assert dfs.status == DataFileSummary.Status.REJECTED diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 595651df4..056ad912c 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -195,12 +195,6 @@ def parse_line(self, line): else: setattr(record, field.name, value) - # if not isinstance(record, dict): - # record.save() - - # if not isinstance(record, dict): - # record.save() - return record def run_field_validators(self, instance, error_func): diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index b1133d1f1..c2a3eadd4 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -17,5 +17,5 @@ def parse(data_file_id): data_file = DataFile.objects.get(id=data_file_id) logger.info(f"DataFile parsing started for file {data_file.filename}") - status, errors = parse_datafile(data_file) + errors = parse_datafile(data_file) logger.info(f"DataFile parsing finished with status {status} and {len(errors)} errors: {errors}") From 21f895463bf311690dc3ce9b8a021f0c3a939c2e Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 27 Apr 2023 16:29:18 -0400 Subject: [PATCH 012/120] Some unit test linting but inherited 1610 issues --- tdrs-backend/tdpservice/parsers/parse.py | 79 ++++++++++--------- .../tdpservice/parsers/test/test_parse.py | 2 - .../tdpservice/parsers/test/test_summary.py | 4 +- .../tdpservice/scheduling/parser_task.py | 2 +- 4 files changed, 43 insertions(+), 44 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index fa3924bb7..9cae2460d 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -109,8 +109,8 @@ def parse_datafile(datafile): util.make_generate_parser_error(datafile, line_number) ) - if not record_is_valid: - errors[line_number] = record_errors + if not record_is_valid: + errors[line_number] = record_errors summary = DataFileSummary(datafile=datafile) summary.set_status(errors) @@ -119,6 +119,7 @@ def parse_datafile(datafile): return errors def parse_multi_record_line(line, schema, error_func): + """Parse a line with a multi-record schema.""" if schema: records = schema.parse_and_validate(line, error_func) @@ -177,41 +178,41 @@ def get_schema(line, section, schema_options): """Return the appropriate schema for the line.""" if section == 'A' and line.startswith('T1'): return schema_options.t1 - elif section == 'A' and line.startswith('T2'): - return None - # return schema_options.t2 - elif section == 'A' and line.startswith('T3'): - return None - # return schema_options.t3 - elif section == 'C' and line.startswith('T4'): - return None - # return schema_options.t4 - elif section == 'C' and line.startswith('T5'): - return None - # return schema_options.t5 - elif section == 'G' and line.startswith('T6'): - return None - # return schema_options.t6 - elif section == 'S' and line.startswith('T7'): - return None - # return schema_options.t7 - elif section == 'A' and line.startswith('M1'): - return schema_options.m1 - elif section == 'A' and line.startswith('M2'): - return schema_options.m2 - elif section == 'A' and line.startswith('M3'): - return schema_options.m3 - elif section == 'C' and line.startswith('M4'): - return None - # return schema_options.t4 - elif section == 'C' and line.startswith('M5'): - return None - # return schema_options.t5 - elif section == 'G' and line.startswith('M6'): - return None - # return schema_options.t6 - elif section == 'S' and line.startswith('M7'): - return None - # return schema_options.t7 - else: + # elif section == 'A' and line.startswith('T2'): + # return None + # # return schema_options.t2 + # elif section == 'A' and line.startswith('T3'): + # return None + # # return schema_options.t3 + # elif section == 'C' and line.startswith('T4'): + # return None + # # return schema_options.t4 + # elif section == 'C' and line.startswith('T5'): + # return None + # # return schema_options.t5 + # elif section == 'G' and line.startswith('T6'): + # return None + # # return schema_options.t6 + # elif section == 'S' and line.startswith('T7'): + # return None + # # return schema_options.t7 + # elif section == 'A' and line.startswith('M1'): + # return schema_options.m1 + # elif section == 'A' and line.startswith('M2'): + # return schema_options.m2 + # elif section == 'A' and line.startswith('M3'): + # return schema_options.m3 + # elif section == 'C' and line.startswith('M4'): + # return None + # # return schema_options.t4 + # elif section == 'C' and line.startswith('M5'): + # return None + # # return schema_options.t5 + # elif section == 'G' and line.startswith('M6'): + # return None + # # return schema_options.t6 + # elif section == 'S' and line.startswith('M7'): + # return None + # # return schema_options.t7 + else: # Just a place-holder for linting until we full implement this. return None diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index bf22207a1..a1e12a40f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -26,13 +26,11 @@ def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): return datafile - @pytest.fixture def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" return create_test_datafile('small_correct_file', stt_user, stt) - @pytest.mark.django_db def test_parse_small_correct_file(test_datafile): """Test parsing of small_correct_file.""" diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index b797f508b..9eb270d00 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -4,7 +4,7 @@ from tdpservice.parsers import parse from tdpservice.parsers.models import DataFileSummary from .factories import DataFileSummaryFactory, ParserErrorFactory -from .test_parse import test_datafile +from .test_parse import test_datafile import logging logger = logging.getLogger(__name__) @@ -44,7 +44,7 @@ def test_dfs_set_status(dfs): # this feels precarious for tests. the defaults in the factory could cause issues should logic change # resulting in failures if we stop keying off category and instead go to content or msg - for i in range(2, 4): + for i in range(2, 4): parser_errors.append(ParserErrorFactory(row_number=i, category=str(i))) dfs.set_status(errors={'document': parser_errors}) diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index c2a3eadd4..649ae0723 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -18,4 +18,4 @@ def parse(data_file_id): logger.info(f"DataFile parsing started for file {data_file.filename}") errors = parse_datafile(data_file) - logger.info(f"DataFile parsing finished with status {status} and {len(errors)} errors: {errors}") + logger.info(f"DataFile parsing finished with {len(errors)} errors: {errors}") From 618d3007c6d8e31c36a5804424e7ba3b30646405 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 27 Apr 2023 17:00:48 -0400 Subject: [PATCH 013/120] Re-ordering job to run tests vs lint first. --- .circleci/build-and-test/jobs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml index 8da5f672f..de9339e5c 100644 --- a/.circleci/build-and-test/jobs.yml +++ b/.circleci/build-and-test/jobs.yml @@ -5,14 +5,14 @@ - checkout - docker-compose-check - docker-compose-up-backend - - run: - name: Execute Python Linting Test - command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ." - run: name: Run Unit Tests And Create Code Coverage Report command: | cd tdrs-backend; docker-compose run --rm web bash -c "./wait_for_services.sh && pytest --cov-report=xml" + - run: + name: Execute Python Linting Test + command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ." - upload-codecov: component: backend coverage-report: ./tdrs-backend/coverage.xml From 953c9ce9c10a50881e1affa04ae7802cbceae8d8 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 27 Apr 2023 17:17:09 -0400 Subject: [PATCH 014/120] Updates to linting and unit tests. --- tdrs-backend/setup.cfg | 2 +- tdrs-backend/tdpservice/parsers/models.py | 9 ++++----- tdrs-backend/tdpservice/users/test/test_permissions.py | 3 +++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/setup.cfg b/tdrs-backend/setup.cfg index ab064a186..f5db722de 100644 --- a/tdrs-backend/setup.cfg +++ b/tdrs-backend/setup.cfg @@ -34,7 +34,7 @@ exclude = # These settings files often have very long strings */settings/common.py/, # D203 conflicts with D211, which is the more conventional of the two -extend-ignore = E226,E302,E41,D203 +extend-ignore = E226,E302,E41,D203,E501 # Reducing line length so flake8 linter forces easier to read code max-line-length = 120 max-complexity = 10 diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index ffe8be4c7..90c6705bc 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -129,9 +129,8 @@ def set_status(self, errors): def check_for_preparsing(errors): """Check for pre-parsing errors.""" - for error in errors['document']: - print(error) - print(error.category) - if error.category == "1": - return True + for key in errors.keys(): # keys are 'header', 'trailer', 'document' + for error in errors[key]: + if error.category == "1": + return True return False diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py index 984e3b226..2f25347aa 100644 --- a/tdrs-backend/tdpservice/users/test/test_permissions.py +++ b/tdrs-backend/tdpservice/users/test/test_permissions.py @@ -111,6 +111,9 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'parsers.add_parsererror', 'parsers.change_parsererror', 'parsers.view_parsererror', + 'parsers.add_datafilesummary', + 'parsers.view_datafilesummary', + 'parsers.change_datafilesummary', 'search_indexes.add_ssp_m1', 'search_indexes.view_ssp_m1', 'search_indexes.change_ssp_m1', From 70692f324c25862f8535ee1e5586fd0e3a0c79ce Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 28 Apr 2023 10:54:32 -0400 Subject: [PATCH 015/120] Fixing linting. --- tdrs-backend/tdpservice/parsers/parse.py | 2 +- tdrs-backend/tdpservice/parsers/test/test_summary.py | 7 ++++++- tdrs-backend/tdpservice/parsers/util.py | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 041d8a5b5..861ca9a26 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -119,7 +119,7 @@ def parse_datafile_lines(datafile, program_type, section): ) if not record_is_valid: - errors[line_number] = record_errors + errors[line_number] = record_errors summary = DataFileSummary(datafile=datafile) summary.set_status(errors) diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 9eb270d00..47be67a0c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -4,11 +4,16 @@ from tdpservice.parsers import parse from tdpservice.parsers.models import DataFileSummary from .factories import DataFileSummaryFactory, ParserErrorFactory -from .test_parse import test_datafile +from .test_parse import create_test_datafile import logging logger = logging.getLogger(__name__) +@pytest.fixture +def test_datafile(stt_user, stt): + """Fixture for small_correct_file.""" + return create_test_datafile('small_correct_file', stt_user, stt) + @pytest.fixture def dfs(): """Fixture for DataFileSummary.""" diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index eaacbc119..a863ab393 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -11,6 +11,7 @@ def value_is_empty(value, length): return value is None or value in empty_values + error_types = { 1: 'File pre-check', 2: 'Record value invalid', From d9f3a43f399888935860d20612c09dada84e1e6a Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Fri, 28 Apr 2023 14:51:25 -0400 Subject: [PATCH 016/120] Update tdrs-backend/setup.cfg --- tdrs-backend/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/setup.cfg b/tdrs-backend/setup.cfg index f5db722de..ab064a186 100644 --- a/tdrs-backend/setup.cfg +++ b/tdrs-backend/setup.cfg @@ -34,7 +34,7 @@ exclude = # These settings files often have very long strings */settings/common.py/, # D203 conflicts with D211, which is the more conventional of the two -extend-ignore = E226,E302,E41,D203,E501 +extend-ignore = E226,E302,E41,D203 # Reducing line length so flake8 linter forces easier to read code max-line-length = 120 max-complexity = 10 From 3db0af610d1eef17b64808b05debe5586a7527e7 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 8 May 2023 14:24:21 -0400 Subject: [PATCH 017/120] updates per PR. --- tdrs-backend/tdpservice/parsers/models.py | 2 +- tdrs-backend/tdpservice/parsers/test/factories.py | 4 ++-- tdrs-backend/tdpservice/parsers/test/test_summary.py | 9 ++------- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 6dec3a621..555b409db 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -132,6 +132,6 @@ def check_for_preparsing(errors): """Check for pre-parsing errors.""" for key in errors.keys(): # keys are 'header', 'trailer', 'document' for error in errors[key]: - if error.category == "1": + if error.category == ParserErrorCategoryChoices.PRE_CHECK: return True return False diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index a355c7f17..96f22ed66 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -1,7 +1,7 @@ """Factories for generating test data for parsers.""" import factory -from tdpservice.parsers.models import DataFileSummary +from tdpservice.parsers.models import DataFileSummary, ParserErrorCategoryChoices from tdpservice.data_files.test.factories import DataFileFactory from tdpservice.users.test.factories import UserFactory from tdpservice.stts.test.factories import STTFactory @@ -70,7 +70,7 @@ class Meta: column_number = 1 item_number = 1 field_name = "test field name" - category = "1" + category = ParserErrorCategoryChoices.PRE_CHECK case_number = '1' rpt_month_year = 202001 error_message = "test error message" diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 47be67a0c..2313af70b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -2,7 +2,7 @@ import pytest from tdpservice.parsers import parse -from tdpservice.parsers.models import DataFileSummary +from tdpservice.parsers.models import DataFileSummary, ParserErrorCategoryChoices from .factories import DataFileSummaryFactory, ParserErrorFactory from .test_parse import create_test_datafile @@ -22,14 +22,11 @@ def dfs(): @pytest.mark.django_db def test_dfs_model(dfs): """Test that the model is created and populated correctly.""" - dfs = dfs - assert dfs.case_aggregates['Jan']['accepted'] == 100 @pytest.mark.django_db(transaction=True) def test_dfs_rejected(test_datafile, dfs): """Ensure that an invalid file generates a rejected status.""" - dfs = dfs test_datafile.section = 'Closed Case Data' test_datafile.save() @@ -39,7 +36,6 @@ def test_dfs_rejected(test_datafile, dfs): @pytest.mark.django_db def test_dfs_set_status(dfs): """Test that the status is set correctly.""" - dfs = dfs assert dfs.status == DataFileSummary.Status.PENDING dfs.set_status(errors={}) assert dfs.status == DataFileSummary.Status.ACCEPTED @@ -56,8 +52,7 @@ def test_dfs_set_status(dfs): assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - print("about to add a category 1 error") - parser_errors.append(ParserErrorFactory(row_number=5, category="1")) + parser_errors.append(ParserErrorFactory(row_number=5, category=ParserErrorCategoryChoices.PRE_CHECK)) dfs.set_status(errors={'document': parser_errors}) assert dfs.status == DataFileSummary.Status.REJECTED From 83009cbaec940cf5d8087e19b99c538f922da46b Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 8 May 2023 14:29:52 -0400 Subject: [PATCH 018/120] Excluding trailers for rejection --- tdrs-backend/tdpservice/parsers/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 555b409db..83243bb00 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -131,7 +131,9 @@ def set_status(self, errors): def check_for_preparsing(errors): """Check for pre-parsing errors.""" for key in errors.keys(): # keys are 'header', 'trailer', 'document' + for error in errors[key]: - if error.category == ParserErrorCategoryChoices.PRE_CHECK: + if (error.category == ParserErrorCategoryChoices.PRE_CHECK) and \ + (key != 'trailer'): return True return False From 223ff08e4f53b89fd33d890bea20fe4b066bd7fc Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 8 May 2023 14:34:08 -0400 Subject: [PATCH 019/120] VSCode merge resolution is garbage. --- tdrs-backend/tdpservice/parsers/parse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index f8fcec33a..06c63c62d 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -2,10 +2,7 @@ import os from . import schema_defs, validators, util -<<<<<<< HEAD -======= from .models import ParserErrorCategoryChoices ->>>>>>> feature/1610-parser-error-generator from tdpservice.data_files.models import DataFile from .models import DataFileSummary From 297e13d5a3ab1df3a13a073f4f480204c8fe1f6b Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 24 May 2023 16:20:31 -0400 Subject: [PATCH 020/120] Fixing precheck for not implemented types --- tdrs-backend/tdpservice/parsers/models.py | 57 ++++++----- tdrs-backend/tdpservice/parsers/parse.py | 94 +++++++++++-------- .../tdpservice/parsers/schema_defs/tanf/t1.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 33 ++++--- .../tdpservice/parsers/test/test_summary.py | 14 ++- tdrs-backend/tdpservice/parsers/util.py | 9 -- 6 files changed, 119 insertions(+), 90 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 83243bb00..38276999c 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -114,26 +114,39 @@ class Status(models.TextChoices): } """ - def set_status(self, errors): + def get_status(self, errors): """Set and return the status field based on errors and models associated with datafile.""" - # to set rejected, we would need to have raised an exception during (pre)parsing - # else if there are errors, we can set to accepted with errors - # else we can set to accepted (default) - if errors == {}: # This feels better than running len() on `errors`...but is it a dict vs list? - self.status = self.Status.ACCEPTED - elif check_for_preparsing(errors): - self.status = self.Status.REJECTED - elif errors: - self.status = self.Status.ACCEPTED_WITH_ERRORS - - return self.status - -def check_for_preparsing(errors): - """Check for pre-parsing errors.""" - for key in errors.keys(): # keys are 'header', 'trailer', 'document' - - for error in errors[key]: - if (error.category == ParserErrorCategoryChoices.PRE_CHECK) and \ - (key != 'trailer'): - return True - return False + if errors is None: + return self.status # aka PENDING + + if type(errors) != dict: + raise TypeError("errors parameter must be a dictionary.") + + if errors == {}: + return self.Status.ACCEPTED + elif DataFileSummary.find_precheck(errors): + return self.Status.REJECTED + else: + return self.Status.ACCEPTED_WITH_ERRORS + + def find_precheck(errors): + """Check for pre-parsing errors. + + @param errors: dict of errors keyed by location in datafile. + e.g. + errors = + { + "trailer": [ParserError, ...], + "header": [ParserError, ...], + "document": [ParserError, ...], + "123": [ParserError, ...], + ... + } + """ + for key in errors.keys(): + if key == 'trailer': + continue + for parserError in errors[key]: + if parserError.category == ParserErrorCategoryChoices.PRE_CHECK: + return True + return False diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 06c63c62d..895bda1ff 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -4,7 +4,6 @@ from . import schema_defs, validators, util from .models import ParserErrorCategoryChoices from tdpservice.data_files.models import DataFile -from .models import DataFileSummary def parse_datafile(datafile): @@ -61,6 +60,9 @@ def parse_datafile(datafile): }, } + # TODO: utility transformations between text to schemas and back + # text > prog > sections > schemas + program_type = header['program_type'] section = header['type'] @@ -77,6 +79,35 @@ def parse_datafile(datafile): errors = errors | line_errors + # errors['summary'] = DataFileSummary.objects.create( + # datafile=datafile, + # status=DataFileSummary.get_status(errors) + # ) + + # or perhaps just invert this? + # what does it look like having the errors dict as a field of the summary? + # summary.errors = errors --- but I don't want/need to store this in DB + # divesting that storage and just using my FK to datafile so I can run querysets later + # perserves the ability to use the summary object to generate the errors dict + + # perhaps just formalize the entire errors struct? + # pros: + # - can be used to generate error report + # - can be used to generate summary + # - can be used to generate error count + # - can be used to generate error count by type + # - can be used to generate error count by record type + # - can be used to generate error count by field + # - can be used to generate error count by field type + # - has a consistent structure between differing file types + # - has testable functions for each of the above + # - has agreed-upon inputs/outputs + # cons: + # - requires boilerplate to generate + # - different structures may be needed for different purposes + # - built-in dict may be easier to reference ala Cameron + # - built-in dict is freer-form and complete already + return errors @@ -97,6 +128,17 @@ def parse_datafile_lines(datafile, program_type, section): continue schema = get_schema(line, section, schema_options) + if schema is None: + errors[line_number] = [util.generate_parser_error( + datafile=datafile, + line_number=line_number, + schema=None, + error_category=ParserErrorCategoryChoices.FIELD_VALUE, + error_message="Record_Type '{}' not yet implemented.", + record=None, + field="Record_Type", + )] + continue if isinstance(schema, util.MultiRecordRowSchema): records = parse_multi_record_line( @@ -123,56 +165,30 @@ def parse_datafile_lines(datafile, program_type, section): if not record_is_valid: errors[line_number] = record_errors - summary = DataFileSummary(datafile=datafile) - summary.set_status(errors) - summary.save() - return errors def parse_multi_record_line(line, schema, generate_error): """Parse and validate a datafile line using MultiRecordRowSchema.""" - if schema: - records = schema.parse_and_validate(line, generate_error) - - for r in records: - record, record_is_valid, record_errors = r + records = schema.parse_and_validate(line, generate_error) - if record: - record.save() + for r in records: + record, record_is_valid, record_errors = r - return records + if record: + record.save() - return [(None, False, [ - generate_error( - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="No schema selected.", - record=None, - field=None - ) - ])] + return records def parse_datafile_line(line, schema, generate_error): """Parse and validate a datafile line and save any errors to the model.""" - if schema: - record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) - - if record: - record.save() + record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) - return record_is_valid, record_errors + if record: + record.save() - return (False, [ - generate_error( - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="No schema selected.", - record=None, - field=None - ) - ]) + return record_is_valid, record_errors def get_schema_options(program_type): @@ -182,8 +198,8 @@ def get_schema_options(program_type): return { 'A': { 'T1': schema_defs.tanf.t1, - # 'T2': schema_options.t2, - # 'T3': schema_options.t3, + # 'T2': schema_defs.tanf.t2, + # 'T3': schema_defs.tanf.t3, }, 'C': { # 'T4': schema_options.t4, diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index ceb59db10..483950ce8 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -1,4 +1,4 @@ -"""Schema for HEADER row of all submission types.""" +"""Schema for t1 record types.""" from ...util import RowSchema, Field diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index caadd3bec..b73bfda1f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -4,11 +4,11 @@ import pytest from pathlib import Path from .. import parse -from ..models import ParserError, ParserErrorCategoryChoices +from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary from tdpservice.data_files.models import DataFile from tdpservice.search_indexes.models.tanf import TANF_T1 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 - +from .factories import DataFileSummaryFactory def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): """Create a test DataFile instance with the given file attached.""" @@ -31,12 +31,18 @@ def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" return create_test_datafile('small_correct_file', stt_user, stt) +@pytest.fixture +def dfs(): + """Fixture for DataFileSummary.""" + return DataFileSummaryFactory() + @pytest.mark.django_db -def test_parse_small_correct_file(test_datafile): +def test_parse_small_correct_file(test_datafile, dfs): """Test parsing of small_correct_file.""" errors = parse.parse_datafile(test_datafile) assert errors == {} + assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED assert TANF_T1.objects.count() == 1 assert ParserError.objects.filter(file=test_datafile).count() == 0 @@ -55,13 +61,13 @@ def test_parse_small_correct_file(test_datafile): @pytest.mark.django_db -def test_parse_section_mismatch(test_datafile): +def test_parse_section_mismatch(test_datafile, dfs): """Test parsing of small_correct_file where the DataFile section doesn't match the rawfile section.""" test_datafile.section = 'Closed Case Data' test_datafile.save() errors = parse.parse_datafile(test_datafile) - + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -76,12 +82,13 @@ def test_parse_section_mismatch(test_datafile): @pytest.mark.django_db -def test_parse_wrong_program_type(test_datafile): +def test_parse_wrong_program_type(test_datafile, dfs): """Test parsing of small_correct_file where the DataFile program type doesn't match the rawfile.""" test_datafile.section = 'SSP Active Case Data' test_datafile.save() errors = parse.parse_datafile(test_datafile) + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -103,12 +110,13 @@ def test_big_file(stt_user, stt): @pytest.mark.django_db -def test_parse_big_file(test_big_file): +def test_parse_big_file(test_big_file, dfs): """Test parsing of ADS.E2J.FTP1.TS06.""" expected_errors_count = 1828 expected_t1_record_count = 815 errors = parse.parse_datafile(test_big_file) + assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS parser_errors = ParserError.objects.filter(file=test_big_file) assert len(errors.keys()) == expected_errors_count @@ -123,10 +131,12 @@ def bad_test_file(stt_user, stt): @pytest.mark.django_db -def test_parse_bad_test_file(bad_test_file): +def test_parse_bad_test_file(bad_test_file, dfs): """Test parsing of bad_TANF_S2.""" errors = parse.parse_datafile(bad_test_file) + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + parser_errors = ParserError.objects.filter(file=bad_test_file) assert parser_errors.count() == 1 @@ -147,10 +157,10 @@ def bad_file_missing_header(stt_user, stt): @pytest.mark.django_db -def test_parse_bad_file_missing_header(bad_file_missing_header): +def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): """Test parsing of bad_missing_header.""" errors = parse.parse_datafile(bad_file_missing_header) - + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_missing_header) assert parser_errors.count() == 1 @@ -171,9 +181,10 @@ def bad_file_multiple_headers(stt_user, stt): @pytest.mark.django_db -def test_parse_bad_file_multiple_headers(bad_file_multiple_headers): +def test_parse_bad_file_multiple_headers(bad_file_multiple_headers, dfs): """Test parsing of bad_two_headers.""" errors = parse.parse_datafile(bad_file_multiple_headers) + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_multiple_headers) assert parser_errors.count() == 1 diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 2313af70b..e68e1a128 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -30,29 +30,27 @@ def test_dfs_rejected(test_datafile, dfs): test_datafile.section = 'Closed Case Data' test_datafile.save() - dfs.set_status(parse.parse_datafile(test_datafile)) - assert dfs.status == DataFileSummary.Status.REJECTED + error_ast = parse.parse_datafile(test_datafile) + dfs.status = dfs.get_status(error_ast) + assert DataFileSummary.Status.REJECTED == dfs.status @pytest.mark.django_db def test_dfs_set_status(dfs): """Test that the status is set correctly.""" assert dfs.status == DataFileSummary.Status.PENDING - dfs.set_status(errors={}) + dfs.status = dfs.get_status(errors={}) assert dfs.status == DataFileSummary.Status.ACCEPTED - # create category 1 ParserError list to prompt rejected status parser_errors = [] - # this feels precarious for tests. the defaults in the factory could cause issues should logic change - # resulting in failures if we stop keying off category and instead go to content or msg for i in range(2, 4): parser_errors.append(ParserErrorFactory(row_number=i, category=str(i))) - dfs.set_status(errors={'document': parser_errors}) + dfs.status = dfs.get_status(errors={'document': parser_errors}) assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS parser_errors.append(ParserErrorFactory(row_number=5, category=ParserErrorCategoryChoices.PRE_CHECK)) - dfs.set_status(errors={'document': parser_errors}) + dfs.status = dfs.get_status(errors={'document': parser_errors}) assert dfs.status == DataFileSummary.Status.REJECTED diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 0f024b189..41e379255 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -12,15 +12,6 @@ def value_is_empty(value, length): return value is None or value in empty_values -error_types = { - 1: 'File pre-check', - 2: 'Record value invalid', - 3: 'Record value consistency', - 4: 'Case consistency', - 5: 'Section consistency', - 6: 'Historical consistency' -} - def generate_parser_error(datafile, line_number, schema, error_category, error_message, record=None, field=None): """Create and return a ParserError using args.""" model = schema.model if schema else None From c0f5a7475dc4a2a209575229ef271cac71b33107 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 5 Jun 2023 10:41:01 -0400 Subject: [PATCH 021/120] Updating to error-handle not implemented schema types --- tdrs-backend/tdpservice/parsers/models.py | 2 +- tdrs-backend/tdpservice/parsers/parse.py | 4 ++-- tdrs-backend/tdpservice/parsers/test/factories.py | 3 +-- tdrs-backend/tdpservice/parsers/test/test_parse.py | 7 +++++-- tdrs-backend/tdpservice/parsers/test/test_summary.py | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 88c9a0ddf..78b12a163 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -146,6 +146,6 @@ def find_precheck(errors): if key == 'trailer': continue for parserError in errors[key]: - if parserError.category == ParserErrorCategoryChoices.PRE_CHECK: + if parserError.error_type == ParserErrorCategoryChoices.PRE_CHECK: return True return False diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index e670246e3..3ee7dffad 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -133,8 +133,8 @@ def parse_datafile_lines(datafile, program_type, section): datafile=datafile, line_number=line_number, schema=None, - error_category=ParserErrorCategoryChoices.FIELD_VALUE, - error_message="Record_Type '{}' not yet implemented.", + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Unknown Record_Type was found.", record=None, field="Record_Type", )] diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 96f22ed66..e45e78be2 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -70,11 +70,10 @@ class Meta: column_number = 1 item_number = 1 field_name = "test field name" - category = ParserErrorCategoryChoices.PRE_CHECK case_number = '1' rpt_month_year = 202001 error_message = "test error message" - error_type = "out of range" + error_type = ParserErrorCategoryChoices.PRE_CHECK created_at = factory.Faker("date_time") fields_json = {"test": "test"} diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index dd15fee4e..00f2c90ee 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -66,6 +66,9 @@ def test_parse_section_mismatch(test_datafile, dfs): test_datafile.section = 'Closed Case Data' test_datafile.save() + dfs.datafile = test_datafile + dfs.save() + errors = parse.parse_datafile(test_datafile) assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) @@ -536,7 +539,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field row_5_error = parser_errors.get(row_number=5) assert row_5_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert row_5_error.error_message == 'No schema selected.' + assert row_5_error.error_message == 'Unknown Record_Type was found.' assert row_5_error.content_type is None assert row_5_error.object_id is None @@ -582,7 +585,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel row_5_error = parser_errors.get(row_number=5) assert row_5_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert row_5_error.error_message == 'No schema selected.' + assert row_5_error.error_message == 'Unknown Record_Type was found.' assert row_5_error.content_type is None assert row_5_error.object_id is None diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index e68e1a128..814fa2596 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -50,7 +50,7 @@ def test_dfs_set_status(dfs): assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - parser_errors.append(ParserErrorFactory(row_number=5, category=ParserErrorCategoryChoices.PRE_CHECK)) + parser_errors.append(ParserErrorFactory(row_number=5, error_type=ParserErrorCategoryChoices.PRE_CHECK)) dfs.status = dfs.get_status(errors={'document': parser_errors}) assert dfs.status == DataFileSummary.Status.REJECTED From 5fca243eb1d25b5daffc6e5ce55e11f5f1c1d73f Mon Sep 17 00:00:00 2001 From: elipe17 Date: Wed, 28 Jun 2023 12:58:08 -0600 Subject: [PATCH 022/120] - Updated view to show latest datafiles - Added admin filter to show newest or all datafile records - Updated indices to allow easier elastic queries --- tdrs-backend/tdpservice/data_files/views.py | 19 +- tdrs-backend/tdpservice/parsers/parse.py | 16 +- .../tdpservice/search_indexes/admin/filter.py | 30 ++++ .../tdpservice/search_indexes/admin/tanf.py | 7 + .../search_indexes/documents/ssp.py | 9 + .../search_indexes/documents/tanf.py | 36 +++- .../migrations/0014_auto_20230628_1654.py | 165 ++++++++++++++++++ .../tdpservice/search_indexes/models/ssp.py | 31 ++++ .../tdpservice/search_indexes/models/tanf.py | 71 ++++++++ 9 files changed, 378 insertions(+), 6 deletions(-) create mode 100644 tdrs-backend/tdpservice/search_indexes/admin/filter.py create mode 100644 tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230628_1654.py diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 5cf0b917b..647d1121a 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -123,12 +123,29 @@ def get_s3_versioning_id(self, file_name, prefix): def get_queryset(self): """Apply custom queryset filters.""" queryset = super().get_queryset().order_by('-created_at') - if self.request.query_params.get('file_type') == 'ssp-moe': queryset = queryset.filter(section__contains='SSP') else: queryset = queryset.exclude(section__contains='SSP') + q1 = queryset.filter(quarter='Q1') + if len(q1): + q1 = q1.filter(created_at=q1.latest('created_at').created_at) + + q2 = queryset.filter(quarter='Q2') + if len(q2): + q2 = q2.filter(created_at=q2.latest('created_at').created_at) + + q3 = queryset.filter(quarter='Q3') + if len(q3): + q3 = q3.filter(created_at=q3.latest('created_at').created_at) + + q4 = queryset.filter(quarter='Q4') + if len(q4): + q4 = q4.filter(created_at=q4.latest('created_at').created_at) + + queryset = q1 | q2 | q3 | q4 + return queryset def filter_queryset(self, queryset): diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 873216404..e6c44cc68 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -87,7 +87,8 @@ def parse_datafile_lines(datafile, program_type, section): records = parse_multi_record_line( line, schema, - util.make_generate_parser_error(datafile, line_number) + util.make_generate_parser_error(datafile, line_number), + datafile ) record_number = 0 @@ -102,7 +103,8 @@ def parse_datafile_lines(datafile, program_type, section): record_is_valid, record_errors = parse_datafile_line( line, schema, - util.make_generate_parser_error(datafile, line_number) + util.make_generate_parser_error(datafile, line_number), + datafile ) if not record_is_valid: errors[line_number] = record_errors @@ -110,7 +112,7 @@ def parse_datafile_lines(datafile, program_type, section): return errors -def parse_multi_record_line(line, schema, generate_error): +def parse_multi_record_line(line, schema, generate_error, datafile): """Parse and validate a datafile line using MultiRecordRowSchema.""" if schema: records = schema.parse_and_validate(line, generate_error) @@ -119,6 +121,9 @@ def parse_multi_record_line(line, schema, generate_error): record, record_is_valid, record_errors = r if record: + record.created_at = datafile.creation_date + record.version = datafile.version + record.parent = datafile record.save() return records @@ -134,12 +139,15 @@ def parse_multi_record_line(line, schema, generate_error): ])] -def parse_datafile_line(line, schema, generate_error): +def parse_datafile_line(line, schema, generate_error, datafile): """Parse and validate a datafile line and save any errors to the model.""" if schema: record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) if record: + record.created_at = datafile.creation_date + record.version = datafile.version + record.parent = datafile record.save() return record_is_valid, record_errors diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filter.py b/tdrs-backend/tdpservice/search_indexes/admin/filter.py new file mode 100644 index 000000000..ecdc61b66 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/admin/filter.py @@ -0,0 +1,30 @@ + +from django.utils.translation import ugettext_lazy as _ +from django.contrib.admin import SimpleListFilter + +class CreationDateFilter(SimpleListFilter): + title = _('Newest') + + parameter_name = 'created_at' + + def lookups(self, request, model_admin): + return ( + (None, _('Newest')), + ('all', _('All')), + ) + + def choices(self, cl): + for lookup, title in self.lookup_choices: + yield { + 'selected': self.value() == lookup, + 'query_string': cl.get_query_string({ + self.parameter_name: lookup, + }, []), + 'display': title, + } + + def queryset(self, request, queryset): + if self.value() == None and len(queryset): + max_date = queryset.latest('created_at').created_at + return queryset.filter(created_at=max_date) + return queryset.order_by("-created_at") diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 27100bcb6..9fa495e08 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -1,11 +1,13 @@ """ModelAdmin classes for parsed TANF data files.""" from django.contrib import admin +from .filter import CreationDateFilter class TANF_T1Admin(admin.ModelAdmin): """ModelAdmin class for parsed T1 data files.""" list_display = [ + 'version', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -15,6 +17,7 @@ class TANF_T1Admin(admin.ModelAdmin): ] list_filter = [ + CreationDateFilter, 'RPT_MONTH_YEAR', 'ZIP_CODE', 'STRATUM', @@ -25,12 +28,14 @@ class TANF_T2Admin(admin.ModelAdmin): """ModelAdmin class for parsed T2 data files.""" list_display = [ + 'version', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', ] list_filter = [ + CreationDateFilter, 'RPT_MONTH_YEAR', ] @@ -39,12 +44,14 @@ class TANF_T3Admin(admin.ModelAdmin): """ModelAdmin class for parsed T3 data files.""" list_display = [ + 'version', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', ] list_filter = [ + CreationDateFilter, 'RPT_MONTH_YEAR', ] diff --git a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py index 888e6c0dc..35c4a3e06 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py @@ -23,6 +23,9 @@ class Django: model = SSP_M1 fields = [ + 'version', + 'created_at', + 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -87,6 +90,9 @@ class Django: model = SSP_M2 fields = [ + 'version', + 'created_at', + 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -175,6 +181,9 @@ class Django: model = SSP_M3 fields = [ + 'version', + 'created_at', + 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index 4f313af25..7f38783bf 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -1,14 +1,23 @@ """Elasticsearch document mappings for TANF submission models.""" -from django_elasticsearch_dsl import Document +from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry from ..models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 +from tdpservice.data_files.models import DataFile @registry.register_document class TANF_T1DataSubmissionDocument(Document): """Elastic search model mapping for a parsed TANF T1 data file.""" + parent = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -23,6 +32,8 @@ class Django: model = TANF_T1 fields = [ + 'version', + 'created_at', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -69,6 +80,7 @@ class Django: 'FAMILY_EXEMPT_TIME_LIMITS', 'FAMILY_NEW_CHILD' ] + related_models = [DataFile] @registry.register_document @@ -89,6 +101,10 @@ class Django: model = TANF_T2 fields = [ + + 'version', + 'created_at', + 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -180,6 +196,10 @@ class Django: model = TANF_T3 fields = [ + + 'version', + 'created_at', + 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -222,6 +242,10 @@ class Django: model = TANF_T4 fields = [ + + 'version', + 'created_at', + 'record', 'rpt_month_year', 'case_number', @@ -257,6 +281,10 @@ class Django: model = TANF_T5 fields = [ + + 'version', + 'created_at', + 'record', 'rpt_month_year', 'case_number', @@ -309,6 +337,9 @@ class Django: model = TANF_T6 fields = [ + + 'version', + 'created_at', 'record', 'rpt_month_year', 'fips_code', @@ -350,6 +381,9 @@ class Django: model = TANF_T7 fields = [ + + 'version', + 'created_at', 'record', 'rpt_month_year', 'fips_code', diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230628_1654.py b/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230628_1654.py new file mode 100644 index 000000000..fc7454c2a --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230628_1654.py @@ -0,0 +1,165 @@ +# Generated by Django 3.2.15 on 2023-06-28 16:54 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_files', '0012_datafile_s3_versioning_id'), + ('search_indexes', '0013_rename_uuid'), + ] + + operations = [ + migrations.AddField( + model_name='ssp_m1', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='ssp_m1', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m1_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='ssp_m1', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='ssp_m2', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='ssp_m2', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m2_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='ssp_m2', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='ssp_m3', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='ssp_m3', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m3_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='ssp_m3', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t1', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='tanf_t1', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t1_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='tanf_t1', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t2', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='tanf_t2', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t2_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='tanf_t2', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t3', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='tanf_t3', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t3_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='tanf_t3', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t4', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='tanf_t4', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t4_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='tanf_t4', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t5', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='tanf_t5', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t5_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='tanf_t5', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t6', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='tanf_t6', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t6_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='tanf_t6', + name='version', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='created_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='tanf_t7', + name='parent', + field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t7_parent', to='data_files.datafile'), + ), + migrations.AddField( + model_name='tanf_t7', + name='version', + field=models.IntegerField(null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index 278eeda9a..7cf84ec13 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -3,6 +3,7 @@ import uuid from django.db import models from django.contrib.contenttypes.fields import GenericRelation +from tdpservice.data_files.models import DataFile from tdpservice.parsers.models import ParserError @@ -14,6 +15,16 @@ class SSP_M1(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='m1_parent' + ) error = GenericRelation(ParserError) RecordType = models.CharField(max_length=156, null=True, blank=False) @@ -77,6 +88,16 @@ class SSP_M2(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='m2_parent' + ) error = GenericRelation(ParserError) RecordType = models.CharField(max_length=156, null=True, blank=False) @@ -160,6 +181,16 @@ class SSP_M3(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='m3_parent' + ) error = GenericRelation(ParserError) RecordType = models.CharField(max_length=156, null=True, blank=False) diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index 4f5fac035..1909ac605 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -3,6 +3,7 @@ import uuid from django.db import models from django.contrib.contenttypes.fields import GenericRelation +from tdpservice.data_files.models import DataFile from tdpservice.parsers.models import ParserError @@ -14,6 +15,16 @@ class TANF_T1(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='t1_parent' + ) error = GenericRelation(ParserError) RecordType = models.CharField(max_length=156, null=True, blank=False) @@ -76,6 +87,16 @@ class TANF_T2(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='t2_parent' + ) RecordType = models.CharField(max_length=156, null=True, blank=False) RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) @@ -157,6 +178,16 @@ class TANF_T3(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='t3_parent' + ) RecordType = models.CharField(max_length=156, null=True, blank=False) RPT_MONTH_YEAR = models.IntegerField(null=True, blank=False) @@ -190,6 +221,16 @@ class TANF_T4(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='t4_parent' + ) record = models.CharField(max_length=156, null=False, blank=False) rpt_month_year = models.IntegerField(null=False, blank=False) @@ -219,6 +260,16 @@ class TANF_T5(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='t5_parent' + ) record = models.CharField(max_length=156, null=False, blank=False) rpt_month_year = models.IntegerField(null=False, blank=False) @@ -261,6 +312,16 @@ class TANF_T6(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='t6_parent' + ) record = models.CharField(max_length=156, null=False, blank=False) rpt_month_year = models.IntegerField(null=False, blank=False) @@ -292,6 +353,16 @@ class TANF_T7(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + version = models.IntegerField(null=True, blank=False) + created_at = models.DateTimeField(null=True, blank=False) + parent = models.ForeignKey( + DataFile, + blank=True, + help_text='The parent file from which this record was created.', + null=True, + on_delete=models.CASCADE, + related_name='t7_parent' + ) record = models.CharField(max_length=156, null=False, blank=False) rpt_month_year = models.IntegerField(null=False, blank=False) From a7109c3696576db0efbce51957e386078abd070a Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 5 Jul 2023 15:40:08 -0600 Subject: [PATCH 023/120] - Updated search indices to have parent FK --- tdrs-backend/Dockerfile | 2 + tdrs-backend/tdpservice/parsers/parse.py | 8 +-- .../search_indexes/documents/ssp.py | 31 +++++++-- .../search_indexes/documents/tanf.py | 63 +++++++++++++++---- ...628_1654.py => 0014_auto_20230705_2138.py} | 8 +-- .../tdpservice/search_indexes/models/ssp.py | 6 +- 6 files changed, 89 insertions(+), 29 deletions(-) rename tdrs-backend/tdpservice/search_indexes/migrations/{0014_auto_20230628_1654.py => 0014_auto_20230705_2138.py} (97%) diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile index b3aac7c82..78177e442 100644 --- a/tdrs-backend/Dockerfile +++ b/tdrs-backend/Dockerfile @@ -28,6 +28,8 @@ RUN groupadd -g ${gid} ${group} && useradd -u ${uid} -g ${group} -s /bin/sh ${us RUN chown -R tdpuser /tdpapp && chmod u+x gunicorn_start.sh wait_for_services.sh +# This is not a great solution to our serious version mismatches in our dependencies. It is however the lowest lift solution. +RUN sed -i 's/collections/collections.abc/g' /usr/local/lib/python3.10/site-packages/django_elasticsearch_dsl/fields.py CMD ["./gunicorn_start.sh"] # if the container crashes/loops, we can shell into it by doing the following: diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index e6c44cc68..864b40db0 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -121,9 +121,9 @@ def parse_multi_record_line(line, schema, generate_error, datafile): record, record_is_valid, record_errors = r if record: - record.created_at = datafile.creation_date + record.created_at = datafile.created_at record.version = datafile.version - record.parent = datafile + record.datafile = datafile record.save() return records @@ -145,9 +145,9 @@ def parse_datafile_line(line, schema, generate_error, datafile): record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) if record: - record.created_at = datafile.creation_date + record.created_at = datafile.created_at record.version = datafile.version - record.parent = datafile + record.datafile = datafile record.save() return record_is_valid, record_errors diff --git a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py index 35c4a3e06..b3f26097a 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py @@ -1,14 +1,22 @@ """Elasticsearch document mappings for SSP submission models.""" -from django_elasticsearch_dsl import Document +from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry from ..models.ssp import SSP_M1, SSP_M2, SSP_M3 - +from tdpservice.data_files.models import DataFile @registry.register_document class SSP_M1DataSubmissionDocument(Document): """Elastic search model mapping for a parsed SSP M1 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -25,7 +33,6 @@ class Django: fields = [ 'version', 'created_at', - 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -76,6 +83,14 @@ class Django: class SSP_M2DataSubmissionDocument(Document): """Elastic search model mapping for a parsed SSP M2 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -92,7 +107,6 @@ class Django: fields = [ 'version', 'created_at', - 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -167,6 +181,14 @@ class Django: class SSP_M3DataSubmissionDocument(Document): """Elastic search model mapping for a parsed SSP M3 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -183,7 +205,6 @@ class Django: fields = [ 'version', 'created_at', - 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index 7f38783bf..6bdea2996 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -1,16 +1,16 @@ """Elasticsearch document mappings for TANF submission models.""" +import collections.abc as collections from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry from ..models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 from tdpservice.data_files.models import DataFile - @registry.register_document class TANF_T1DataSubmissionDocument(Document): """Elastic search model mapping for a parsed TANF T1 data file.""" - parent = fields.ObjectField(properties={ + datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), }) @@ -80,13 +80,20 @@ class Django: 'FAMILY_EXEMPT_TIME_LIMITS', 'FAMILY_NEW_CHILD' ] - related_models = [DataFile] @registry.register_document class TANF_T2DataSubmissionDocument(Document): """Elastic search model mapping for a parsed TANF T2 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -104,11 +111,9 @@ class Django: 'version', 'created_at', - 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', - 'FAMILY_AFFILIATION', 'NONCUSTODIAL_PARENT', 'DATE_OF_BIRTH', @@ -182,6 +187,14 @@ class Django: class TANF_T3DataSubmissionDocument(Document): """Elastic search model mapping for a parsed TANF T3 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -199,7 +212,6 @@ class Django: 'version', 'created_at', - 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -228,6 +240,14 @@ class Django: class TANF_T4DataSubmissionDocument(Document): """Elastic search model mapping for a parsed TANF T4 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -245,7 +265,6 @@ class Django: 'version', 'created_at', - 'record', 'rpt_month_year', 'case_number', @@ -267,6 +286,14 @@ class Django: class TANF_T5DataSubmissionDocument(Document): """Elastic search model mapping for a parsed TANF T5 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -284,12 +311,10 @@ class Django: 'version', 'created_at', - 'record', 'rpt_month_year', 'case_number', 'fips_code', - 'family_affiliation', 'date_of_birth', 'ssn', @@ -323,6 +348,14 @@ class Django: class TANF_T6DataSubmissionDocument(Document): """Elastic search model mapping for a parsed TANF T6 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -337,13 +370,11 @@ class Django: model = TANF_T6 fields = [ - 'version', 'created_at', 'record', 'rpt_month_year', 'fips_code', - 'calendar_quarter', 'applications', 'approved', @@ -367,6 +398,14 @@ class Django: class TANF_T7DataSubmissionDocument(Document): """Elastic search model mapping for a parsed TANF T7 data file.""" + datafile = fields.ObjectField(properties={ + 'pk': fields.IntegerField(), + }) + + def get_instances_from_related(self, related_instance): + if isinstance(related_instance, DataFile): + return related_instance + class Index: """ElasticSearch index generation settings.""" @@ -381,13 +420,11 @@ class Django: model = TANF_T7 fields = [ - 'version', 'created_at', 'record', 'rpt_month_year', 'fips_code', - 'calendar_quarter', 'tdrs_section_ind', 'stratum', diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230628_1654.py b/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230705_2138.py similarity index 97% rename from tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230628_1654.py rename to tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230705_2138.py index fc7454c2a..3cac10921 100644 --- a/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230628_1654.py +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230705_2138.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.15 on 2023-06-28 16:54 +# Generated by Django 3.2.15 on 2023-07-05 21:38 from django.db import migrations, models import django.db.models.deletion @@ -19,7 +19,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='ssp_m1', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m1_parent', to='data_files.datafile'), ), migrations.AddField( @@ -34,7 +34,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='ssp_m2', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m2_parent', to='data_files.datafile'), ), migrations.AddField( @@ -49,7 +49,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='ssp_m3', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m3_parent', to='data_files.datafile'), ), migrations.AddField( diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index 7cf84ec13..a1f5bd2f1 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -17,7 +17,7 @@ class SSP_M1(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', @@ -90,7 +90,7 @@ class SSP_M2(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', @@ -183,7 +183,7 @@ class SSP_M3(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', From 21a700621f037c74a3035608c17bf729ebadc5a0 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 5 Jul 2023 15:47:43 -0600 Subject: [PATCH 024/120] - Fix lint errors --- tdrs-backend/tdpservice/search_indexes/admin/filter.py | 9 +++++++-- tdrs-backend/tdpservice/search_indexes/documents/ssp.py | 4 ++++ tdrs-backend/tdpservice/search_indexes/documents/tanf.py | 8 +++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filter.py b/tdrs-backend/tdpservice/search_indexes/admin/filter.py index ecdc61b66..a6524633e 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filter.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filter.py @@ -1,19 +1,23 @@ - +"""Filter classes.""" from django.utils.translation import ugettext_lazy as _ from django.contrib.admin import SimpleListFilter class CreationDateFilter(SimpleListFilter): + """Simple filter class to show newest created datafile records.""" + title = _('Newest') parameter_name = 'created_at' def lookups(self, request, model_admin): + """Available options in dropdown.""" return ( (None, _('Newest')), ('all', _('All')), ) def choices(self, cl): + """Update query string based on selection.""" for lookup, title in self.lookup_choices: yield { 'selected': self.value() == lookup, @@ -24,7 +28,8 @@ def choices(self, cl): } def queryset(self, request, queryset): - if self.value() == None and len(queryset): + """Sort queryset to show latest records.""" + if self.value() is None and len(queryset): max_date = queryset.latest('created_at').created_at return queryset.filter(created_at=max_date) return queryset.order_by("-created_at") diff --git a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py index b3f26097a..6e9be7732 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py @@ -14,6 +14,7 @@ class SSP_M1DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance @@ -88,6 +89,8 @@ class SSP_M2DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance @@ -186,6 +189,7 @@ class SSP_M3DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index 6bdea2996..96dd3d72b 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -1,6 +1,5 @@ """Elasticsearch document mappings for TANF submission models.""" -import collections.abc as collections from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry from ..models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 @@ -15,6 +14,7 @@ class TANF_T1DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance @@ -91,6 +91,7 @@ class TANF_T2DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance @@ -192,6 +193,7 @@ class TANF_T3DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance @@ -245,6 +247,7 @@ class TANF_T4DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance @@ -291,6 +294,7 @@ class TANF_T5DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance @@ -353,6 +357,7 @@ class TANF_T6DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance @@ -403,6 +408,7 @@ class TANF_T7DataSubmissionDocument(Document): }) def get_instances_from_related(self, related_instance): + """Return correct instance.""" if isinstance(related_instance, DataFile): return related_instance From 4597d216b792a4f6dec4fc29a368f4b9cfec726d Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 6 Jul 2023 09:48:21 -0600 Subject: [PATCH 025/120] - Updated submission tests - Moved create_datafile to util --- .../tdpservice/parsers/test/test_parse.py | 20 +- tdrs-backend/tdpservice/parsers/util.py | 19 + .../search_indexes/test/test_model_mapping.py | 511 +++++++++--------- 3 files changed, 290 insertions(+), 260 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index c63d4b039..96ebd7dc5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2,31 +2,13 @@ import pytest -from pathlib import Path +from ..util import create_test_datafile from .. import parse from ..models import ParserError, ParserErrorCategoryChoices -from tdpservice.data_files.models import DataFile from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 -def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): - """Create a test DataFile instance with the given file attached.""" - path = str(Path(__file__).parent.joinpath('data')) + f'/{filename}' - datafile = DataFile.create_new_version({ - 'quarter': '4', - 'year': 2022, - 'section': section, - 'user': stt_user, - 'stt': stt - }) - - with open(path, 'rb') as file: - datafile.file.save(filename, file) - - return datafile - - @pytest.fixture def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 9d2a7a2ed..f9270c567 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -1,6 +1,25 @@ """Utility file for functions shared between all parsers even preparser.""" from .models import ParserError, ParserErrorCategoryChoices from django.contrib.contenttypes.models import ContentType +from tdpservice.data_files.models import DataFile +from pathlib import Path + + +def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): + """Create a test DataFile instance with the given file attached.""" + path = str(Path(__file__).parent.joinpath('test/data')) + f'/{filename}' + datafile = DataFile.create_new_version({ + 'quarter': '4', + 'year': 2022, + 'section': section, + 'user': stt_user, + 'stt': stt + }) + + with open(path, 'rb') as file: + datafile.file.save(filename, file) + + return datafile def value_is_empty(value, length): diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index 9e2b247d3..39b26f36b 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -5,63 +5,74 @@ from django.db.utils import IntegrityError from tdpservice.search_indexes import models from tdpservice.search_indexes import documents +from tdpservice.parsers.util import create_test_datafile fake = Faker() +@pytest.fixture +def test_datafile(stt_user, stt): + """Fixture for small_correct_file.""" + return create_test_datafile('small_correct_file', stt_user, stt) + + @pytest.mark.django_db -def test_can_create_and_index_tanf_t1_submission(): +def test_can_create_and_index_tanf_t1_submission(test_datafile): """TANF T1 submissions can be created and mapped.""" record_num = fake.uuid4() - submission = models.tanf.TANF_T1.objects.create( - RecordType=record_num, - RPT_MONTH_YEAR=1, - CASE_NUMBER=1, - COUNTY_FIPS_CODE=1, - STRATUM=1, - ZIP_CODE=1, - FUNDING_STREAM=1, - DISPOSITION=1, - NEW_APPLICANT=1, - NBR_FAMILY_MEMBERS=1, - FAMILY_TYPE=1, - RECEIVES_SUB_HOUSING=1, - RECEIVES_MED_ASSISTANCE=1, - RECEIVES_FOOD_STAMPS=1, - AMT_FOOD_STAMP_ASSISTANCE=1, - RECEIVES_SUB_CC=1, - AMT_SUB_CC=1, - CHILD_SUPPORT_AMT=1, - FAMILY_CASH_RESOURCES=1, - CASH_AMOUNT=1, - NBR_MONTHS=1, - CC_AMOUNT=1, - CHILDREN_COVERED=1, - CC_NBR_MONTHS=1, - TRANSP_AMOUNT=1, - TRANSP_NBR_MONTHS=1, - TRANSITION_SERVICES_AMOUNT=1, - TRANSITION_NBR_MONTHS=1, - OTHER_AMOUNT=1, - OTHER_NBR_MONTHS=1, - SANC_REDUCTION_AMT=1, - WORK_REQ_SANCTION=1, - FAMILY_SANC_ADULT=1, - SANC_TEEN_PARENT=1, - NON_COOPERATION_CSE=1, - FAILURE_TO_COMPLY=1, - OTHER_SANCTION=1, - RECOUPMENT_PRIOR_OVRPMT=1, - OTHER_TOTAL_REDUCTIONS=1, - FAMILY_CAP=1, - REDUCTIONS_ON_RECEIPTS=1, - OTHER_NON_SANCTION=1, - WAIVER_EVAL_CONTROL_GRPS=1, - FAMILY_EXEMPT_TIME_LIMITS=1, - FAMILY_NEW_CHILD=1, - ) + submission = models.tanf.TANF_T1() + submission.version=test_datafile.version + submission.created_at=test_datafile.created_at + submission.datafile=test_datafile + submission.RecordType=record_num + submission.RPT_MONTH_YEAR=1 + submission.CASE_NUMBER=1 + submission.COUNTY_FIPS_CODE=1 + submission.STRATUM=1 + submission.ZIP_CODE=1 + submission.FUNDING_STREAM=1 + submission.DISPOSITION=1 + submission.NEW_APPLICANT=1 + submission.NBR_FAMILY_MEMBERS=1 + submission.FAMILY_TYPE=1 + submission.RECEIVES_SUB_HOUSING=1 + submission.RECEIVES_MED_ASSISTANCE=1 + submission.RECEIVES_FOOD_STAMPS=1 + submission.AMT_FOOD_STAMP_ASSISTANCE=1 + submission.RECEIVES_SUB_CC=1 + submission.AMT_SUB_CC=1 + submission.CHILD_SUPPORT_AMT=1 + submission.FAMILY_CASH_RESOURCES=1 + submission.CASH_AMOUNT=1 + submission.NBR_MONTHS=1 + submission.CC_AMOUNT=1 + submission.CHILDREN_COVERED=1 + submission.CC_NBR_MONTHS=1 + submission.TRANSP_AMOUNT=1 + submission.TRANSP_NBR_MONTHS=1 + submission.TRANSITION_SERVICES_AMOUNT=1 + submission.TRANSITION_NBR_MONTHS=1 + submission.OTHER_AMOUNT=1 + submission.OTHER_NBR_MONTHS=1 + submission.SANC_REDUCTION_AMT=1 + submission.WORK_REQ_SANCTION=1 + submission.FAMILY_SANC_ADULT=1 + submission.SANC_TEEN_PARENT=1 + submission.NON_COOPERATION_CSE=1 + submission.FAILURE_TO_COMPLY=1 + submission.OTHER_SANCTION=1 + submission.RECOUPMENT_PRIOR_OVRPMT=1 + submission.OTHER_TOTAL_REDUCTIONS=1 + submission.FAMILY_CAP=1 + submission.REDUCTIONS_ON_RECEIPTS=1 + submission.OTHER_NON_SANCTION=1 + submission.WAIVER_EVAL_CONTROL_GRPS=1 + submission.FAMILY_EXEMPT_TIME_LIMITS=1 + submission.FAMILY_NEW_CHILD=1 + + submission.save() # submission.full_clean() @@ -77,82 +88,85 @@ def test_can_create_and_index_tanf_t1_submission(): @pytest.mark.django_db -def test_can_create_and_index_tanf_t2_submission(): +def test_can_create_and_index_tanf_t2_submission(test_datafile): """TANF T2 submissions can be created and mapped.""" record_num = fake.uuid4() - submission = models.tanf.TANF_T2.objects.create( - RecordType=record_num, - RPT_MONTH_YEAR=1, - CASE_NUMBER='1', - - FAMILY_AFFILIATION=1, - NONCUSTODIAL_PARENT=1, - DATE_OF_BIRTH=1, - SSN='1', - RACE_HISPANIC=1, - RACE_AMER_INDIAN=1, - RACE_ASIAN=1, - RACE_BLACK=1, - RACE_HAWAIIAN=1, - RACE_WHITE=1, - GENDER=1, - FED_OASDI_PROGRAM=1, - FED_DISABILITY_STATUS=1, - DISABLED_TITLE_XIVAPDT=1, - AID_AGED_BLIND=1, - RECEIVE_SSI=1, - MARITAL_STATUS=1, - RELATIONSHIP_HOH=1, - PARENT_WITH_MINOR_CHILD=1, - NEEDS_PREGNANT_WOMAN=1, - EDUCATION_LEVEL=1, - CITIZENSHIP_STATUS=1, - COOPERATION_CHILD_SUPPORT=1, - MONTHS_FED_TIME_LIMIT=1, - MONTHS_STATE_TIME_LIMIT=1, - CURRENT_MONTH_STATE_EXEMPT=1, - EMPLOYMENT_STATUS=1, - WORK_ELIGIBLE_INDICATOR=1, - WORK_PART_STATUS=1, - UNSUB_EMPLOYMENT=1, - SUB_PRIVATE_EMPLOYMENT=1, - SUB_PUBLIC_EMPLOYMENT=1, - WORK_EXPERIENCE_HOP=1, - WORK_EXPERIENCE_EA=1, - WORK_EXPERIENCE_HOL=1, - OJT=1, - JOB_SEARCH_HOP=1, - JOB_SEARCH_EA=1, - JOB_SEARCH_HOL=1, - COMM_SERVICES_HOP=1, - COMM_SERVICES_EA=1, - COMM_SERVICES_HOL=1, - VOCATIONAL_ED_TRAINING_HOP=1, - VOCATIONAL_ED_TRAINING_EA=1, - VOCATIONAL_ED_TRAINING_HOL=1, - JOB_SKILLS_TRAINING_HOP=1, - JOB_SKILLS_TRAINING_EA=1, - JOB_SKILLS_TRAINING_HOL=1, - ED_NO_HIGH_SCHOOL_DIPL_HOP=1, - ED_NO_HIGH_SCHOOL_DIPL_EA=1, - ED_NO_HIGH_SCHOOL_DIPL_HOL=1, - SCHOOL_ATTENDENCE_HOP=1, - SCHOOL_ATTENDENCE_EA=1, - SCHOOL_ATTENDENCE_HOL=1, - PROVIDE_CC_HOP=1, - PROVIDE_CC_EA=1, - PROVIDE_CC_HOL=1, - OTHER_WORK_ACTIVITIES=1, - DEEMED_HOURS_FOR_OVERALL=1, - DEEMED_HOURS_FOR_TWO_PARENT=1, - EARNED_INCOME=1, - UNEARNED_INCOME_TAX_CREDIT=1, - UNEARNED_SOCIAL_SECURITY=1, - UNEARNED_SSI=1, - UNEARNED_WORKERS_COMP=1, - OTHER_UNEARNED_INCOME=1 - ) + submission = models.tanf.TANF_T2() + submission.version=test_datafile.version + submission.created_at=test_datafile.created_at + submission.datafile=test_datafile + submission.RecordType=record_num + submission.RPT_MONTH_YEAR=1 + submission.CASE_NUMBER='1' + submission.FAMILY_AFFILIATION=1 + submission.NONCUSTODIAL_PARENT=1 + submission.DATE_OF_BIRTH=1 + submission.SSN='1' + submission.RACE_HISPANIC=1 + submission.RACE_AMER_INDIAN=1 + submission.RACE_ASIAN=1 + submission.RACE_BLACK=1 + submission.RACE_HAWAIIAN=1 + submission.RACE_WHITE=1 + submission.GENDER=1 + submission.FED_OASDI_PROGRAM=1 + submission.FED_DISABILITY_STATUS=1 + submission.DISABLED_TITLE_XIVAPDT=1 + submission.AID_AGED_BLIND=1 + submission.RECEIVE_SSI=1 + submission.MARITAL_STATUS=1 + submission.RELATIONSHIP_HOH=1 + submission.PARENT_WITH_MINOR_CHILD=1 + submission.NEEDS_PREGNANT_WOMAN=1 + submission.EDUCATION_LEVEL=1 + submission.CITIZENSHIP_STATUS=1 + submission.COOPERATION_CHILD_SUPPORT=1 + submission.MONTHS_FED_TIME_LIMIT=1 + submission.MONTHS_STATE_TIME_LIMIT=1 + submission.CURRENT_MONTH_STATE_EXEMPT=1 + submission.EMPLOYMENT_STATUS=1 + submission.WORK_ELIGIBLE_INDICATOR=1 + submission.WORK_PART_STATUS=1 + submission.UNSUB_EMPLOYMENT=1 + submission.SUB_PRIVATE_EMPLOYMENT=1 + submission.SUB_PUBLIC_EMPLOYMENT=1 + submission.WORK_EXPERIENCE_HOP=1 + submission.WORK_EXPERIENCE_EA=1 + submission.WORK_EXPERIENCE_HOL=1 + submission.OJT=1 + submission.JOB_SEARCH_HOP=1 + submission.JOB_SEARCH_EA=1 + submission.JOB_SEARCH_HOL=1 + submission.COMM_SERVICES_HOP=1 + submission.COMM_SERVICES_EA=1 + submission.COMM_SERVICES_HOL=1 + submission.VOCATIONAL_ED_TRAINING_HOP=1 + submission.VOCATIONAL_ED_TRAINING_EA=1 + submission.VOCATIONAL_ED_TRAINING_HOL=1 + submission.JOB_SKILLS_TRAINING_HOP=1 + submission.JOB_SKILLS_TRAINING_EA=1 + submission.JOB_SKILLS_TRAINING_HOL=1 + submission.ED_NO_HIGH_SCHOOL_DIPL_HOP=1 + submission.ED_NO_HIGH_SCHOOL_DIPL_EA=1 + submission.ED_NO_HIGH_SCHOOL_DIPL_HOL=1 + submission.SCHOOL_ATTENDENCE_HOP=1 + submission.SCHOOL_ATTENDENCE_EA=1 + submission.SCHOOL_ATTENDENCE_HOL=1 + submission.PROVIDE_CC_HOP=1 + submission.PROVIDE_CC_EA=1 + submission.PROVIDE_CC_HOL=1 + submission.OTHER_WORK_ACTIVITIES=1 + submission.DEEMED_HOURS_FOR_OVERALL=1 + submission.DEEMED_HOURS_FOR_TWO_PARENT=1 + submission.EARNED_INCOME=1 + submission.UNEARNED_INCOME_TAX_CREDIT=1 + submission.UNEARNED_SOCIAL_SECURITY=1 + submission.UNEARNED_SSI=1 + submission.UNEARNED_WORKERS_COMP=1 + submission.OTHER_UNEARNED_INCOME=1 + + submission.save() assert submission.id is not None @@ -166,34 +180,37 @@ def test_can_create_and_index_tanf_t2_submission(): @pytest.mark.django_db -def test_can_create_and_index_tanf_t3_submission(): +def test_can_create_and_index_tanf_t3_submission(test_datafile): """TANF T3 submissions can be created and mapped.""" record_num = fake.uuid4() - submission = models.tanf.TANF_T3.objects.create( - RecordType=record_num, - RPT_MONTH_YEAR=1, - CASE_NUMBER='1', - - FAMILY_AFFILIATION=1, - DATE_OF_BIRTH=1, - SSN='1', - RACE_HISPANIC=1, - RACE_AMER_INDIAN=1, - RACE_ASIAN=1, - RACE_BLACK=1, - RACE_HAWAIIAN=1, - RACE_WHITE=1, - GENDER=1, - RECEIVE_NONSSA_BENEFITS=1, - RECEIVE_SSI=1, - RELATIONSHIP_HOH=1, - PARENT_MINOR_CHILD=1, - EDUCATION_LEVEL=1, - CITIZENSHIP_STATUS=1, - UNEARNED_SSI=1, - OTHER_UNEARNED_INCOME=1, - ) + submission = models.tanf.TANF_T3() + submission.version=test_datafile.version + submission.created_at=test_datafile.created_at + submission.datafile=test_datafile + submission.RecordType=record_num + submission.RPT_MONTH_YEAR=1 + submission.CASE_NUMBER='1' + submission.FAMILY_AFFILIATION=1 + submission.DATE_OF_BIRTH=1 + submission.SSN='1' + submission.RACE_HISPANIC=1 + submission.RACE_AMER_INDIAN=1 + submission.RACE_ASIAN=1 + submission.RACE_BLACK=1 + submission.RACE_HAWAIIAN=1 + submission.RACE_WHITE=1 + submission.GENDER=1 + submission.RECEIVE_NONSSA_BENEFITS=1 + submission.RECEIVE_SSI=1 + submission.RELATIONSHIP_HOH=1 + submission.PARENT_MINOR_CHILD=1 + submission.EDUCATION_LEVEL=1 + submission.CITIZENSHIP_STATUS=1 + submission.UNEARNED_SSI=1 + submission.OTHER_UNEARNED_INCOME=1 + + submission.save() assert submission.id is not None @@ -207,26 +224,29 @@ def test_can_create_and_index_tanf_t3_submission(): @pytest.mark.django_db -def test_can_create_and_index_tanf_t4_submission(): +def test_can_create_and_index_tanf_t4_submission(test_datafile): """TANF T4 submissions can be created and mapped.""" record_num = fake.uuid4() - submission = models.tanf.TANF_T4.objects.create( - record=record_num, - rpt_month_year=1, - case_number='1', - disposition=1, - fips_code='1', - - county_fips_code='1', - stratum=1, - zip_code='1', - closure_reason=1, - rec_sub_housing=1, - rec_med_assist=1, - rec_food_stamps=1, - rec_sub_cc=1, - ) + submission = models.tanf.TANF_T4() + submission.version=test_datafile.version + submission.created_at=test_datafile.created_at + submission.datafile=test_datafile + submission.record=record_num + submission.rpt_month_year=1 + submission.case_number='1' + submission.disposition=1 + submission.fips_code='1' + submission.county_fips_code='1' + submission.stratum=1 + submission.zip_code='1' + submission.closure_reason=1 + submission.rec_sub_housing=1 + submission.rec_med_assist=1 + submission.rec_food_stamps=1 + submission.rec_sub_cc=1 + + submission.save() assert submission.id is not None @@ -240,43 +260,46 @@ def test_can_create_and_index_tanf_t4_submission(): @pytest.mark.django_db -def test_can_create_and_index_tanf_t5_submission(): +def test_can_create_and_index_tanf_t5_submission(test_datafile): """TANF T5 submissions can be created and mapped.""" record_num = fake.uuid4() - submission = models.tanf.TANF_T5.objects.create( - record=record_num, - rpt_month_year=1, - case_number='1', - fips_code='1', - - family_affiliation=1, - date_of_birth='1', - ssn='1', - race_hispanic=1, - race_amer_indian=1, - race_asian=1, - race_black=1, - race_hawaiian=1, - race_white=1, - gender=1, - rec_oasdi_insurance=1, - rec_federal_disability=1, - rec_aid_totally_disabled=1, - rec_aid_aged_blind=1, - rec_ssi=1, - marital_status=1, - relationship_hoh=1, - parent_minor_child=1, - needs_of_pregnant_woman=1, - education_level=1, - citizenship_status=1, - countable_month_fed_time=1, - countable_months_state_tribe=1, - employment_status=1, - amount_earned_income=1, - amount_unearned_income=1 - ) + submission = models.tanf.TANF_T5() + submission.version=test_datafile.version + submission.created_at=test_datafile.created_at + submission.datafile=test_datafile + submission.record=record_num + submission.rpt_month_year=1 + submission.case_number='1' + submission.fips_code='1' + submission.family_affiliation=1 + submission.date_of_birth='1' + submission.ssn='1' + submission.race_hispanic=1 + submission.race_amer_indian=1 + submission.race_asian=1 + submission.race_black=1 + submission.race_hawaiian=1 + submission.race_white=1 + submission.gender=1 + submission.rec_oasdi_insurance=1 + submission.rec_federal_disability=1 + submission.rec_aid_totally_disabled=1 + submission.rec_aid_aged_blind=1 + submission.rec_ssi=1 + submission.marital_status=1 + submission.relationship_hoh=1 + submission.parent_minor_child=1 + submission.needs_of_pregnant_woman=1 + submission.education_level=1 + submission.citizenship_status=1 + submission.countable_month_fed_time=1 + submission.countable_months_state_tribe=1 + submission.employment_status=1 + submission.amount_earned_income=1 + submission.amount_unearned_income=1 + + submission.save() assert submission.id is not None @@ -290,32 +313,35 @@ def test_can_create_and_index_tanf_t5_submission(): @pytest.mark.django_db -def test_can_create_and_index_tanf_t6_submission(): +def test_can_create_and_index_tanf_t6_submission(test_datafile): """TANF T6 submissions can be created and mapped.""" record_num = fake.uuid4() - submission = models.tanf.TANF_T6.objects.create( - record=record_num, - rpt_month_year=1, - fips_code='1', - - calendar_quarter=1, - applications=1, - approved=1, - denied=1, - assistance=1, - families=1, - num_2_parents=1, - num_1_parents=1, - num_no_parents=1, - recipients=1, - adult_recipients=1, - child_recipients=1, - noncustodials=1, - births=1, - outwedlock_births=1, - closed_cases=1, - ) + submission = models.tanf.TANF_T6() + submission.version=test_datafile.version + submission.created_at=test_datafile.created_at + submission.datafile=test_datafile + submission.record=record_num + submission.rpt_month_year=1 + submission.fips_code='1' + submission.calendar_quarter=1 + submission.applications=1 + submission.approved=1 + submission.denied=1 + submission.assistance=1 + submission.families=1 + submission.num_2_parents=1 + submission.num_1_parents=1 + submission.num_no_parents=1 + submission.recipients=1 + submission.adult_recipients=1 + submission.child_recipients=1 + submission.noncustodials=1 + submission.births=1 + submission.outwedlock_births=1 + submission.closed_cases=1 + + submission.save() assert submission.id is not None @@ -329,30 +355,33 @@ def test_can_create_and_index_tanf_t6_submission(): @pytest.mark.django_db -def test_can_create_and_index_tanf_t7_submission(): +def test_can_create_and_index_tanf_t7_submission(test_datafile): """TANF T7 submissions can be created and mapped.""" record_num = fake.uuid4() - submission = models.tanf.TANF_T7.objects.create( - record=record_num, - rpt_month_year=1, - fips_code='2', - - calendar_quarter=1, - tdrs_section_ind='1', - stratum='1', - families=1 - ) - - assert submission.id is not None - - search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( - 'match', - record=record_num - ) - response = search.execute() - - assert response.hits.total.value == 1 + submission = models.tanf.TANF_T7() + submission.version=test_datafile.version + submission.created_at=test_datafile.created_at + submission.datafile=test_datafile + submission.record=record_num + submission.rpt_month_year=1 + submission.fips_code='2' + submission.calendar_quarter=1 + submission.tdrs_section_ind='1' + submission.stratum='1' + submission.families=1 + + submission.save() + + # assert submission.id is not None + + # search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( + # 'match', + # record=record_num + # ) + # response = search.execute() + + # assert response.hits.total.value == 1 @pytest.mark.django_db From 7892115c8dfd920caea0b2160be7f2a194d484d7 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 6 Jul 2023 10:00:25 -0600 Subject: [PATCH 026/120] - fix lint errors --- .../search_indexes/test/test_model_mapping.py | 456 +++++++++--------- 1 file changed, 228 insertions(+), 228 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index 39b26f36b..f0ef0cfb6 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -23,55 +23,55 @@ def test_can_create_and_index_tanf_t1_submission(test_datafile): record_num = fake.uuid4() submission = models.tanf.TANF_T1() - submission.version=test_datafile.version - submission.created_at=test_datafile.created_at - submission.datafile=test_datafile - submission.RecordType=record_num - submission.RPT_MONTH_YEAR=1 - submission.CASE_NUMBER=1 - submission.COUNTY_FIPS_CODE=1 - submission.STRATUM=1 - submission.ZIP_CODE=1 - submission.FUNDING_STREAM=1 - submission.DISPOSITION=1 - submission.NEW_APPLICANT=1 - submission.NBR_FAMILY_MEMBERS=1 - submission.FAMILY_TYPE=1 - submission.RECEIVES_SUB_HOUSING=1 - submission.RECEIVES_MED_ASSISTANCE=1 - submission.RECEIVES_FOOD_STAMPS=1 - submission.AMT_FOOD_STAMP_ASSISTANCE=1 - submission.RECEIVES_SUB_CC=1 - submission.AMT_SUB_CC=1 - submission.CHILD_SUPPORT_AMT=1 - submission.FAMILY_CASH_RESOURCES=1 - submission.CASH_AMOUNT=1 - submission.NBR_MONTHS=1 - submission.CC_AMOUNT=1 - submission.CHILDREN_COVERED=1 - submission.CC_NBR_MONTHS=1 - submission.TRANSP_AMOUNT=1 - submission.TRANSP_NBR_MONTHS=1 - submission.TRANSITION_SERVICES_AMOUNT=1 - submission.TRANSITION_NBR_MONTHS=1 - submission.OTHER_AMOUNT=1 - submission.OTHER_NBR_MONTHS=1 - submission.SANC_REDUCTION_AMT=1 - submission.WORK_REQ_SANCTION=1 - submission.FAMILY_SANC_ADULT=1 - submission.SANC_TEEN_PARENT=1 - submission.NON_COOPERATION_CSE=1 - submission.FAILURE_TO_COMPLY=1 - submission.OTHER_SANCTION=1 - submission.RECOUPMENT_PRIOR_OVRPMT=1 - submission.OTHER_TOTAL_REDUCTIONS=1 - submission.FAMILY_CAP=1 - submission.REDUCTIONS_ON_RECEIPTS=1 - submission.OTHER_NON_SANCTION=1 - submission.WAIVER_EVAL_CONTROL_GRPS=1 - submission.FAMILY_EXEMPT_TIME_LIMITS=1 - submission.FAMILY_NEW_CHILD=1 - + submission.version = test_datafile.version + submission.created_at = test_datafile.created_at + submission.datafile = test_datafile + submission.RecordType = record_num + submission.RPT_MONTH_YEAR = 1 + submission.CASE_NUMBER = 1 + submission.COUNTY_FIPS_CODE = 1 + submission.STRATUM = 1 + submission.ZIP_CODE = 1 + submission.FUNDING_STREAM = 1 + submission.DISPOSITION = 1 + submission.NEW_APPLICANT = 1 + submission.NBR_FAMILY_MEMBERS = 1 + submission.FAMILY_TYPE = 1 + submission.RECEIVES_SUB_HOUSING = 1 + submission.RECEIVES_MED_ASSISTANCE = 1 + submission.RECEIVES_FOOD_STAMPS = 1 + submission.AMT_FOOD_STAMP_ASSISTANCE = 1 + submission.RECEIVES_SUB_CC = 1 + submission.AMT_SUB_CC = 1 + submission.CHILD_SUPPORT_AMT = 1 + submission.FAMILY_CASH_RESOURCES = 1 + submission.CASH_AMOUNT = 1 + submission.NBR_MONTHS = 1 + submission.CC_AMOUNT = 1 + submission.CHILDREN_COVERED = 1 + submission.CC_NBR_MONTHS = 1 + submission.TRANSP_AMOUNT = 1 + submission.TRANSP_NBR_MONTHS = 1 + submission.TRANSITION_SERVICES_AMOUNT = 1 + submission.TRANSITION_NBR_MONTHS = 1 + submission.OTHER_AMOUNT = 1 + submission.OTHER_NBR_MONTHS = 1 + submission.SANC_REDUCTION_AMT = 1 + submission.WORK_REQ_SANCTION = 1 + submission.FAMILY_SANC_ADULT = 1 + submission.SANC_TEEN_PARENT = 1 + submission.NON_COOPERATION_CSE = 1 + submission.FAILURE_TO_COMPLY = 1 + submission.OTHER_SANCTION = 1 + submission.RECOUPMENT_PRIOR_OVRPMT = 1 + submission.OTHER_TOTAL_REDUCTIONS = 1 + submission.FAMILY_CAP = 1 + submission.REDUCTIONS_ON_RECEIPTS = 1 + submission.OTHER_NON_SANCTION = 1 + submission.WAIVER_EVAL_CONTROL_GRPS = 1 + submission.FAMILY_EXEMPT_TIME_LIMITS = 1 + submission.FAMILY_NEW_CHILD = 1 + submission.save() # submission.full_clean() @@ -93,78 +93,78 @@ def test_can_create_and_index_tanf_t2_submission(test_datafile): record_num = fake.uuid4() submission = models.tanf.TANF_T2() - submission.version=test_datafile.version - submission.created_at=test_datafile.created_at - submission.datafile=test_datafile - submission.RecordType=record_num - submission.RPT_MONTH_YEAR=1 - submission.CASE_NUMBER='1' - submission.FAMILY_AFFILIATION=1 - submission.NONCUSTODIAL_PARENT=1 - submission.DATE_OF_BIRTH=1 - submission.SSN='1' - submission.RACE_HISPANIC=1 - submission.RACE_AMER_INDIAN=1 - submission.RACE_ASIAN=1 - submission.RACE_BLACK=1 - submission.RACE_HAWAIIAN=1 - submission.RACE_WHITE=1 - submission.GENDER=1 - submission.FED_OASDI_PROGRAM=1 - submission.FED_DISABILITY_STATUS=1 - submission.DISABLED_TITLE_XIVAPDT=1 - submission.AID_AGED_BLIND=1 - submission.RECEIVE_SSI=1 - submission.MARITAL_STATUS=1 - submission.RELATIONSHIP_HOH=1 - submission.PARENT_WITH_MINOR_CHILD=1 - submission.NEEDS_PREGNANT_WOMAN=1 - submission.EDUCATION_LEVEL=1 - submission.CITIZENSHIP_STATUS=1 - submission.COOPERATION_CHILD_SUPPORT=1 - submission.MONTHS_FED_TIME_LIMIT=1 - submission.MONTHS_STATE_TIME_LIMIT=1 - submission.CURRENT_MONTH_STATE_EXEMPT=1 - submission.EMPLOYMENT_STATUS=1 - submission.WORK_ELIGIBLE_INDICATOR=1 - submission.WORK_PART_STATUS=1 - submission.UNSUB_EMPLOYMENT=1 - submission.SUB_PRIVATE_EMPLOYMENT=1 - submission.SUB_PUBLIC_EMPLOYMENT=1 - submission.WORK_EXPERIENCE_HOP=1 - submission.WORK_EXPERIENCE_EA=1 - submission.WORK_EXPERIENCE_HOL=1 - submission.OJT=1 - submission.JOB_SEARCH_HOP=1 - submission.JOB_SEARCH_EA=1 - submission.JOB_SEARCH_HOL=1 - submission.COMM_SERVICES_HOP=1 - submission.COMM_SERVICES_EA=1 - submission.COMM_SERVICES_HOL=1 - submission.VOCATIONAL_ED_TRAINING_HOP=1 - submission.VOCATIONAL_ED_TRAINING_EA=1 - submission.VOCATIONAL_ED_TRAINING_HOL=1 - submission.JOB_SKILLS_TRAINING_HOP=1 - submission.JOB_SKILLS_TRAINING_EA=1 - submission.JOB_SKILLS_TRAINING_HOL=1 - submission.ED_NO_HIGH_SCHOOL_DIPL_HOP=1 - submission.ED_NO_HIGH_SCHOOL_DIPL_EA=1 - submission.ED_NO_HIGH_SCHOOL_DIPL_HOL=1 - submission.SCHOOL_ATTENDENCE_HOP=1 - submission.SCHOOL_ATTENDENCE_EA=1 - submission.SCHOOL_ATTENDENCE_HOL=1 - submission.PROVIDE_CC_HOP=1 - submission.PROVIDE_CC_EA=1 - submission.PROVIDE_CC_HOL=1 - submission.OTHER_WORK_ACTIVITIES=1 - submission.DEEMED_HOURS_FOR_OVERALL=1 - submission.DEEMED_HOURS_FOR_TWO_PARENT=1 - submission.EARNED_INCOME=1 - submission.UNEARNED_INCOME_TAX_CREDIT=1 - submission.UNEARNED_SOCIAL_SECURITY=1 - submission.UNEARNED_SSI=1 - submission.UNEARNED_WORKERS_COMP=1 - submission.OTHER_UNEARNED_INCOME=1 + submission.version = test_datafile.version + submission.created_at = test_datafile.created_at + submission.datafile = test_datafile + submission.RecordType = record_num + submission.RPT_MONTH_YEAR = 1 + submission.CASE_NUMBER = '1' + submission.FAMILY_AFFILIATION = 1 + submission.NONCUSTODIAL_PARENT = 1 + submission.DATE_OF_BIRTH = 1 + submission.SSN = '1' + submission.RACE_HISPANIC = 1 + submission.RACE_AMER_INDIAN = 1 + submission.RACE_ASIAN = 1 + submission.RACE_BLACK = 1 + submission.RACE_HAWAIIAN = 1 + submission.RACE_WHITE = 1 + submission.GENDER = 1 + submission.FED_OASDI_PROGRAM = 1 + submission.FED_DISABILITY_STATUS = 1 + submission.DISABLED_TITLE_XIVAPDT = 1 + submission.AID_AGED_BLIND = 1 + submission.RECEIVE_SSI = 1 + submission.MARITAL_STATUS = 1 + submission.RELATIONSHIP_HOH = 1 + submission.PARENT_WITH_MINOR_CHILD = 1 + submission.NEEDS_PREGNANT_WOMAN = 1 + submission.EDUCATION_LEVEL = 1 + submission.CITIZENSHIP_STATUS = 1 + submission.COOPERATION_CHILD_SUPPORT = 1 + submission.MONTHS_FED_TIME_LIMIT = 1 + submission.MONTHS_STATE_TIME_LIMIT = 1 + submission.CURRENT_MONTH_STATE_EXEMPT = 1 + submission.EMPLOYMENT_STATUS = 1 + submission.WORK_ELIGIBLE_INDICATOR = 1 + submission.WORK_PART_STATUS = 1 + submission.UNSUB_EMPLOYMENT = 1 + submission.SUB_PRIVATE_EMPLOYMENT = 1 + submission.SUB_PUBLIC_EMPLOYMENT = 1 + submission.WORK_EXPERIENCE_HOP = 1 + submission.WORK_EXPERIENCE_EA = 1 + submission.WORK_EXPERIENCE_HOL = 1 + submission.OJT = 1 + submission.JOB_SEARCH_HOP = 1 + submission.JOB_SEARCH_EA = 1 + submission.JOB_SEARCH_HOL = 1 + submission.COMM_SERVICES_HOP = 1 + submission.COMM_SERVICES_EA = 1 + submission.COMM_SERVICES_HOL = 1 + submission.VOCATIONAL_ED_TRAINING_HOP = 1 + submission.VOCATIONAL_ED_TRAINING_EA = 1 + submission.VOCATIONAL_ED_TRAINING_HOL = 1 + submission.JOB_SKILLS_TRAINING_HOP = 1 + submission.JOB_SKILLS_TRAINING_EA = 1 + submission.JOB_SKILLS_TRAINING_HOL = 1 + submission.ED_NO_HIGH_SCHOOL_DIPL_HOP = 1 + submission.ED_NO_HIGH_SCHOOL_DIPL_EA = 1 + submission.ED_NO_HIGH_SCHOOL_DIPL_HOL = 1 + submission.SCHOOL_ATTENDENCE_HOP = 1 + submission.SCHOOL_ATTENDENCE_EA = 1 + submission.SCHOOL_ATTENDENCE_HOL = 1 + submission.PROVIDE_CC_HOP = 1 + submission.PROVIDE_CC_EA = 1 + submission.PROVIDE_CC_HOL = 1 + submission.OTHER_WORK_ACTIVITIES = 1 + submission.DEEMED_HOURS_FOR_OVERALL = 1 + submission.DEEMED_HOURS_FOR_TWO_PARENT = 1 + submission.EARNED_INCOME = 1 + submission.UNEARNED_INCOME_TAX_CREDIT = 1 + submission.UNEARNED_SOCIAL_SECURITY = 1 + submission.UNEARNED_SSI = 1 + submission.UNEARNED_WORKERS_COMP = 1 + submission.OTHER_UNEARNED_INCOME = 1 submission.save() @@ -185,30 +185,30 @@ def test_can_create_and_index_tanf_t3_submission(test_datafile): record_num = fake.uuid4() submission = models.tanf.TANF_T3() - submission.version=test_datafile.version - submission.created_at=test_datafile.created_at - submission.datafile=test_datafile - submission.RecordType=record_num - submission.RPT_MONTH_YEAR=1 - submission.CASE_NUMBER='1' - submission.FAMILY_AFFILIATION=1 - submission.DATE_OF_BIRTH=1 - submission.SSN='1' - submission.RACE_HISPANIC=1 - submission.RACE_AMER_INDIAN=1 - submission.RACE_ASIAN=1 - submission.RACE_BLACK=1 - submission.RACE_HAWAIIAN=1 - submission.RACE_WHITE=1 - submission.GENDER=1 - submission.RECEIVE_NONSSA_BENEFITS=1 - submission.RECEIVE_SSI=1 - submission.RELATIONSHIP_HOH=1 - submission.PARENT_MINOR_CHILD=1 - submission.EDUCATION_LEVEL=1 - submission.CITIZENSHIP_STATUS=1 - submission.UNEARNED_SSI=1 - submission.OTHER_UNEARNED_INCOME=1 + submission.version = test_datafile.version + submission.created_at = test_datafile.created_at + submission.datafile = test_datafile + submission.RecordType = record_num + submission.RPT_MONTH_YEAR = 1 + submission.CASE_NUMBER = '1' + submission.FAMILY_AFFILIATION = 1 + submission.DATE_OF_BIRTH = 1 + submission.SSN = '1' + submission.RACE_HISPANIC = 1 + submission.RACE_AMER_INDIAN = 1 + submission.RACE_ASIAN = 1 + submission.RACE_BLACK = 1 + submission.RACE_HAWAIIAN = 1 + submission.RACE_WHITE = 1 + submission.GENDER = 1 + submission.RECEIVE_NONSSA_BENEFITS = 1 + submission.RECEIVE_SSI = 1 + submission.RELATIONSHIP_HOH = 1 + submission.PARENT_MINOR_CHILD = 1 + submission.EDUCATION_LEVEL = 1 + submission.CITIZENSHIP_STATUS = 1 + submission.UNEARNED_SSI = 1 + submission.OTHER_UNEARNED_INCOME = 1 submission.save() @@ -229,22 +229,22 @@ def test_can_create_and_index_tanf_t4_submission(test_datafile): record_num = fake.uuid4() submission = models.tanf.TANF_T4() - submission.version=test_datafile.version - submission.created_at=test_datafile.created_at - submission.datafile=test_datafile - submission.record=record_num - submission.rpt_month_year=1 - submission.case_number='1' - submission.disposition=1 - submission.fips_code='1' - submission.county_fips_code='1' - submission.stratum=1 - submission.zip_code='1' - submission.closure_reason=1 - submission.rec_sub_housing=1 - submission.rec_med_assist=1 - submission.rec_food_stamps=1 - submission.rec_sub_cc=1 + submission.version = test_datafile.version + submission.created_at = test_datafile.created_at + submission.datafile = test_datafile + submission.record = record_num + submission.rpt_month_year = 1 + submission.case_number = '1' + submission.disposition = 1 + submission.fips_code = '1' + submission.county_fips_code = '1' + submission.stratum = 1 + submission.zip_code = '1' + submission.closure_reason = 1 + submission.rec_sub_housing = 1 + submission.rec_med_assist = 1 + submission.rec_food_stamps = 1 + submission.rec_sub_cc = 1 submission.save() @@ -265,39 +265,39 @@ def test_can_create_and_index_tanf_t5_submission(test_datafile): record_num = fake.uuid4() submission = models.tanf.TANF_T5() - submission.version=test_datafile.version - submission.created_at=test_datafile.created_at - submission.datafile=test_datafile - submission.record=record_num - submission.rpt_month_year=1 - submission.case_number='1' - submission.fips_code='1' - submission.family_affiliation=1 - submission.date_of_birth='1' - submission.ssn='1' - submission.race_hispanic=1 - submission.race_amer_indian=1 - submission.race_asian=1 - submission.race_black=1 - submission.race_hawaiian=1 - submission.race_white=1 - submission.gender=1 - submission.rec_oasdi_insurance=1 - submission.rec_federal_disability=1 - submission.rec_aid_totally_disabled=1 - submission.rec_aid_aged_blind=1 - submission.rec_ssi=1 - submission.marital_status=1 - submission.relationship_hoh=1 - submission.parent_minor_child=1 - submission.needs_of_pregnant_woman=1 - submission.education_level=1 - submission.citizenship_status=1 - submission.countable_month_fed_time=1 - submission.countable_months_state_tribe=1 - submission.employment_status=1 - submission.amount_earned_income=1 - submission.amount_unearned_income=1 + submission.version = test_datafile.version + submission.created_at = test_datafile.created_at + submission.datafile = test_datafile + submission.record = record_num + submission.rpt_month_year = 1 + submission.case_number = '1' + submission.fips_code = '1' + submission.family_affiliation = 1 + submission.date_of_birth = '1' + submission.ssn = '1' + submission.race_hispanic = 1 + submission.race_amer_indian = 1 + submission.race_asian = 1 + submission.race_black = 1 + submission.race_hawaiian = 1 + submission.race_white = 1 + submission.gender = 1 + submission.rec_oasdi_insurance = 1 + submission.rec_federal_disability = 1 + submission.rec_aid_totally_disabled = 1 + submission.rec_aid_aged_blind = 1 + submission.rec_ssi = 1 + submission.marital_status = 1 + submission.relationship_hoh = 1 + submission.parent_minor_child = 1 + submission.needs_of_pregnant_woman = 1 + submission.education_level = 1 + submission.citizenship_status = 1 + submission.countable_month_fed_time = 1 + submission.countable_months_state_tribe = 1 + submission.employment_status = 1 + submission.amount_earned_income = 1 + submission.amount_unearned_income = 1 submission.save() @@ -318,29 +318,29 @@ def test_can_create_and_index_tanf_t6_submission(test_datafile): record_num = fake.uuid4() submission = models.tanf.TANF_T6() - submission.version=test_datafile.version - submission.created_at=test_datafile.created_at - submission.datafile=test_datafile - submission.record=record_num - submission.rpt_month_year=1 - submission.fips_code='1' - submission.calendar_quarter=1 - submission.applications=1 - submission.approved=1 - submission.denied=1 - submission.assistance=1 - submission.families=1 - submission.num_2_parents=1 - submission.num_1_parents=1 - submission.num_no_parents=1 - submission.recipients=1 - submission.adult_recipients=1 - submission.child_recipients=1 - submission.noncustodials=1 - submission.births=1 - submission.outwedlock_births=1 - submission.closed_cases=1 - + submission.version = test_datafile.version + submission.created_at = test_datafile.created_at + submission.datafile = test_datafile + submission.record = record_num + submission.rpt_month_year = 1 + submission.fips_code = '1' + submission.calendar_quarter = 1 + submission.applications = 1 + submission.approved = 1 + submission.denied = 1 + submission.assistance = 1 + submission.families = 1 + submission.num_2_parents = 1 + submission.num_1_parents = 1 + submission.num_no_parents = 1 + submission.recipients = 1 + submission.adult_recipients = 1 + submission.child_recipients = 1 + submission.noncustodials = 1 + submission.births = 1 + submission.outwedlock_births = 1 + submission.closed_cases = 1 + submission.save() assert submission.id is not None @@ -360,17 +360,17 @@ def test_can_create_and_index_tanf_t7_submission(test_datafile): record_num = fake.uuid4() submission = models.tanf.TANF_T7() - submission.version=test_datafile.version - submission.created_at=test_datafile.created_at - submission.datafile=test_datafile - submission.record=record_num - submission.rpt_month_year=1 - submission.fips_code='2' - submission.calendar_quarter=1 - submission.tdrs_section_ind='1' - submission.stratum='1' - submission.families=1 - + submission.version = test_datafile.version + submission.created_at = test_datafile.created_at + submission.datafile = test_datafile + submission.record = record_num + submission.rpt_month_year = 1 + submission.fips_code = '2' + submission.calendar_quarter = 1 + submission.tdrs_section_ind = '1' + submission.stratum = '1' + submission.families = 1 + submission.save() # assert submission.id is not None From 0c2d57b83309be0d54b72faad3d3bf6a8b0fe736 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 6 Jul 2023 11:43:13 -0600 Subject: [PATCH 027/120] - removing frontend filtering --- tdrs-backend/tdpservice/data_files/views.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 647d1121a..305985512 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -1,5 +1,4 @@ """Check if user is authorized.""" - import logging from django.http import FileResponse from django_filters import rest_framework as filters @@ -128,24 +127,6 @@ def get_queryset(self): else: queryset = queryset.exclude(section__contains='SSP') - q1 = queryset.filter(quarter='Q1') - if len(q1): - q1 = q1.filter(created_at=q1.latest('created_at').created_at) - - q2 = queryset.filter(quarter='Q2') - if len(q2): - q2 = q2.filter(created_at=q2.latest('created_at').created_at) - - q3 = queryset.filter(quarter='Q3') - if len(q3): - q3 = q3.filter(created_at=q3.latest('created_at').created_at) - - q4 = queryset.filter(quarter='Q4') - if len(q4): - q4 = q4.filter(created_at=q4.latest('created_at').created_at) - - queryset = q1 | q2 | q3 | q4 - return queryset def filter_queryset(self, queryset): From 35a6f24c36c3a4c00ddcfc40f20833530b0199f4 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 6 Jul 2023 13:36:44 -0600 Subject: [PATCH 028/120] - addding datafile to admin model --- tdrs-backend/tdpservice/search_indexes/admin/ssp.py | 6 ++++++ tdrs-backend/tdpservice/search_indexes/admin/tanf.py | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index 65cddb7ad..348771654 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -6,6 +6,8 @@ class SSP_M1Admin(admin.ModelAdmin): """ModelAdmin class for parsed M1 data files.""" list_display = [ + 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -25,6 +27,8 @@ class SSP_M2Admin(admin.ModelAdmin): """ModelAdmin class for parsed M2 data files.""" list_display = [ + 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -39,6 +43,8 @@ class SSP_M3Admin(admin.ModelAdmin): """ModelAdmin class for parsed M3 data files.""" list_display = [ + 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 9fa495e08..9c2dfd332 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -8,6 +8,7 @@ class TANF_T1Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -29,6 +30,7 @@ class TANF_T2Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -45,6 +47,7 @@ class TANF_T3Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -60,6 +63,8 @@ class TANF_T4Admin(admin.ModelAdmin): """ModelAdmin class for parsed T4 data files.""" list_display = [ + 'version', + 'datafile', 'record', 'rpt_month_year', 'case_number', @@ -74,6 +79,8 @@ class TANF_T5Admin(admin.ModelAdmin): """ModelAdmin class for parsed T5 data files.""" list_display = [ + 'version', + 'datafile', 'record', 'rpt_month_year', 'case_number', @@ -88,6 +95,8 @@ class TANF_T6Admin(admin.ModelAdmin): """ModelAdmin class for parsed T6 data files.""" list_display = [ + 'version', + 'datafile', 'record', 'rpt_month_year', ] @@ -101,6 +110,8 @@ class TANF_T7Admin(admin.ModelAdmin): """ModelAdmin class for parsed T7 data files.""" list_display = [ + 'version', + 'datafile', 'record', 'rpt_month_year', ] From 0146a0fced75a718f2eb879c6c7c191b9d7a2edb Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 6 Jul 2023 13:43:09 -0600 Subject: [PATCH 029/120] Revert "- addding datafile to admin model" This reverts commit 35a6f24c36c3a4c00ddcfc40f20833530b0199f4. --- tdrs-backend/tdpservice/search_indexes/admin/ssp.py | 6 ------ tdrs-backend/tdpservice/search_indexes/admin/tanf.py | 11 ----------- 2 files changed, 17 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index 348771654..65cddb7ad 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -6,8 +6,6 @@ class SSP_M1Admin(admin.ModelAdmin): """ModelAdmin class for parsed M1 data files.""" list_display = [ - 'version', - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -27,8 +25,6 @@ class SSP_M2Admin(admin.ModelAdmin): """ModelAdmin class for parsed M2 data files.""" list_display = [ - 'version', - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -43,8 +39,6 @@ class SSP_M3Admin(admin.ModelAdmin): """ModelAdmin class for parsed M3 data files.""" list_display = [ - 'version', - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 9c2dfd332..9fa495e08 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -8,7 +8,6 @@ class TANF_T1Admin(admin.ModelAdmin): list_display = [ 'version', - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -30,7 +29,6 @@ class TANF_T2Admin(admin.ModelAdmin): list_display = [ 'version', - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -47,7 +45,6 @@ class TANF_T3Admin(admin.ModelAdmin): list_display = [ 'version', - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -63,8 +60,6 @@ class TANF_T4Admin(admin.ModelAdmin): """ModelAdmin class for parsed T4 data files.""" list_display = [ - 'version', - 'datafile', 'record', 'rpt_month_year', 'case_number', @@ -79,8 +74,6 @@ class TANF_T5Admin(admin.ModelAdmin): """ModelAdmin class for parsed T5 data files.""" list_display = [ - 'version', - 'datafile', 'record', 'rpt_month_year', 'case_number', @@ -95,8 +88,6 @@ class TANF_T6Admin(admin.ModelAdmin): """ModelAdmin class for parsed T6 data files.""" list_display = [ - 'version', - 'datafile', 'record', 'rpt_month_year', ] @@ -110,8 +101,6 @@ class TANF_T7Admin(admin.ModelAdmin): """ModelAdmin class for parsed T7 data files.""" list_display = [ - 'version', - 'datafile', 'record', 'rpt_month_year', ] From d354f71ac22c885b0ba054ef7306d89ef7fd040f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 6 Jul 2023 13:50:14 -0600 Subject: [PATCH 030/120] - Fixed issue where datafile FK wasnt populating - Regenerated migration --- .../tdpservice/search_indexes/admin/ssp.py | 3 +++ .../tdpservice/search_indexes/admin/tanf.py | 4 ++++ ...230705_2138.py => 0014_auto_20230706_1946.py} | 16 ++++++++-------- .../tdpservice/search_indexes/models/tanf.py | 14 +++++++------- 4 files changed, 22 insertions(+), 15 deletions(-) rename tdrs-backend/tdpservice/search_indexes/migrations/{0014_auto_20230705_2138.py => 0014_auto_20230706_1946.py} (96%) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index 65cddb7ad..6b0056de2 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -6,6 +6,7 @@ class SSP_M1Admin(admin.ModelAdmin): """ModelAdmin class for parsed M1 data files.""" list_display = [ + 'version', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -25,6 +26,7 @@ class SSP_M2Admin(admin.ModelAdmin): """ModelAdmin class for parsed M2 data files.""" list_display = [ + 'version', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -39,6 +41,7 @@ class SSP_M3Admin(admin.ModelAdmin): """ModelAdmin class for parsed M3 data files.""" list_display = [ + 'version', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 9fa495e08..32547a8b5 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -60,6 +60,7 @@ class TANF_T4Admin(admin.ModelAdmin): """ModelAdmin class for parsed T4 data files.""" list_display = [ + 'version', 'record', 'rpt_month_year', 'case_number', @@ -74,6 +75,7 @@ class TANF_T5Admin(admin.ModelAdmin): """ModelAdmin class for parsed T5 data files.""" list_display = [ + 'version', 'record', 'rpt_month_year', 'case_number', @@ -88,6 +90,7 @@ class TANF_T6Admin(admin.ModelAdmin): """ModelAdmin class for parsed T6 data files.""" list_display = [ + 'version', 'record', 'rpt_month_year', ] @@ -101,6 +104,7 @@ class TANF_T7Admin(admin.ModelAdmin): """ModelAdmin class for parsed T7 data files.""" list_display = [ + 'version', 'record', 'rpt_month_year', ] diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230705_2138.py b/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230706_1946.py similarity index 96% rename from tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230705_2138.py rename to tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230706_1946.py index 3cac10921..d474c8602 100644 --- a/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230705_2138.py +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230706_1946.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.15 on 2023-07-05 21:38 +# Generated by Django 3.2.15 on 2023-07-06 19:46 from django.db import migrations, models import django.db.models.deletion @@ -64,7 +64,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='tanf_t1', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t1_parent', to='data_files.datafile'), ), migrations.AddField( @@ -79,7 +79,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='tanf_t2', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t2_parent', to='data_files.datafile'), ), migrations.AddField( @@ -94,7 +94,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='tanf_t3', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t3_parent', to='data_files.datafile'), ), migrations.AddField( @@ -109,7 +109,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='tanf_t4', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t4_parent', to='data_files.datafile'), ), migrations.AddField( @@ -124,7 +124,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='tanf_t5', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t5_parent', to='data_files.datafile'), ), migrations.AddField( @@ -139,7 +139,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='tanf_t6', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t6_parent', to='data_files.datafile'), ), migrations.AddField( @@ -154,7 +154,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='tanf_t7', - name='parent', + name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t7_parent', to='data_files.datafile'), ), migrations.AddField( diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index 1909ac605..e3b5503ec 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -17,7 +17,7 @@ class TANF_T1(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', @@ -89,7 +89,7 @@ class TANF_T2(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', @@ -180,7 +180,7 @@ class TANF_T3(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', @@ -223,7 +223,7 @@ class TANF_T4(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', @@ -262,7 +262,7 @@ class TANF_T5(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', @@ -314,7 +314,7 @@ class TANF_T6(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', @@ -355,7 +355,7 @@ class TANF_T7(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) version = models.IntegerField(null=True, blank=False) created_at = models.DateTimeField(null=True, blank=False) - parent = models.ForeignKey( + datafile = models.ForeignKey( DataFile, blank=True, help_text='The parent file from which this record was created.', From 2807425059fd1b5b355edfb16d30d170cf869d7b Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 6 Jul 2023 14:01:15 -0600 Subject: [PATCH 031/120] - Readding datafile back to admin view now that the error is resolved --- tdrs-backend/Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index 33c42b395..259ae2897 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -53,7 +53,7 @@ django-celery-beat = "==2.2.1" paramiko = "==2.11.0" pytest_sftpserver = "==1.3.0" elasticsearch = "==7.13.4" # REQUIRED - v7.14.0 introduces breaking changes -django-elasticsearch-dsl = "==7.1.1" +django-elasticsearch-dsl = "==7.3.3" django-elasticsearch-dsl-drf = "==0.22.5" requests-aws4auth = "==1.1.2" cerberus = "==1.3.4" From 6c5452868de673223950800f439c20701aab436f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 6 Jul 2023 14:01:45 -0600 Subject: [PATCH 032/120] - adding datafile back --- tdrs-backend/tdpservice/search_indexes/admin/ssp.py | 3 +++ tdrs-backend/tdpservice/search_indexes/admin/tanf.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index 6b0056de2..348771654 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -7,6 +7,7 @@ class SSP_M1Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -27,6 +28,7 @@ class SSP_M2Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -42,6 +44,7 @@ class SSP_M3Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 32547a8b5..9c2dfd332 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -8,6 +8,7 @@ class TANF_T1Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -29,6 +30,7 @@ class TANF_T2Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -45,6 +47,7 @@ class TANF_T3Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -61,6 +64,7 @@ class TANF_T4Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'record', 'rpt_month_year', 'case_number', @@ -76,6 +80,7 @@ class TANF_T5Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'record', 'rpt_month_year', 'case_number', @@ -91,6 +96,7 @@ class TANF_T6Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'record', 'rpt_month_year', ] @@ -105,6 +111,7 @@ class TANF_T7Admin(admin.ModelAdmin): list_display = [ 'version', + 'datafile', 'record', 'rpt_month_year', ] From d1edc20c8efbb4038c8deb69921da9e264a229c4 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 7 Jul 2023 09:31:20 -0600 Subject: [PATCH 033/120] Revert "- Readding datafile back to admin view now that the error is resolved" This reverts commit 2807425059fd1b5b355edfb16d30d170cf869d7b. --- tdrs-backend/Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index 259ae2897..33c42b395 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -53,7 +53,7 @@ django-celery-beat = "==2.2.1" paramiko = "==2.11.0" pytest_sftpserver = "==1.3.0" elasticsearch = "==7.13.4" # REQUIRED - v7.14.0 introduces breaking changes -django-elasticsearch-dsl = "==7.3.3" +django-elasticsearch-dsl = "==7.1.1" django-elasticsearch-dsl-drf = "==0.22.5" requests-aws4auth = "==1.1.2" cerberus = "==1.3.4" From f75fbca1432ea0f57e8ef3e302c010a12c6a0ebc Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 7 Jul 2023 14:23:07 -0600 Subject: [PATCH 034/120] - Removed unnecessary fields - Updated dependencies - Updated filter --- tdrs-backend/Dockerfile | 3 - tdrs-backend/Pipfile | 2 +- tdrs-backend/Pipfile.lock | 971 +++++++++--------- tdrs-backend/tdpservice/parsers/parse.py | 4 - .../tdpservice/search_indexes/admin/filter.py | 8 +- .../tdpservice/search_indexes/admin/ssp.py | 3 - .../tdpservice/search_indexes/admin/tanf.py | 7 - .../search_indexes/documents/ssp.py | 6 - .../search_indexes/documents/tanf.py | 18 - ...706_1946.py => 0014_auto_20230707_1952.py} | 102 +- .../tdpservice/search_indexes/models/ssp.py | 6 - .../tdpservice/search_indexes/models/tanf.py | 14 - 12 files changed, 488 insertions(+), 656 deletions(-) rename tdrs-backend/tdpservice/search_indexes/migrations/{0014_auto_20230706_1946.py => 0014_auto_20230707_1952.py} (53%) diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile index 78177e442..17eb68655 100644 --- a/tdrs-backend/Dockerfile +++ b/tdrs-backend/Dockerfile @@ -28,9 +28,6 @@ RUN groupadd -g ${gid} ${group} && useradd -u ${uid} -g ${group} -s /bin/sh ${us RUN chown -R tdpuser /tdpapp && chmod u+x gunicorn_start.sh wait_for_services.sh -# This is not a great solution to our serious version mismatches in our dependencies. It is however the lowest lift solution. -RUN sed -i 's/collections/collections.abc/g' /usr/local/lib/python3.10/site-packages/django_elasticsearch_dsl/fields.py - CMD ["./gunicorn_start.sh"] # if the container crashes/loops, we can shell into it by doing the following: # docker ps -a # to get the container id diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index 33c42b395..d0c70486a 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -53,7 +53,7 @@ django-celery-beat = "==2.2.1" paramiko = "==2.11.0" pytest_sftpserver = "==1.3.0" elasticsearch = "==7.13.4" # REQUIRED - v7.14.0 introduces breaking changes -django-elasticsearch-dsl = "==7.1.1" +django-elasticsearch-dsl = "==7.3" django-elasticsearch-dsl-drf = "==0.22.5" requests-aws4auth = "==1.1.2" cerberus = "==1.3.4" diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index 4180899be..214c1e06b 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1908db591c255893bdf412619ca7851b509f38167c2ea8e6a4f0378fd0183fb3" + "sha256": "46f5a4b6025ec6a2cd0fc0eb4ed23e1be550110a9117cfb03b95a523e5c0d4ae" }, "pipfile-spec": 6, "requires": { @@ -26,11 +26,11 @@ }, "asgiref": { "hashes": [ - "sha256:71e68008da809b957b7ee4b43dbccff33d1b23519fb8344e33f049897077afac", - "sha256:9567dfe7bd8d3c8c892227827c41cce860b368104c3431da67a0c5a65a949506" + "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e", + "sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed" ], "markers": "python_version >= '3.7'", - "version": "==3.6.0" + "version": "==3.7.2" }, "asttokens": { "hashes": [ @@ -90,11 +90,11 @@ }, "botocore": { "hashes": [ - "sha256:40406466f5c416b1f54bfbfc11aef90d783103f7ea77a1992dcaf1768ab04e12", - "sha256:783e7fa97bb5bf3759e4b333b8da2bcaffdb54828ea1d759b55329cc39003b98" + "sha256:6f35d59e230095aed7cd747604fe248fa384bebb7d09549077892f936a8ca3df", + "sha256:988b948be685006b43c4bbd8f5c0cb93e77c66deb70561994e0c5b31b5a67210" ], "markers": "python_version >= '3.7'", - "version": "==1.29.71" + "version": "==1.29.165" }, "celery": { "hashes": [ @@ -113,11 +113,11 @@ }, "certifi": { "hashes": [ - "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", - "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7", + "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716" ], "markers": "python_version >= '3.6'", - "version": "==2022.12.7" + "version": "==2023.5.7" }, "cffi": { "hashes": [ @@ -198,11 +198,11 @@ }, "click": { "hashes": [ - "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", - "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" + "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3", + "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37" ], "markers": "python_version >= '3.7'", - "version": "==8.1.3" + "version": "==8.1.4" }, "click-didyoumean": { "hashes": [ @@ -221,10 +221,11 @@ }, "click-repl": { "hashes": [ - "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b", - "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8" + "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", + "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812" ], - "version": "==0.2.0" + "markers": "python_version >= '3.6'", + "version": "==0.3.0" }, "coreapi": { "hashes": [ @@ -270,11 +271,11 @@ }, "deprecated": { "hashes": [ - "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d", - "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d" + "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c", + "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2.13" + "version": "==1.2.14" }, "dj-database-url": { "hashes": [ @@ -318,10 +319,10 @@ }, "django-colorfield": { "hashes": [ - "sha256:b7a25e2c430f4e80726ab171805d0d76d1f82cc7ff3799d60c2682f50d2ca03c", - "sha256:e50423af639858d01c39980eed9970f7ae475610bf0709b81f6b21a3dc9c3f92" + "sha256:208e04dae669ed95f18c4aef02865e9c785873521ffbe5b46fb6ea7a1500c979", + "sha256:7df5486de759dce70e54a1854f69bafd0a1e1bf98b66b09661d4d84342f447fc" ], - "version": "==0.8.0" + "version": "==0.9.0" }, "django-configurations": { "hashes": [ @@ -349,11 +350,11 @@ }, "django-elasticsearch-dsl": { "hashes": [ - "sha256:33dafbfe014829605176bdf79c7d08fd0be7a27232cbf3b0f0921d6026c57f63", - "sha256:ca0ff7489e18d680674a97cd8f3fb11ea239fb5dc1713abda48f799b8cf7c022" + "sha256:084a790edf26131a7897d13a3b5c08a6e129ca145b2b3fcbc2b2d4e902a919c1", + "sha256:c1be1c6d25ee6e99dc246282a4f6de8cc0630d0f8d9863ed512c0baa9d21be94" ], "index": "pypi", - "version": "==7.1.1" + "version": "==7.3" }, "django-elasticsearch-dsl-drf": { "hashes": [ @@ -456,11 +457,11 @@ }, "elasticsearch-dsl": { "hashes": [ - "sha256:046ea10820b94c075081b528b4526c5bc776bda4226d702f269a5f203232064b", - "sha256:c4a7b93882918a413b63bed54018a1685d7410ffd8facbc860ee7fd57f214a6d" + "sha256:07ee9c87dc28cc3cae2daa19401e1e18a172174ad9e5ca67938f752e3902a1d5", + "sha256:97f79239a252be7c4cce554c29e64695d7ef6a4828372316a5e5ff815e7a7498" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==7.4.0" + "version": "==7.4.1" }, "executing": { "hashes": [ @@ -487,11 +488,11 @@ }, "humanize": { "hashes": [ - "sha256:401201aca462749773f02920139f302450cb548b70489b9b4b92be39fe3c3c50", - "sha256:5f1f22bc65911eb1a6ffe7659bd6598e33dcfeeb904eb16ee1e705a09bf75916" + "sha256:7ca0e43e870981fa684acb5b062deb307218193bca1a01f2b2676479df849b3a", + "sha256:df7c429c2d27372b249d3f26eb53b07b166b661326e0325793e0a988082e3889" ], - "markers": "python_version >= '3.7'", - "version": "==4.6.0" + "markers": "python_version >= '3.8'", + "version": "==4.7.0" }, "idna": { "hashes": [ @@ -518,11 +519,11 @@ }, "ipython": { "hashes": [ - "sha256:b13a1d6c1f5818bd388db53b7107d17454129a70de2b87481d555daede5eb49e", - "sha256:b38c31e8fc7eff642fc7c597061fff462537cf2314e3225a19c906b7b0d8a345" + "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1", + "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf" ], "markers": "python_version >= '3.7'", - "version": "==8.10.0" + "version": "==8.14.0" }, "itypes": { "hashes": [ @@ -564,11 +565,11 @@ }, "kombu": { "hashes": [ - "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610", - "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4" + "sha256:48ee589e8833126fd01ceaa08f8a2041334e9f5894e5763c8486a550454551e9", + "sha256:fbd7572d92c0bf71c112a6b45163153dea5a7b6a701ec16b568c27d0fd2370f2" ], - "markers": "python_version >= '3.7'", - "version": "==5.2.4" + "markers": "python_version >= '3.8'", + "version": "==5.3.1" }, "markdown": { "hashes": [ @@ -580,59 +581,59 @@ }, "markupsafe": { "hashes": [ - "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed", - "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc", - "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2", - "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460", - "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7", - "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0", - "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1", - "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa", - "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03", - "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323", - "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65", - "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013", - "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036", - "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f", - "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4", - "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419", - "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2", - "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619", - "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a", - "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a", - "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd", - "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7", - "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666", - "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65", - "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859", - "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625", - "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff", - "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156", - "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd", - "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba", - "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f", - "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1", - "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094", - "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a", - "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513", - "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed", - "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d", - "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3", - "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147", - "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c", - "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603", - "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601", - "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a", - "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1", - "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d", - "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3", - "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54", - "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2", - "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6", - "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58" + "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", + "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", + "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", + "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", + "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", + "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", + "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", + "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", + "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", + "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", + "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", + "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", + "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", + "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", + "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", + "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", + "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", + "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", + "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", + "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", + "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", + "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", + "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", + "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", + "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", + "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", + "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", + "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", + "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", + "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", + "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", + "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", + "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", + "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", + "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", + "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", + "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", + "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", + "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", + "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", + "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", + "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" ], "markers": "python_version >= '3.7'", - "version": "==2.1.2" + "version": "==2.1.3" }, "matplotlib-inline": { "hashes": [ @@ -644,11 +645,11 @@ }, "packaging": { "hashes": [ - "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2", - "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97" + "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", + "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" ], "markers": "python_version >= '3.7'", - "version": "==23.0" + "version": "==23.1" }, "paramiko": { "hashes": [ @@ -683,102 +684,79 @@ }, "pillow": { "hashes": [ - "sha256:013016af6b3a12a2f40b704677f8b51f72cb007dac785a9933d5c86a72a7fe33", - "sha256:0845adc64fe9886db00f5ab68c4a8cd933ab749a87747555cec1c95acea64b0b", - "sha256:0884ba7b515163a1a05440a138adeb722b8a6ae2c2b33aea93ea3118dd3a899e", - "sha256:09b89ddc95c248ee788328528e6a2996e09eaccddeeb82a5356e92645733be35", - "sha256:0dd4c681b82214b36273c18ca7ee87065a50e013112eea7d78c7a1b89a739153", - "sha256:0e51f608da093e5d9038c592b5b575cadc12fd748af1479b5e858045fff955a9", - "sha256:0f3269304c1a7ce82f1759c12ce731ef9b6e95b6df829dccd9fe42912cc48569", - "sha256:16a8df99701f9095bea8a6c4b3197da105df6f74e6176c5b410bc2df2fd29a57", - "sha256:19005a8e58b7c1796bc0167862b1f54a64d3b44ee5d48152b06bb861458bc0f8", - "sha256:1b4b4e9dda4f4e4c4e6896f93e84a8f0bcca3b059de9ddf67dac3c334b1195e1", - "sha256:28676836c7796805914b76b1837a40f76827ee0d5398f72f7dcc634bae7c6264", - "sha256:2968c58feca624bb6c8502f9564dd187d0e1389964898f5e9e1fbc8533169157", - "sha256:3f4cc516e0b264c8d4ccd6b6cbc69a07c6d582d8337df79be1e15a5056b258c9", - "sha256:3fa1284762aacca6dc97474ee9c16f83990b8eeb6697f2ba17140d54b453e133", - "sha256:43521ce2c4b865d385e78579a082b6ad1166ebed2b1a2293c3be1d68dd7ca3b9", - "sha256:451f10ef963918e65b8869e17d67db5e2f4ab40e716ee6ce7129b0cde2876eab", - "sha256:46c259e87199041583658457372a183636ae8cd56dbf3f0755e0f376a7f9d0e6", - "sha256:46f39cab8bbf4a384ba7cb0bc8bae7b7062b6a11cfac1ca4bc144dea90d4a9f5", - "sha256:519e14e2c49fcf7616d6d2cfc5c70adae95682ae20f0395e9280db85e8d6c4df", - "sha256:53dcb50fbdc3fb2c55431a9b30caeb2f7027fcd2aeb501459464f0214200a503", - "sha256:54614444887e0d3043557d9dbc697dbb16cfb5a35d672b7a0fcc1ed0cf1c600b", - "sha256:575d8912dca808edd9acd6f7795199332696d3469665ef26163cd090fa1f8bfa", - "sha256:5dd5a9c3091a0f414a963d427f920368e2b6a4c2f7527fdd82cde8ef0bc7a327", - "sha256:5f532a2ad4d174eb73494e7397988e22bf427f91acc8e6ebf5bb10597b49c493", - "sha256:60e7da3a3ad1812c128750fc1bc14a7ceeb8d29f77e0a2356a8fb2aa8925287d", - "sha256:653d7fb2df65efefbcbf81ef5fe5e5be931f1ee4332c2893ca638c9b11a409c4", - "sha256:6663977496d616b618b6cfa43ec86e479ee62b942e1da76a2c3daa1c75933ef4", - "sha256:6abfb51a82e919e3933eb137e17c4ae9c0475a25508ea88993bb59faf82f3b35", - "sha256:6c6b1389ed66cdd174d040105123a5a1bc91d0aa7059c7261d20e583b6d8cbd2", - "sha256:6d9dfb9959a3b0039ee06c1a1a90dc23bac3b430842dcb97908ddde05870601c", - "sha256:765cb54c0b8724a7c12c55146ae4647e0274a839fb6de7bcba841e04298e1011", - "sha256:7a21222644ab69ddd9967cfe6f2bb420b460dae4289c9d40ff9a4896e7c35c9a", - "sha256:7ac7594397698f77bce84382929747130765f66406dc2cd8b4ab4da68ade4c6e", - "sha256:7cfc287da09f9d2a7ec146ee4d72d6ea1342e770d975e49a8621bf54eaa8f30f", - "sha256:83125753a60cfc8c412de5896d10a0a405e0bd88d0470ad82e0869ddf0cb3848", - "sha256:847b114580c5cc9ebaf216dd8c8dbc6b00a3b7ab0131e173d7120e6deade1f57", - "sha256:87708d78a14d56a990fbf4f9cb350b7d89ee8988705e58e39bdf4d82c149210f", - "sha256:8a2b5874d17e72dfb80d917213abd55d7e1ed2479f38f001f264f7ce7bae757c", - "sha256:8f127e7b028900421cad64f51f75c051b628db17fb00e099eb148761eed598c9", - "sha256:94cdff45173b1919350601f82d61365e792895e3c3a3443cf99819e6fbf717a5", - "sha256:99d92d148dd03fd19d16175b6d355cc1b01faf80dae93c6c3eb4163709edc0a9", - "sha256:9a3049a10261d7f2b6514d35bbb7a4dfc3ece4c4de14ef5876c4b7a23a0e566d", - "sha256:9d9a62576b68cd90f7075876f4e8444487db5eeea0e4df3ba298ee38a8d067b0", - "sha256:9e5f94742033898bfe84c93c831a6f552bb629448d4072dd312306bab3bd96f1", - "sha256:a1c2d7780448eb93fbcc3789bf3916aa5720d942e37945f4056680317f1cd23e", - "sha256:a2e0f87144fcbbe54297cae708c5e7f9da21a4646523456b00cc956bd4c65815", - "sha256:a4dfdae195335abb4e89cc9762b2edc524f3c6e80d647a9a81bf81e17e3fb6f0", - "sha256:a96e6e23f2b79433390273eaf8cc94fec9c6370842e577ab10dabdcc7ea0a66b", - "sha256:aabdab8ec1e7ca7f1434d042bf8b1e92056245fb179790dc97ed040361f16bfd", - "sha256:b222090c455d6d1a64e6b7bb5f4035c4dff479e22455c9eaa1bdd4c75b52c80c", - "sha256:b52ff4f4e002f828ea6483faf4c4e8deea8d743cf801b74910243c58acc6eda3", - "sha256:b70756ec9417c34e097f987b4d8c510975216ad26ba6e57ccb53bc758f490dab", - "sha256:b8c2f6eb0df979ee99433d8b3f6d193d9590f735cf12274c108bd954e30ca858", - "sha256:b9b752ab91e78234941e44abdecc07f1f0d8f51fb62941d32995b8161f68cfe5", - "sha256:ba6612b6548220ff5e9df85261bddc811a057b0b465a1226b39bfb8550616aee", - "sha256:bd752c5ff1b4a870b7661234694f24b1d2b9076b8bf337321a814c612665f343", - "sha256:c3c4ed2ff6760e98d262e0cc9c9a7f7b8a9f61aa4d47c58835cdaf7b0b8811bb", - "sha256:c5c1362c14aee73f50143d74389b2c158707b4abce2cb055b7ad37ce60738d47", - "sha256:cb362e3b0976dc994857391b776ddaa8c13c28a16f80ac6522c23d5257156bed", - "sha256:d197df5489004db87d90b918033edbeee0bd6df3848a204bca3ff0a903bef837", - "sha256:d3b56206244dc8711f7e8b7d6cad4663917cd5b2d950799425076681e8766286", - "sha256:d5b2f8a31bd43e0f18172d8ac82347c8f37ef3e0b414431157718aa234991b28", - "sha256:d7081c084ceb58278dd3cf81f836bc818978c0ccc770cbbb202125ddabec6628", - "sha256:db74f5562c09953b2c5f8ec4b7dfd3f5421f31811e97d1dbc0a7c93d6e3a24df", - "sha256:df41112ccce5d47770a0c13651479fbcd8793f34232a2dd9faeccb75eb5d0d0d", - "sha256:e1339790c083c5a4de48f688b4841f18df839eb3c9584a770cbd818b33e26d5d", - "sha256:e621b0246192d3b9cb1dc62c78cfa4c6f6d2ddc0ec207d43c0dedecb914f152a", - "sha256:e8c5cf126889a4de385c02a2c3d3aba4b00f70234bfddae82a5eaa3ee6d5e3e6", - "sha256:e9d7747847c53a16a729b6ee5e737cf170f7a16611c143d95aa60a109a59c336", - "sha256:eaef5d2de3c7e9b21f1e762f289d17b726c2239a42b11e25446abf82b26ac132", - "sha256:ed3e4b4e1e6de75fdc16d3259098de7c6571b1a6cc863b1a49e7d3d53e036070", - "sha256:ef21af928e807f10bf4141cad4746eee692a0dd3ff56cfb25fce076ec3cc8abe", - "sha256:f09598b416ba39a8f489c124447b007fe865f786a89dbfa48bb5cf395693132a", - "sha256:f0caf4a5dcf610d96c3bd32932bfac8aee61c96e60481c2a0ea58da435e25acd", - "sha256:f6e78171be3fb7941f9910ea15b4b14ec27725865a73c15277bc39f5ca4f8391", - "sha256:f715c32e774a60a337b2bb8ad9839b4abf75b267a0f18806f6f4f5f1688c4b5a", - "sha256:fb5c1ad6bad98c57482236a21bf985ab0ef42bd51f7ad4e4538e89a997624e12" - ], - "markers": "python_version >= '3.7'", - "version": "==9.4.0" + "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5", + "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530", + "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d", + "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891", + "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992", + "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7", + "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3", + "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba", + "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3", + "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3", + "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f", + "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538", + "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3", + "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d", + "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c", + "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017", + "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3", + "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223", + "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e", + "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3", + "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6", + "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640", + "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334", + "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1", + "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba", + "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa", + "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0", + "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396", + "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d", + "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485", + "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf", + "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43", + "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2", + "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd", + "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86", + "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967", + "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629", + "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568", + "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed", + "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f", + "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551", + "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3", + "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614", + "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff", + "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d", + "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883", + "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684", + "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0", + "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de", + "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b", + "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3", + "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199", + "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51", + "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90" + ], + "markers": "python_version >= '3.8'", + "version": "==10.0.0" }, "prometheus-client": { "hashes": [ - "sha256:0836af6eb2c8f4fed712b2f279f6c0a8bbab29f9f4aa15276b91c7cb0d1616ab", - "sha256:a03e35b359f14dd1630898543e2120addfdeacd1a6069c1367ae90fd93ad3f48" + "sha256:9c3b26f1535945e85b8934fb374678d263137b78ef85f305b1156c7c881cd11b", + "sha256:a77b708cf083f4d1a3fb3ce5c95b4afa32b9c521ae363354a4a910204ea095ce" ], "markers": "python_version >= '3.6'", - "version": "==0.16.0" + "version": "==0.17.0" }, "prompt-toolkit": { "hashes": [ - "sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63", - "sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305" + "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac", + "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88" ], - "markers": "python_full_version >= '3.6.2'", - "version": "==3.0.36" + "markers": "python_full_version >= '3.7.0'", + "version": "==3.0.39" }, "psycopg2-binary": { "hashes": [ @@ -868,11 +846,11 @@ }, "pygments": { "hashes": [ - "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297", - "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717" + "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c", + "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" ], - "markers": "python_version >= '3.6'", - "version": "==2.14.0" + "markers": "python_version >= '3.7'", + "version": "==2.15.1" }, "pyjwt": { "hashes": [ @@ -923,11 +901,11 @@ }, "pytz": { "hashes": [ - "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0", - "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a" + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" ], "index": "pypi", - "version": "==2022.7.1" + "version": "==2022.1" }, "redis": { "hashes": [ @@ -963,17 +941,18 @@ }, "ruamel.yaml": { "hashes": [ - "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7", - "sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af" + "sha256:23cd2ed620231677564646b0c6a89d138b6822a0d78656df7abda5879ec4f447", + "sha256:ec939063761914e14542972a5cba6d33c23b0859ab6342f61cf070cfc600efc2" ], "markers": "python_version >= '3'", - "version": "==0.17.21" + "version": "==0.17.32" }, "ruamel.yaml.clib": { "hashes": [ "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e", "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3", "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5", + "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81", "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497", "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f", "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac", @@ -991,6 +970,7 @@ "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb", "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f", "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307", + "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf", "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8", "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b", "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b", @@ -999,31 +979,33 @@ "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a", "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71", "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8", + "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122", "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7", "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80", "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e", "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab", "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0", - "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646" + "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646", + "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38" ], - "markers": "python_version < '3.11' and platform_python_implementation == 'CPython'", + "markers": "python_version < '3.12' and platform_python_implementation == 'CPython'", "version": "==0.2.7" }, "s3transfer": { "hashes": [ - "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", - "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346", + "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9" ], "markers": "python_version >= '3.7'", - "version": "==0.6.0" + "version": "==0.6.1" }, "setuptools": { "hashes": [ - "sha256:95f00380ef2ffa41d9bba85d95b27689d923c93dfbafed4aecd7cf988a25e012", - "sha256:bb6d8e508de562768f2027902929f8523932fcd1fb784e6d573d2cafac995a48" + "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", + "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235" ], "markers": "python_version >= '3.7'", - "version": "==67.3.2" + "version": "==68.0.0" }, "six": { "hashes": [ @@ -1035,11 +1017,11 @@ }, "sqlparse": { "hashes": [ - "sha256:0323c0ec29cd52bceabc1b4d9d579e311f3e4961b98d174201d5622a23b85e34", - "sha256:69ca804846bb114d2ec380e4360a8a340db83f0ccf3afceeb1404df028f57268" + "sha256:5430a4fe2ac7d0f93e66f1efc6e1338a41884b7ddf2a350cedd20ccc4d9d28f3", + "sha256:d446183e84b8349fa3061f0fe7f06ca94ba65b426946ffebe6e3e8295332420c" ], "markers": "python_version >= '3.5'", - "version": "==0.4.3" + "version": "==0.4.4" }, "stack-data": { "hashes": [ @@ -1058,20 +1040,20 @@ }, "tornado": { "hashes": [ - "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca", - "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72", - "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23", - "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8", - "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b", - "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9", - "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13", - "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75", - "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac", - "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", - "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" - ], - "markers": "python_version >= '3.7'", - "version": "==6.2" + "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4", + "sha256:0c325e66c8123c606eea33084976c832aa4e766b7dff8aedd7587ea44a604cdf", + "sha256:29e71c847a35f6e10ca3b5c2990a52ce38b233019d8e858b755ea6ce4dcdd19d", + "sha256:4b927c4f19b71e627b13f3db2324e4ae660527143f9e1f2e2fb404f3a187e2ba", + "sha256:5b17b1cf5f8354efa3d37c6e28fdfd9c1c1e5122f2cb56dac121ac61baa47cbe", + "sha256:6a0848f1aea0d196a7c4f6772197cbe2abc4266f836b0aac76947872cd29b411", + "sha256:7efcbcc30b7c654eb6a8c9c9da787a851c18f8ccd4a5a3a95b05c7accfa068d2", + "sha256:834ae7540ad3a83199a8da8f9f2d383e3c3d5130a328889e4cc991acc81e87a0", + "sha256:b46a6ab20f5c7c1cb949c72c1994a4585d2eaa0be4853f50a03b5031e964fc7c", + "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f", + "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829" + ], + "markers": "python_version >= '3.8'", + "version": "==6.3.2" }, "traitlets": { "hashes": [ @@ -1081,6 +1063,14 @@ "markers": "python_version >= '3.7'", "version": "==5.9.0" }, + "typing-extensions": { + "hashes": [ + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" + ], + "markers": "python_version < '3.11'", + "version": "==4.7.1" + }, "uritemplate": { "hashes": [ "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", @@ -1091,11 +1081,11 @@ }, "urllib3": { "hashes": [ - "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72", - "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1" + "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f", + "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.14" + "version": "==1.26.16" }, "vine": { "hashes": [ @@ -1122,73 +1112,84 @@ }, "wrapt": { "hashes": [ - "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3", - "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b", - "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4", - "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2", - "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656", - "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3", - "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff", - "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310", - "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a", - "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57", - "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069", - "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383", - "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe", - "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87", - "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d", - "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b", - "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907", - "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f", - "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0", - "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28", - "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1", - "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853", - "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc", - "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3", - "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3", - "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164", - "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1", - "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c", - "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1", - "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7", - "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1", - "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320", - "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed", - "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1", - "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248", - "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c", - "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456", - "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77", - "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef", - "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1", - "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7", - "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86", - "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4", - "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d", - "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d", - "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8", - "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5", - "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471", - "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00", - "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68", - "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3", - "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d", - "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735", - "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d", - "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569", - "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7", - "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59", - "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5", - "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb", - "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b", - "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f", - "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462", - "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015", - "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af" + "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0", + "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420", + "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a", + "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c", + "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079", + "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923", + "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f", + "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1", + "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8", + "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86", + "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0", + "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364", + "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e", + "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c", + "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e", + "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c", + "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727", + "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff", + "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e", + "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29", + "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7", + "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72", + "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475", + "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a", + "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317", + "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2", + "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd", + "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640", + "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98", + "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248", + "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e", + "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d", + "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec", + "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1", + "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e", + "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9", + "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92", + "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb", + "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094", + "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46", + "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29", + "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd", + "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705", + "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8", + "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975", + "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb", + "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e", + "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b", + "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418", + "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019", + "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1", + "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba", + "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6", + "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2", + "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3", + "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7", + "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752", + "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416", + "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f", + "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1", + "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc", + "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145", + "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee", + "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a", + "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7", + "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b", + "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653", + "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0", + "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90", + "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29", + "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6", + "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034", + "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09", + "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559", + "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==1.14.1" + "version": "==1.15.0" }, "xlsxwriter": { "hashes": [ @@ -1200,21 +1201,13 @@ } }, "develop": { - "attrs": { - "hashes": [ - "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836", - "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99" - ], - "markers": "python_version >= '3.6'", - "version": "==22.2.0" - }, "awscli": { "hashes": [ - "sha256:6784687cee79f3c9bb99428589f39925a5c56ca300e6efc61ad39766cb54bd2d", - "sha256:a9c3cb77117046fda527e3aac852045bf31e0e5cdc4d8850aaabc4280d8fb940" + "sha256:0d8670fe3e0c126ab86f01fcef3b7082be88b6dbb1ed2cd1697f28440a289879", + "sha256:7dd8647c53caf496bca5036fa1bdef99dc0307db30019be10f6f34a8d3ed7587" ], "index": "pypi", - "version": "==1.27.71" + "version": "==1.27.5" }, "awscli-local": { "hashes": [ @@ -1233,19 +1226,19 @@ }, "botocore": { "hashes": [ - "sha256:40406466f5c416b1f54bfbfc11aef90d783103f7ea77a1992dcaf1768ab04e12", - "sha256:783e7fa97bb5bf3759e4b333b8da2bcaffdb54828ea1d759b55329cc39003b98" + "sha256:6f35d59e230095aed7cd747604fe248fa384bebb7d09549077892f936a8ca3df", + "sha256:988b948be685006b43c4bbd8f5c0cb93e77c66deb70561994e0c5b31b5a67210" ], "markers": "python_version >= '3.7'", - "version": "==1.29.71" + "version": "==1.29.165" }, "click": { "hashes": [ - "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", - "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" + "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3", + "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37" ], "markers": "python_version >= '3.7'", - "version": "==8.1.3" + "version": "==8.1.4" }, "colorama": { "hashes": [ @@ -1260,60 +1253,69 @@ "toml" ], "hashes": [ - "sha256:04481245ef966fbd24ae9b9e537ce899ae584d521dfbe78f89cad003c38ca2ab", - "sha256:0c45948f613d5d18c9ec5eaa203ce06a653334cf1bd47c783a12d0dd4fd9c851", - "sha256:10188fe543560ec4874f974b5305cd1a8bdcfa885ee00ea3a03733464c4ca265", - "sha256:218fe982371ac7387304153ecd51205f14e9d731b34fb0568181abaf7b443ba0", - "sha256:29571503c37f2ef2138a306d23e7270687c0efb9cab4bd8038d609b5c2393a3a", - "sha256:2a60d6513781e87047c3e630b33b4d1e89f39836dac6e069ffee28c4786715f5", - "sha256:2bf1d5f2084c3932b56b962a683074a3692bce7cabd3aa023c987a2a8e7612f6", - "sha256:3164d31078fa9efe406e198aecd2a02d32a62fecbdef74f76dad6a46c7e48311", - "sha256:32df215215f3af2c1617a55dbdfb403b772d463d54d219985ac7cd3bf124cada", - "sha256:33d1ae9d4079e05ac4cc1ef9e20c648f5afabf1a92adfaf2ccf509c50b85717f", - "sha256:33ff26d0f6cc3ca8de13d14fde1ff8efe1456b53e3f0273e63cc8b3c84a063d8", - "sha256:38da2db80cc505a611938d8624801158e409928b136c8916cd2e203970dde4dc", - "sha256:3b155caf3760408d1cb903b21e6a97ad4e2bdad43cbc265e3ce0afb8e0057e73", - "sha256:3b946bbcd5a8231383450b195cfb58cb01cbe7f8949f5758566b881df4b33baf", - "sha256:3baf5f126f30781b5e93dbefcc8271cb2491647f8283f20ac54d12161dff080e", - "sha256:4b14d5e09c656de5038a3f9bfe5228f53439282abcab87317c9f7f1acb280352", - "sha256:51b236e764840a6df0661b67e50697aaa0e7d4124ca95e5058fa3d7cbc240b7c", - "sha256:63ffd21aa133ff48c4dff7adcc46b7ec8b565491bfc371212122dd999812ea1c", - "sha256:6a43c7823cd7427b4ed763aa7fb63901ca8288591323b58c9cd6ec31ad910f3c", - "sha256:755e89e32376c850f826c425ece2c35a4fc266c081490eb0a841e7c1cb0d3bda", - "sha256:7a726d742816cb3a8973c8c9a97539c734b3a309345236cd533c4883dda05b8d", - "sha256:7c7c0d0827e853315c9bbd43c1162c006dd808dbbe297db7ae66cd17b07830f0", - "sha256:7ed681b0f8e8bcbbffa58ba26fcf5dbc8f79e7997595bf071ed5430d8c08d6f3", - "sha256:7ee5c9bb51695f80878faaa5598040dd6c9e172ddcf490382e8aedb8ec3fec8d", - "sha256:8361be1c2c073919500b6601220a6f2f98ea0b6d2fec5014c1d9cfa23dd07038", - "sha256:8ae125d1134bf236acba8b83e74c603d1b30e207266121e76484562bc816344c", - "sha256:9817733f0d3ea91bea80de0f79ef971ae94f81ca52f9b66500c6a2fea8e4b4f8", - "sha256:98b85dd86514d889a2e3dd22ab3c18c9d0019e696478391d86708b805f4ea0fa", - "sha256:9ccb092c9ede70b2517a57382a601619d20981f56f440eae7e4d7eaafd1d1d09", - "sha256:9d58885215094ab4a86a6aef044e42994a2bd76a446dc59b352622655ba6621b", - "sha256:b643cb30821e7570c0aaf54feaf0bfb630b79059f85741843e9dc23f33aaca2c", - "sha256:bc7c85a150501286f8b56bd8ed3aa4093f4b88fb68c0843d21ff9656f0009d6a", - "sha256:beeb129cacea34490ffd4d6153af70509aa3cda20fdda2ea1a2be870dfec8d52", - "sha256:c31b75ae466c053a98bf26843563b3b3517b8f37da4d47b1c582fdc703112bc3", - "sha256:c4e4881fa9e9667afcc742f0c244d9364d197490fbc91d12ac3b5de0bf2df146", - "sha256:c5b15ed7644ae4bee0ecf74fee95808dcc34ba6ace87e8dfbf5cb0dc20eab45a", - "sha256:d12d076582507ea460ea2a89a8c85cb558f83406c8a41dd641d7be9a32e1274f", - "sha256:d248cd4a92065a4d4543b8331660121b31c4148dd00a691bfb7a5cdc7483cfa4", - "sha256:d47dd659a4ee952e90dc56c97d78132573dc5c7b09d61b416a9deef4ebe01a0c", - "sha256:d4a5a5879a939cb84959d86869132b00176197ca561c664fc21478c1eee60d75", - "sha256:da9b41d4539eefd408c46725fb76ecba3a50a3367cafb7dea5f250d0653c1040", - "sha256:db61a79c07331e88b9a9974815c075fbd812bc9dbc4dc44b366b5368a2936063", - "sha256:ddb726cb861c3117a553f940372a495fe1078249ff5f8a5478c0576c7be12050", - "sha256:ded59300d6330be27bc6cf0b74b89ada58069ced87c48eaf9344e5e84b0072f7", - "sha256:e2617759031dae1bf183c16cef8fcfb3de7617f394c813fa5e8e46e9b82d4222", - "sha256:e5cdbb5cafcedea04924568d990e20ce7f1945a1dd54b560f879ee2d57226912", - "sha256:ec8e767f13be637d056f7e07e61d089e555f719b387a7070154ad80a0ff31801", - "sha256:ef382417db92ba23dfb5864a3fc9be27ea4894e86620d342a116b243ade5d35d", - "sha256:f2cba5c6db29ce991029b5e4ac51eb36774458f0a3b8d3137241b32d1bb91f06", - "sha256:f5b4198d85a3755d27e64c52f8c95d6333119e49fd001ae5798dac872c95e0f8", - "sha256:ffeeb38ee4a80a30a6877c5c4c359e5498eec095878f1581453202bfacc8fbc2" + "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f", + "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2", + "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a", + "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a", + "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01", + "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6", + "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7", + "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f", + "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02", + "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c", + "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063", + "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a", + "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5", + "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959", + "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97", + "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6", + "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f", + "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9", + "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5", + "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f", + "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562", + "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe", + "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9", + "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f", + "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb", + "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb", + "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1", + "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb", + "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250", + "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e", + "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511", + "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5", + "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59", + "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2", + "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d", + "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3", + "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4", + "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de", + "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9", + "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833", + "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0", + "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9", + "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d", + "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050", + "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d", + "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6", + "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353", + "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb", + "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e", + "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8", + "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495", + "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2", + "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd", + "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27", + "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1", + "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818", + "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4", + "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e", + "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850", + "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3" ], "markers": "python_version >= '3.7'", - "version": "==7.1.0" + "version": "==7.2.7" }, "docutils": { "hashes": [ @@ -1325,11 +1327,11 @@ }, "exceptiongroup": { "hashes": [ - "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e", - "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23" + "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5", + "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f" ], "markers": "python_version < '3.11'", - "version": "==1.1.0" + "version": "==1.1.2" }, "factory-boy": { "hashes": [ @@ -1341,11 +1343,11 @@ }, "faker": { "hashes": [ - "sha256:17cf85aeb0363a3384ccd4c1f52b52ec8f414c7afaab74ae1f4c3e09a06e14de", - "sha256:21c3c6c45183308151c14f62afe59bf54ace68f663e0180973698ba2a9a3b2c4" + "sha256:801d1a2d71f1fc54d332de2ab19de7452454309937233ea2f7485402882d67b3", + "sha256:84bcf92bb725dd7341336eea4685df9a364f16f2470c4d29c1d7e6c5fd5a457d" ], "markers": "python_version >= '3.7'", - "version": "==17.0.0" + "version": "==18.13.0" }, "flake8": { "hashes": [ @@ -1412,9 +1414,9 @@ }, "localstack-client": { "hashes": [ - "sha256:71124983d15418c90ec9a82c4bde0460b29c62896cd44527b4b3346f8d5f8a89" + "sha256:3af9ab57d7744f64deb1912c1145b453db0233d8caaf6f71bd97380b5c4e45bb" ], - "version": "==1.39" + "version": "==2.2" }, "markdown": { "hashes": [ @@ -1426,59 +1428,59 @@ }, "markupsafe": { "hashes": [ - "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed", - "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc", - "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2", - "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460", - "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7", - "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0", - "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1", - "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa", - "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03", - "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323", - "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65", - "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013", - "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036", - "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f", - "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4", - "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419", - "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2", - "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619", - "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a", - "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a", - "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd", - "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7", - "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666", - "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65", - "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859", - "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625", - "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff", - "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156", - "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd", - "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba", - "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f", - "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1", - "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094", - "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a", - "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513", - "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed", - "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d", - "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3", - "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147", - "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c", - "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603", - "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601", - "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a", - "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1", - "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d", - "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3", - "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54", - "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2", - "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6", - "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58" + "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", + "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", + "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", + "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", + "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", + "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", + "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", + "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", + "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", + "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", + "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", + "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", + "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", + "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", + "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", + "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", + "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", + "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", + "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", + "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", + "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", + "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", + "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", + "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", + "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", + "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", + "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", + "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", + "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", + "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", + "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", + "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", + "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", + "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", + "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", + "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", + "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", + "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", + "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", + "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", + "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", + "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" ], "markers": "python_version >= '3.7'", - "version": "==2.1.2" + "version": "==2.1.3" }, "mccabe": { "hashes": [ @@ -1498,45 +1500,35 @@ }, "mkdocs": { "hashes": [ - "sha256:8947af423a6d0facf41ea1195b8e1e8c85ad94ac95ae307fe11232e0424b11c5", - "sha256:c8856a832c1e56702577023cd64cc5f84948280c1c0fcc6af4cd39006ea6aa8c" + "sha256:5955093bbd4dd2e9403c5afaf57324ad8b04f16886512a3ee6ef828956481c57", + "sha256:6ee46d309bda331aac915cd24aab882c179a933bd9e77b80ce7d2eaaa3f689dd" ], "index": "pypi", - "version": "==1.4.2" + "version": "==1.4.3" }, "packaging": { "hashes": [ - "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2", - "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97" + "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", + "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" ], "markers": "python_version >= '3.7'", - "version": "==23.0" + "version": "==23.1" }, "pluggy": { "hashes": [ - "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", - "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849", + "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3" ], - "markers": "python_version >= '3.6'", - "version": "==1.0.0" + "markers": "python_version >= '3.7'", + "version": "==1.2.0" }, "pyasn1": { "hashes": [ - "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359", - "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576", - "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf", - "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7", - "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", - "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00", - "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8", - "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86", - "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12", - "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776", - "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", - "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2", - "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3" - ], - "version": "==0.4.8" + "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57", + "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==0.5.0" }, "pycodestyle": { "hashes": [ @@ -1564,34 +1556,34 @@ }, "pygraphviz": { "hashes": [ - "sha256:457e093a888128903251a266a8cc16b4ba93f3f6334b3ebfed92c7471a74d867" + "sha256:a97eb5ced266f45053ebb1f2c6c6d29091690503e3a5c14be7f908b37b06f2d4" ], "index": "pypi", - "version": "==1.10" + "version": "==1.11" }, "pyparsing": { "hashes": [ - "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", - "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + "sha256:d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1", + "sha256:edb662d6fe322d6e990b1594b5feaeadf806803359e3d4d42f11e295e588f0ea" ], "index": "pypi", - "version": "==3.0.9" + "version": "==3.1.0" }, "pytest": { "hashes": [ - "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5", - "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42" + "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32", + "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a" ], "index": "pypi", - "version": "==7.2.1" + "version": "==7.4.0" }, "pytest-cov": { "hashes": [ - "sha256:2feb1b751d66a8bd934e5edfa2e961d11309dc37b73b0eabe73b5945fee20f6b", - "sha256:996b79efde6433cdbd0088872dbc5fb3ed7fe1578b68cdbba634f14bb8dd0470" + "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6", + "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a" ], "index": "pypi", - "version": "==4.0.0" + "version": "==4.1.0" }, "pytest-django": { "hashes": [ @@ -1611,11 +1603,11 @@ }, "pytest-mock": { "hashes": [ - "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b", - "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f" + "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39", + "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f" ], "index": "pypi", - "version": "==3.10.0" + "version": "==3.11.1" }, "python-dateutil": { "hashes": [ @@ -1678,11 +1670,11 @@ }, "s3transfer": { "hashes": [ - "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", - "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346", + "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9" ], "markers": "python_version >= '3.7'", - "version": "==0.6.0" + "version": "==0.6.1" }, "six": { "hashes": [ @@ -1709,53 +1701,52 @@ }, "typing-extensions": { "hashes": [ - "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", - "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4" + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" ], - "markers": "python_version >= '3.7'", - "version": "==4.5.0" + "markers": "python_version < '3.11'", + "version": "==4.7.1" }, "urllib3": { "hashes": [ - "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72", - "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1" + "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f", + "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.14" + "version": "==1.26.16" }, "watchdog": { "hashes": [ - "sha256:102a60093090fc3ff76c983367b19849b7cc24ec414a43c0333680106e62aae1", - "sha256:17f1708f7410af92ddf591e94ae71a27a13974559e72f7e9fde3ec174b26ba2e", - "sha256:195ab1d9d611a4c1e5311cbf42273bc541e18ea8c32712f2fb703cfc6ff006f9", - "sha256:4cb5ecc332112017fbdb19ede78d92e29a8165c46b68a0b8ccbd0a154f196d5e", - "sha256:5100eae58133355d3ca6c1083a33b81355c4f452afa474c2633bd2fbbba398b3", - "sha256:61fdb8e9c57baf625e27e1420e7ca17f7d2023929cd0065eb79c83da1dfbeacd", - "sha256:6ccd8d84b9490a82b51b230740468116b8205822ea5fdc700a553d92661253a3", - "sha256:6e01d699cd260d59b84da6bda019dce0a3353e3fcc774408ae767fe88ee096b7", - "sha256:748ca797ff59962e83cc8e4b233f87113f3cf247c23e6be58b8a2885c7337aa3", - "sha256:83a7cead445008e880dbde833cb9e5cc7b9a0958edb697a96b936621975f15b9", - "sha256:8586d98c494690482c963ffb24c49bf9c8c2fe0589cec4dc2f753b78d1ec301d", - "sha256:8b5cde14e5c72b2df5d074774bdff69e9b55da77e102a91f36ef26ca35f9819c", - "sha256:8c28c23972ec9c524967895ccb1954bc6f6d4a557d36e681a36e84368660c4ce", - "sha256:967636031fa4c4955f0f3f22da3c5c418aa65d50908d31b73b3b3ffd66d60640", - "sha256:96cbeb494e6cbe3ae6aacc430e678ce4b4dd3ae5125035f72b6eb4e5e9eb4f4e", - "sha256:978a1aed55de0b807913b7482d09943b23a2d634040b112bdf31811a422f6344", - "sha256:a09483249d25cbdb4c268e020cb861c51baab2d1affd9a6affc68ffe6a231260", - "sha256:a480d122740debf0afac4ddd583c6c0bb519c24f817b42ed6f850e2f6f9d64a8", - "sha256:adaf2ece15f3afa33a6b45f76b333a7da9256e1360003032524d61bdb4c422ae", - "sha256:bc43c1b24d2f86b6e1cc15f68635a959388219426109233e606517ff7d0a5a73", - "sha256:c27d8c1535fd4474e40a4b5e01f4ba6720bac58e6751c667895cbc5c8a7af33c", - "sha256:cdcc23c9528601a8a293eb4369cbd14f6b4f34f07ae8769421252e9c22718b6f", - "sha256:cece1aa596027ff56369f0b50a9de209920e1df9ac6d02c7f9e5d8162eb4f02b", - "sha256:d0f29fd9f3f149a5277929de33b4f121a04cf84bb494634707cfa8ea8ae106a8", - "sha256:d6b87477752bd86ac5392ecb9eeed92b416898c30bd40c7e2dd03c3146105646", - "sha256:e038be858425c4f621900b8ff1a3a1330d9edcfeaa1c0468aeb7e330fb87693e", - "sha256:e618a4863726bc7a3c64f95c218437f3349fb9d909eb9ea3a1ed3b567417c661", - "sha256:f8ac23ff2c2df4471a61af6490f847633024e5aa120567e08d07af5718c9d092" + "sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a", + "sha256:13bbbb462ee42ec3c5723e1205be8ced776f05b100e4737518c67c8325cf6100", + "sha256:233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8", + "sha256:25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc", + "sha256:2b57a1e730af3156d13b7fdddfc23dea6487fceca29fc75c5a868beed29177ae", + "sha256:336adfc6f5cc4e037d52db31194f7581ff744b67382eb6021c868322e32eef41", + "sha256:3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0", + "sha256:3ed7c71a9dccfe838c2f0b6314ed0d9b22e77d268c67e015450a29036a81f60f", + "sha256:4c9956d27be0bb08fc5f30d9d0179a855436e655f046d288e2bcc11adfae893c", + "sha256:4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9", + "sha256:4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3", + "sha256:5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709", + "sha256:51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83", + "sha256:5d9f3a10e02d7371cd929b5d8f11e87d4bad890212ed3901f9b4d68767bee759", + "sha256:7ade88d0d778b1b222adebcc0927428f883db07017618a5e684fd03b83342bd9", + "sha256:7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3", + "sha256:7e447d172af52ad204d19982739aa2346245cc5ba6f579d16dac4bfec226d2e7", + "sha256:8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f", + "sha256:8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346", + "sha256:9fac43a7466eb73e64a9940ac9ed6369baa39b3bf221ae23493a9ec4d0022674", + "sha256:a70a8dcde91be523c35b2bf96196edc5730edb347e374c7de7cd20c43ed95397", + "sha256:adfdeab2da79ea2f76f87eb42a3ab1966a5313e5a69a0213a3cc06ef692b0e96", + "sha256:ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d", + "sha256:c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a", + "sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64", + "sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44", + "sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33" ], - "markers": "python_version >= '3.6'", - "version": "==2.2.1" + "markers": "python_version >= '3.7'", + "version": "==3.0.0" } } } diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 864b40db0..17710a0f5 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -121,8 +121,6 @@ def parse_multi_record_line(line, schema, generate_error, datafile): record, record_is_valid, record_errors = r if record: - record.created_at = datafile.created_at - record.version = datafile.version record.datafile = datafile record.save() @@ -145,8 +143,6 @@ def parse_datafile_line(line, schema, generate_error, datafile): record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) if record: - record.created_at = datafile.created_at - record.version = datafile.version record.datafile = datafile record.save() diff --git a/tdrs-backend/tdpservice/search_indexes/admin/filter.py b/tdrs-backend/tdpservice/search_indexes/admin/filter.py index a6524633e..6aab618e8 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/filter.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/filter.py @@ -1,6 +1,7 @@ """Filter classes.""" from django.utils.translation import ugettext_lazy as _ from django.contrib.admin import SimpleListFilter +from tdpservice.data_files.models import DataFile class CreationDateFilter(SimpleListFilter): """Simple filter class to show newest created datafile records.""" @@ -30,6 +31,7 @@ def choices(self, cl): def queryset(self, request, queryset): """Sort queryset to show latest records.""" if self.value() is None and len(queryset): - max_date = queryset.latest('created_at').created_at - return queryset.filter(created_at=max_date) - return queryset.order_by("-created_at") + max_date = DataFile.objects.all().latest('created_at').created_at + datafile = DataFile.objects.get(created_at=max_date) + return queryset.filter(datafile=datafile) + return queryset diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index 348771654..59cc4709d 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -6,7 +6,6 @@ class SSP_M1Admin(admin.ModelAdmin): """ModelAdmin class for parsed M1 data files.""" list_display = [ - 'version', 'datafile', 'RecordType', 'RPT_MONTH_YEAR', @@ -27,7 +26,6 @@ class SSP_M2Admin(admin.ModelAdmin): """ModelAdmin class for parsed M2 data files.""" list_display = [ - 'version', 'datafile', 'RecordType', 'RPT_MONTH_YEAR', @@ -43,7 +41,6 @@ class SSP_M3Admin(admin.ModelAdmin): """ModelAdmin class for parsed M3 data files.""" list_display = [ - 'version', 'datafile', 'RecordType', 'RPT_MONTH_YEAR', diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index 9c2dfd332..b03313662 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -7,7 +7,6 @@ class TANF_T1Admin(admin.ModelAdmin): """ModelAdmin class for parsed T1 data files.""" list_display = [ - 'version', 'datafile', 'RecordType', 'RPT_MONTH_YEAR', @@ -29,7 +28,6 @@ class TANF_T2Admin(admin.ModelAdmin): """ModelAdmin class for parsed T2 data files.""" list_display = [ - 'version', 'datafile', 'RecordType', 'RPT_MONTH_YEAR', @@ -46,7 +44,6 @@ class TANF_T3Admin(admin.ModelAdmin): """ModelAdmin class for parsed T3 data files.""" list_display = [ - 'version', 'datafile', 'RecordType', 'RPT_MONTH_YEAR', @@ -63,7 +60,6 @@ class TANF_T4Admin(admin.ModelAdmin): """ModelAdmin class for parsed T4 data files.""" list_display = [ - 'version', 'datafile', 'record', 'rpt_month_year', @@ -79,7 +75,6 @@ class TANF_T5Admin(admin.ModelAdmin): """ModelAdmin class for parsed T5 data files.""" list_display = [ - 'version', 'datafile', 'record', 'rpt_month_year', @@ -95,7 +90,6 @@ class TANF_T6Admin(admin.ModelAdmin): """ModelAdmin class for parsed T6 data files.""" list_display = [ - 'version', 'datafile', 'record', 'rpt_month_year', @@ -110,7 +104,6 @@ class TANF_T7Admin(admin.ModelAdmin): """ModelAdmin class for parsed T7 data files.""" list_display = [ - 'version', 'datafile', 'record', 'rpt_month_year', diff --git a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py index 6e9be7732..ca7d1473c 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py @@ -32,8 +32,6 @@ class Django: model = SSP_M1 fields = [ - 'version', - 'created_at', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -108,8 +106,6 @@ class Django: model = SSP_M2 fields = [ - 'version', - 'created_at', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -207,8 +203,6 @@ class Django: model = SSP_M3 fields = [ - 'version', - 'created_at', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index 96dd3d72b..814ffacae 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -32,8 +32,6 @@ class Django: model = TANF_T1 fields = [ - 'version', - 'created_at', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -109,9 +107,6 @@ class Django: model = TANF_T2 fields = [ - - 'version', - 'created_at', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -211,9 +206,6 @@ class Django: model = TANF_T3 fields = [ - - 'version', - 'created_at', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', @@ -265,9 +257,6 @@ class Django: model = TANF_T4 fields = [ - - 'version', - 'created_at', 'record', 'rpt_month_year', 'case_number', @@ -312,9 +301,6 @@ class Django: model = TANF_T5 fields = [ - - 'version', - 'created_at', 'record', 'rpt_month_year', 'case_number', @@ -375,8 +361,6 @@ class Django: model = TANF_T6 fields = [ - 'version', - 'created_at', 'record', 'rpt_month_year', 'fips_code', @@ -426,8 +410,6 @@ class Django: model = TANF_T7 fields = [ - 'version', - 'created_at', 'record', 'rpt_month_year', 'fips_code', diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230706_1946.py b/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230707_1952.py similarity index 53% rename from tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230706_1946.py rename to tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230707_1952.py index d474c8602..8949911d1 100644 --- a/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230706_1946.py +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0014_auto_20230707_1952.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.15 on 2023-07-06 19:46 +# Generated by Django 3.2.15 on 2023-07-07 19:52 from django.db import migrations, models import django.db.models.deletion @@ -12,154 +12,54 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddField( - model_name='ssp_m1', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='ssp_m1', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m1_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='ssp_m1', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='ssp_m2', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='ssp_m2', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m2_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='ssp_m2', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='ssp_m3', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='ssp_m3', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='m3_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='ssp_m3', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='tanf_t1', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='tanf_t1', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t1_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='tanf_t1', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='tanf_t2', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='tanf_t2', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t2_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='tanf_t2', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='tanf_t3', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='tanf_t3', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t3_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='tanf_t3', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='tanf_t4', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='tanf_t4', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t4_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='tanf_t4', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='tanf_t5', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='tanf_t5', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t5_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='tanf_t5', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='tanf_t6', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='tanf_t6', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t6_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='tanf_t6', - name='version', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='tanf_t7', - name='created_at', - field=models.DateTimeField(null=True), - ), migrations.AddField( model_name='tanf_t7', name='datafile', field=models.ForeignKey(blank=True, help_text='The parent file from which this record was created.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='t7_parent', to='data_files.datafile'), ), - migrations.AddField( - model_name='tanf_t7', - name='version', - field=models.IntegerField(null=True), - ), ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index a1f5bd2f1..e61e07ed6 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -15,8 +15,6 @@ class SSP_M1(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, @@ -88,8 +86,6 @@ class SSP_M2(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, @@ -181,8 +177,6 @@ class SSP_M3(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, diff --git a/tdrs-backend/tdpservice/search_indexes/models/tanf.py b/tdrs-backend/tdpservice/search_indexes/models/tanf.py index e3b5503ec..8df02f99b 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/models/tanf.py @@ -15,8 +15,6 @@ class TANF_T1(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, @@ -87,8 +85,6 @@ class TANF_T2(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, @@ -178,8 +174,6 @@ class TANF_T3(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, @@ -221,8 +215,6 @@ class TANF_T4(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, @@ -260,8 +252,6 @@ class TANF_T5(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, @@ -312,8 +302,6 @@ class TANF_T6(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, @@ -353,8 +341,6 @@ class TANF_T7(models.Model): """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - version = models.IntegerField(null=True, blank=False) - created_at = models.DateTimeField(null=True, blank=False) datafile = models.ForeignKey( DataFile, blank=True, From 764e2126e2d2737abed7d394e6a9c2b0500aebb9 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Fri, 7 Jul 2023 15:00:05 -0600 Subject: [PATCH 035/120] - Updated document to include required fields --- .../search_indexes/documents/ssp.py | 9 ++++++++ .../search_indexes/documents/tanf.py | 21 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py index ca7d1473c..088dc6344 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py @@ -11,6 +11,9 @@ class SSP_M1DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): @@ -84,6 +87,9 @@ class SSP_M2DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): @@ -182,6 +188,9 @@ class SSP_M3DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index 814ffacae..9d7770b56 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -11,6 +11,9 @@ class TANF_T1DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): @@ -86,6 +89,9 @@ class TANF_T2DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): @@ -185,6 +191,9 @@ class TANF_T3DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): @@ -236,6 +245,9 @@ class TANF_T4DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): @@ -280,6 +292,9 @@ class TANF_T5DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): @@ -340,6 +355,9 @@ class TANF_T6DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): @@ -389,6 +407,9 @@ class TANF_T7DataSubmissionDocument(Document): datafile = fields.ObjectField(properties={ 'pk': fields.IntegerField(), + 'created_at': fields.DateField(), + 'version': fields.IntegerField(), + 'quarter': fields.TextField() }) def get_instances_from_related(self, related_instance): From 45f296d9646aeb2e1ad6e4d2c15a5d63df01ece9 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 09:15:23 -0600 Subject: [PATCH 036/120] - Fixed failing test --- tdrs-backend/tdpservice/parsers/test/test_summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 814fa2596..4f92ab865 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -44,7 +44,7 @@ def test_dfs_set_status(dfs): parser_errors = [] for i in range(2, 4): - parser_errors.append(ParserErrorFactory(row_number=i, category=str(i))) + parser_errors.append(ParserErrorFactory(row_number=i, error_type=str(i))) dfs.status = dfs.get_status(errors={'document': parser_errors}) From c0bce698754b102ae33dd6e74de769a2c230a8d7 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 5 May 2023 11:21:53 -0400 Subject: [PATCH 037/120] add adminUrl to deployment cypress overrides --- .circleci/deployment/jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index 078fb3172..dc1ee1d5b 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -43,7 +43,7 @@ app-dir: tdrs-frontend - run: name: Run Cypress e2e tests - command: cd tdrs-frontend; npm run test:e2e-ci -- --config baseUrl="https://tdp-frontend-develop.acf.hhs.gov" --env cypressToken=$CYPRESS_TOKEN,apiUrl="https://tdp-frontend-develop.acf.hhs.gov/v1" + command: cd tdrs-frontend; npm run test:e2e-ci -- --config baseUrl="https://tdp-frontend-develop.acf.hhs.gov" --env cypressToken=$CYPRESS_TOKEN,apiUrl="https://tdp-frontend-develop.acf.hhs.gov/v1",adminUrl="https://tdp-frontend-develop.acf.hhs.gov/admin" - store_artifacts: path: tdrs-frontend/cypress/screenshots/ - store_artifacts: From 57c179db5c89c30aa4a3846202def6a1b4f1b99f Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Fri, 2 Jun 2023 13:43:27 -0400 Subject: [PATCH 038/120] Adding "beta" banners to relevant error report sections (#2522) * Update views.py * Update views.py * Update SubmissionHistory.jsx * Update SubmissionHistory.test.js * Apply suggestions from code review Co-authored-by: Miles Reiter * lint fixes --------- Co-authored-by: Miles Reiter Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: andrew-jameson --- tdrs-backend/tdpservice/parsers/views.py | 6 ++++++ .../src/components/SubmissionHistory/SubmissionHistory.jsx | 2 +- .../components/SubmissionHistory/SubmissionHistory.test.js | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/views.py b/tdrs-backend/tdpservice/parsers/views.py index 90023597b..d39965ee3 100644 --- a/tdrs-backend/tdpservice/parsers/views.py +++ b/tdrs-backend/tdpservice/parsers/views.py @@ -52,6 +52,12 @@ def _get_xls_serialized_file(self, data): 'row_number', 'column_number'] + # write beta banner + worksheet.write(row, col, + "Error reporting in TDP is still in development." + + "We'll be in touch when it's ready to use!" + + "For now please refer to the reports you receive via email") + row, col = 2, 0 # write csv header [worksheet.write(row, col, key) for col, key in enumerate(report_columns)] diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index 7174f3999..844b0c744 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -78,7 +78,7 @@ const SectionSubmissionHistory = ({ section, label, files }) => { Submitted On Submitted By File Name - Error Reports + Error Reports (In development) diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js index 23d538df3..72f3e65f8 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js @@ -241,7 +241,10 @@ describe('SubmissionHistory', () => { expect(screen.queryByText('test5.txt')).not.toBeInTheDocument() expect(screen.queryByText('test6.txt')).toBeInTheDocument() - expect(screen.queryByText('Error Reports')).toBeInTheDocument() + expect( + screen.queryByText('Error Reports (In development)') + ).toBeInTheDocument() + expect(screen.queryByText('Currently Unavailable')).toBeInTheDocument() }) From 8e7d0e3917e37b107e0fcc8dd2773b0df0adf591 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Thu, 8 Jun 2023 07:30:21 -0700 Subject: [PATCH 039/120] Create sprint-73-summary.md (#2565) --- docs/Sprint-Review/sprint-73-summary.md | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/Sprint-Review/sprint-73-summary.md diff --git a/docs/Sprint-Review/sprint-73-summary.md b/docs/Sprint-Review/sprint-73-summary.md new file mode 100644 index 000000000..9d9c0c9b9 --- /dev/null +++ b/docs/Sprint-Review/sprint-73-summary.md @@ -0,0 +1,50 @@ +# Sprint 73 Summary + +05/10/23 - 5/23/23 + +Velocity: 10 + +## Sprint Goal +* Release v3.1.0. Continue parsing engine development and close out integration test epic (310). +* UX will maintain STT onboarding velocity, errors research sessions, provide copy for 2509 (e-mail notification for data submission and errors/transmission report) and iterate on communication method ahead of release v1/beta of downloadable errors report from 1610.1. +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + +## Tickets + +#### Completed/Merged +* [#2466 As sys admin, I need the API viewsets updated for the user, stt, and data file apps](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2466) +* [#1113 SSP Active Data (01)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1113) + + +#### Submitted (QASP Review, OCIO Review) +* [#2439 [Design] Embed YouTube videos in Knowledge Center](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2439) +* [#2424 Complete TANF section parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) + + +#### Closed (not merged) +* N/A + +### Moved to Next Sprint (Blocked, Raft Review, In Progress) + +#### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +#### Raft Review +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2503 [Design] May Knowledge Center Enhancements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2503) + +#### In Progress +* [#2116 [DevOps] Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2368 .csv errors reporting](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2368) + + +### Sprint 73 Demo +* Internal: + * [#2466 As sys admin, I need the API viewsets updated for the user, stt, and data file apps](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2466) + * [#1113 SSP Active Data (01)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1113) +* External: + * N/A From bd3558ff3b625a29ccf50b632082e0b7d25e4c83 Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Thu, 8 Jun 2023 11:09:07 -0400 Subject: [PATCH 040/120] hotfix for large file sizes (#2542) * hotfix for large file sizes * apply timeouts/req limits to dev * filter identity pages from scan * IGNORE sql injection --------- Co-authored-by: Jan Timpe Co-authored-by: mo sohani Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> --- scripts/zap-scanner.sh | 3 +++ tdrs-backend/gunicorn_dev_cfg.py | 2 +- tdrs-backend/gunicorn_prod_cfg.py | 2 +- tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf | 2 +- tdrs-frontend/nginx/local/default.conf.template | 4 ++-- tdrs-frontend/reports/zap.conf | 10 +++++----- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh index 2864746c3..4e017eb7e 100755 --- a/scripts/zap-scanner.sh +++ b/scripts/zap-scanner.sh @@ -65,6 +65,9 @@ ZAP_CLI_OPTIONS="\ -config globalexcludeurl.url_list.url\(2\).regex='^https?://.*\.amazonaws\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(2\).description='TDP S3 buckets' \ -config globalexcludeurl.url_list.url\(2\).enabled=true \ + -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.acf.hhs.gov\/v1\/login\/.*$' \ + -config globalexcludeurl.url_list.url\(3\).description='Site - identity pages' \ + -config globalexcludeurl.url_list.url\(3\).enabled=true \ -config spider.postform=true" # How long ZAP will crawl the app with the spider process diff --git a/tdrs-backend/gunicorn_dev_cfg.py b/tdrs-backend/gunicorn_dev_cfg.py index 6c4cd7254..04c04e6ee 100644 --- a/tdrs-backend/gunicorn_dev_cfg.py +++ b/tdrs-backend/gunicorn_dev_cfg.py @@ -21,4 +21,4 @@ # Daemonize the Gunicorn process (detach & enter background) # daemon = True -timeout = 10 +timeout = 100 diff --git a/tdrs-backend/gunicorn_prod_cfg.py b/tdrs-backend/gunicorn_prod_cfg.py index 3174beb2f..b6a16164c 100644 --- a/tdrs-backend/gunicorn_prod_cfg.py +++ b/tdrs-backend/gunicorn_prod_cfg.py @@ -21,4 +21,4 @@ # Daemonize the Gunicorn process (detach & enter background) # daemon = True -timeout = 10 +timeout = 100 diff --git a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf index a07fdcaaa..27f912245 100644 --- a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf +++ b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf @@ -40,7 +40,7 @@ http { access_log /home/vcap/app/nginx_access.log compression; include locations.conf; - client_max_body_size 50m; + client_max_body_size 100m; # Block all requests except ones listed in whitelist; disabled for local # First have to correct the source IP address using real_ip_header, otherwise diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template index c40cf188d..4e6e7e408 100644 --- a/tdrs-frontend/nginx/local/default.conf.template +++ b/tdrs-frontend/nginx/local/default.conf.template @@ -10,7 +10,7 @@ http { server { listen 80; - client_max_body_size 50m; + client_max_body_size 100m; root /usr/share/nginx/html; include locations.conf; @@ -84,7 +84,7 @@ http { add_header Content-Security-Policy $CSP; } - limit_req_zone $binary_remote_addr zone=limitreqsbyaddr:200m rate=100r/s; + limit_req_zone $binary_remote_addr zone=limitreqsbyaddr:20m rate=1000r/s; limit_req_status 444; # Logging format diff --git a/tdrs-frontend/reports/zap.conf b/tdrs-frontend/reports/zap.conf index 88bc3b7a3..854c0ea39 100644 --- a/tdrs-frontend/reports/zap.conf +++ b/tdrs-frontend/reports/zap.conf @@ -79,13 +79,13 @@ 40014 FAIL (Cross Site Scripting (Persistent) - Active/release) 40016 FAIL (Cross Site Scripting (Persistent) - Prime - Active/release) 40017 FAIL (Cross Site Scripting (Persistent) - Spider - Active/release) -40018 FAIL (SQL Injection - Active/release) -40019 FAIL (SQL Injection - MySQL - Active/beta) -40020 FAIL (SQL Injection - Hypersonic SQL - Active/beta) -40021 FAIL (SQL Injection - Oracle - Active/beta) +40018 IGNORE (SQL Injection - Active/release) +40019 IGNORE (SQL Injection - MySQL - Active/beta) +40020 IGNORE (SQL Injection - Hypersonic SQL - Active/beta) +40021 IGNORE (SQL Injection - Oracle - Active/beta) 40022 FAIL (SQL Injection - PostgreSQL - Active/beta) 40023 FAIL (Possible Username Enumeration - Active/beta) -40024 FAIL (SQL Injection - SQLite - Active/beta) +40024 IGNORE (SQL Injection - SQLite - Active/beta) 40025 IGNORE (Proxy Disclosure - Active/beta) 40026 FAIL (Cross Site Scripting (DOM Based) - Active/beta) 40027 FAIL (SQL Injection - MsSQL - Active/beta) From 2946dad1f04b4cca107e7681f0b9e0c26e3bf06f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 12 Jun 2023 09:29:41 -0400 Subject: [PATCH 041/120] updating validation error language --- tdrs-backend/tdpservice/parsers/parse.py | 31 +++----- .../tdpservice/parsers/schema_defs/header.py | 13 +++- .../tdpservice/parsers/schema_defs/trailer.py | 6 +- .../tdpservice/parsers/test/test_parse.py | 16 ++-- tdrs-backend/tdpservice/parsers/validators.py | 26 +++++-- tdrs-backend/tdpservice/test/test_backends.py | 78 +++++++++++++++++++ 6 files changed, 135 insertions(+), 35 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 3ee7dffad..66a18a2dc 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -3,7 +3,6 @@ import os from . import schema_defs, validators, util from .models import ParserErrorCategoryChoices -from tdpservice.data_files.models import DataFile def parse_datafile(datafile): @@ -45,30 +44,13 @@ def parse_datafile(datafile): errors['trailer'] = trailer_errors # ensure file section matches upload section - section_names = { - 'TAN': { - 'A': DataFile.Section.ACTIVE_CASE_DATA, - 'C': DataFile.Section.CLOSED_CASE_DATA, - 'G': DataFile.Section.AGGREGATE_DATA, - 'S': DataFile.Section.STRATUM_DATA, - }, - 'SSP': { - 'A': DataFile.Section.SSP_ACTIVE_CASE_DATA, - 'C': DataFile.Section.SSP_CLOSED_CASE_DATA, - 'G': DataFile.Section.SSP_AGGREGATE_DATA, - 'S': DataFile.Section.SSP_STRATUM_DATA, - }, - } - - # TODO: utility transformations between text to schemas and back - # text > prog > sections > schemas - program_type = header['program_type'] section = header['type'] section_is_valid, section_error = validators.validate_header_section_matches_submission( datafile, - section_names.get(program_type, {}).get(section) + program_type, + section, ) if not section_is_valid: @@ -189,6 +171,15 @@ def parse_datafile_line(line, schema, generate_error): record.save() return record_is_valid, record_errors + return (False, [ + generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Record Type is missing from record.", + record=None, + field=None + ) + ]) def get_schema_options(program_type): diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index c2b0b55c0..e57eccd63 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -5,10 +5,21 @@ from .. import validators +# def header_length_validator(length): +# return validators.make_validator( +# lambda value: len(value) == length, +# lambda value: f'Header length is {len(value)} but must be {length} characters.' +# ) + + header = RowSchema( model=dict, preparsing_validators=[ - validators.hasLength(23), + # header_length_validator(23), + validators.hasLength( + 23, + lambda value, length: f'Header length is {len(value)} but must be {length} characters.' + ), validators.startsWith('HEADER'), ], postparsing_validators=[], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 625b2042e..48bf4ccca 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -8,7 +8,11 @@ trailer = RowSchema( model=dict, preparsing_validators=[ - validators.hasLength(23), + # validators.hasLength(23), + validators.hasLength( + 23, + lambda value, length: f'Trailer length is {len(value)} but must be {length} characters.' + ), validators.startsWith('TRAILER') ], postparsing_validators=[], diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 00f2c90ee..dbab07165 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -78,7 +78,7 @@ def test_parse_section_mismatch(test_datafile, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Section does not match.' + assert err.error_message == 'Data does not match the expected layout for Closed Case Data.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -102,7 +102,7 @@ def test_parse_wrong_program_type(test_datafile, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Section does not match.' + assert err.error_message == 'Data does not match the expected layout for SSP Active Case Data.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -162,7 +162,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Value length 24 does not match 23.' + assert err.error_message == 'Header length is 24 but must be 23 characters.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -265,7 +265,7 @@ def test_parse_bad_trailer_file(bad_trailer_file): trailer_error = parser_errors.get(row_number=-1) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'Value length 11 does not match 23.' + assert trailer_error.error_message == 'Trailer length is 11 but must be 23 characters.' assert trailer_error.content_type is None assert trailer_error.object_id is None @@ -299,7 +299,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2): trailer_error_1 = trailer_errors.first() assert trailer_error_1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error_1.error_message == 'Value length 7 does not match 23.' + assert trailer_error_1.error_message == 'Trailer length is 7 but must be 23 characters.' assert trailer_error_1.content_type is None assert trailer_error_1.object_id is None @@ -379,7 +379,7 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile): assert err.row_number == -1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Value length 15 does not match 23.' + assert err.error_message == 'Trailer length is 15 but must be 23 characters.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -410,7 +410,7 @@ def ssp_section1_datafile(stt_user, stt): # trailer_error = parser_errors.get(row_number=-1) # assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK -# assert trailer_error.error_message == 'Value length 14 does not match 23.' +# assert trailer_error.error_message == 'Trailer length is 14 but must be 23 characters.' # row_12430_error = parser_errors.get(row_number=12430) # assert row_12430_error.error_type == ParserErrorCategoryChoices.PRE_CHECK @@ -591,7 +591,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel trailer_error = parser_errors.get(row_number=-1) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'Value length 15 does not match 23.' + assert trailer_error.error_message == 'Trailer length is 15 but must be 23 characters.' assert trailer_error.content_type is None assert trailer_error.object_id is None diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index bd7603c22..c3b58b8d1 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -2,6 +2,7 @@ from .util import generate_parser_error from .models import ParserErrorCategoryChoices +from tdpservice.data_files.models import DataFile # higher order validator func @@ -36,11 +37,11 @@ def between(min, max): ) -def hasLength(length): +def hasLength(length, error_func=None): """Validate that value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) == length, - lambda value: f'Value length {len(value)} does not match {length}.' + lambda value: error_func(value, length) if error_func else f'Value length {len(value)} does not match {length}.' ) @@ -116,9 +117,24 @@ def validate_single_header_trailer(datafile): return is_valid, error -def validate_header_section_matches_submission(datafile, section): +def validate_header_section_matches_submission(datafile, program_type, section): """Validate header section matches submission section.""" - is_valid = datafile.section == section + section_names = { + 'TAN': { + 'A': DataFile.Section.ACTIVE_CASE_DATA, + 'C': DataFile.Section.CLOSED_CASE_DATA, + 'G': DataFile.Section.AGGREGATE_DATA, + 'S': DataFile.Section.STRATUM_DATA, + }, + 'SSP': { + 'A': DataFile.Section.SSP_ACTIVE_CASE_DATA, + 'C': DataFile.Section.SSP_CLOSED_CASE_DATA, + 'G': DataFile.Section.SSP_AGGREGATE_DATA, + 'S': DataFile.Section.SSP_STRATUM_DATA, + }, + } + + is_valid = datafile.section == section_names.get(program_type, {}).get(section) error = None if not is_valid: @@ -127,7 +143,7 @@ def validate_header_section_matches_submission(datafile, section): line_number=1, schema=None, error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="Section does not match.", + error_message=f"Data does not match the expected layout for {datafile.section}.", record=None, field=None ) diff --git a/tdrs-backend/tdpservice/test/test_backends.py b/tdrs-backend/tdpservice/test/test_backends.py index 6e6a0b25d..ef7842c75 100644 --- a/tdrs-backend/tdpservice/test/test_backends.py +++ b/tdrs-backend/tdpservice/test/test_backends.py @@ -1,5 +1,6 @@ """Tests for storage backends available for use within tdpservice.""" import pytest +import math from tdpservice.backends import DataFilesS3Storage, StaticFilesS3Storage @@ -43,3 +44,80 @@ def test_datafiles_and_staticfiles_storages_have_distinct_credentials( """Test that the credentials used differ between backends.""" assert datafiles_backend.access_key != staticfiles_backend.access_key assert datafiles_backend.secret_key != staticfiles_backend.secret_key + + +char_map = [ + (1, 'I'), + (4, 'IV'), + (5, 'V'), + (9, 'IX'), + (10, 'X'), + (40, 'XL'), + (50, 'L'), + (90, 'XC'), + (100, 'C'), + (400, 'CD'), + (500, 'D'), + (900, 'CM'), + (1000, 'M'), +] + + +def arabic_to_roman(arabic): + for (a, r) in reversed(char_map): + if arabic == a: + return r + if arabic > a: + return f"{r}{arabic_to_roman(arabic - a)}" + + +@pytest.mark.parametrize('arabic,roman', [ + (1, 'I'), + (2, 'II'), + (3, 'III'), + (4, 'IV'), + (5, 'V'), + (6, 'VI'), + (7, 'VII'), + (8, 'VIII'), + (9, 'IX'), + (10, 'X'), + (11, 'XI'), + (20, 'XX'), + (23, 'XXIII'), + (24, 'XXIV'), + (25, 'XXV'), + (27, 'XXVII'), + (29, 'XXIX'), + (30, 'XXX'), + (39, 'XXXIX'), + (40, 'XL'), + (50, 'L'), + (60, 'LX'), + (70, 'LXX'), + (73, 'LXXIII'), + (80, 'LXXX'), + (90, 'XC'), + (100, 'C'), + (110, 'CX'), + (190, 'CXC'), + (200, 'CC'), + (300, 'CCC'), + (400, 'CD'), + (492, 'CDXCII'), + (500, 'D'), + (600, 'DC'), + (700, 'DCC'), + (800, 'DCCC'), + (900, 'CM'), + (1000, 'M'), + (1048, 'MXLVIII'), + (1100, 'MC'), + (2000, 'MM'), + (2001, 'MMI'), + (3000, 'MMM'), + (3549, 'MMMDXLIX'), + (3999, 'MMMCMXCIX'), +]) +def test_arabic_to_roman(arabic, roman): + assert arabic_to_roman(arabic) == roman From d9d6be90334fe1d22078df1c9df80e0057e73993 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 12 Jun 2023 09:37:24 -0400 Subject: [PATCH 042/120] accidentally included coding challenge --- tdrs-backend/tdpservice/test/test_backends.py | 78 ------------------- 1 file changed, 78 deletions(-) diff --git a/tdrs-backend/tdpservice/test/test_backends.py b/tdrs-backend/tdpservice/test/test_backends.py index ef7842c75..6e6a0b25d 100644 --- a/tdrs-backend/tdpservice/test/test_backends.py +++ b/tdrs-backend/tdpservice/test/test_backends.py @@ -1,6 +1,5 @@ """Tests for storage backends available for use within tdpservice.""" import pytest -import math from tdpservice.backends import DataFilesS3Storage, StaticFilesS3Storage @@ -44,80 +43,3 @@ def test_datafiles_and_staticfiles_storages_have_distinct_credentials( """Test that the credentials used differ between backends.""" assert datafiles_backend.access_key != staticfiles_backend.access_key assert datafiles_backend.secret_key != staticfiles_backend.secret_key - - -char_map = [ - (1, 'I'), - (4, 'IV'), - (5, 'V'), - (9, 'IX'), - (10, 'X'), - (40, 'XL'), - (50, 'L'), - (90, 'XC'), - (100, 'C'), - (400, 'CD'), - (500, 'D'), - (900, 'CM'), - (1000, 'M'), -] - - -def arabic_to_roman(arabic): - for (a, r) in reversed(char_map): - if arabic == a: - return r - if arabic > a: - return f"{r}{arabic_to_roman(arabic - a)}" - - -@pytest.mark.parametrize('arabic,roman', [ - (1, 'I'), - (2, 'II'), - (3, 'III'), - (4, 'IV'), - (5, 'V'), - (6, 'VI'), - (7, 'VII'), - (8, 'VIII'), - (9, 'IX'), - (10, 'X'), - (11, 'XI'), - (20, 'XX'), - (23, 'XXIII'), - (24, 'XXIV'), - (25, 'XXV'), - (27, 'XXVII'), - (29, 'XXIX'), - (30, 'XXX'), - (39, 'XXXIX'), - (40, 'XL'), - (50, 'L'), - (60, 'LX'), - (70, 'LXX'), - (73, 'LXXIII'), - (80, 'LXXX'), - (90, 'XC'), - (100, 'C'), - (110, 'CX'), - (190, 'CXC'), - (200, 'CC'), - (300, 'CCC'), - (400, 'CD'), - (492, 'CDXCII'), - (500, 'D'), - (600, 'DC'), - (700, 'DCC'), - (800, 'DCCC'), - (900, 'CM'), - (1000, 'M'), - (1048, 'MXLVIII'), - (1100, 'MC'), - (2000, 'MM'), - (2001, 'MMI'), - (3000, 'MMM'), - (3549, 'MMMDXLIX'), - (3999, 'MMMCMXCIX'), -]) -def test_arabic_to_roman(arabic, roman): - assert arabic_to_roman(arabic) == roman From 5cdb8869a721df31e5cdf7caf9933db9965d7f9c Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 12 Jun 2023 11:53:31 -0400 Subject: [PATCH 043/120] rm comments --- tdrs-backend/tdpservice/parsers/schema_defs/header.py | 8 -------- tdrs-backend/tdpservice/parsers/schema_defs/trailer.py | 1 - 2 files changed, 9 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index e57eccd63..b51c433ed 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -5,17 +5,9 @@ from .. import validators -# def header_length_validator(length): -# return validators.make_validator( -# lambda value: len(value) == length, -# lambda value: f'Header length is {len(value)} but must be {length} characters.' -# ) - - header = RowSchema( model=dict, preparsing_validators=[ - # header_length_validator(23), validators.hasLength( 23, lambda value, length: f'Header length is {len(value)} but must be {length} characters.' diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 48bf4ccca..3657cfc51 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -8,7 +8,6 @@ trailer = RowSchema( model=dict, preparsing_validators=[ - # validators.hasLength(23), validators.hasLength( 23, lambda value, length: f'Trailer length is {len(value)} but must be {length} characters.' From fe32cdcebff6238b91389974ede290249799714a Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 20 Jun 2023 07:55:04 -0600 Subject: [PATCH 044/120] 2550 deactivation email link (#2557) * - updated nginx buildpack * - specifying different nginx version * - Updating changelog * - added script to update certain apps in cf - added workflow for each environment in circi * - fixed base config * - fixing jobs * - Updated based on feedback in OH * - Updating defaults * - Removing defaults * - Fixing indent * - Adding params to config * test * test * - updating work dir * - Adding checkout * - adding cf check * - logging into cf * - update cf check to install required binary * - removing unnecessary switch * - Forcing plugin installation * - test installing plugin from script also * - Adding url to email * - test code for sandbox * - using my email * Revert "Merge branch 'update-cf-os' into 2551-deactivation-email-link" This reverts commit e963b9df48dd1f72ca0c5b192c979bac11851d11, reversing changes made to cc9cf81e9d76c42f51ffd5e102f6027d3eb5e645. * Revert "- using my email" This reverts commit cc9cf81e9d76c42f51ffd5e102f6027d3eb5e645. * Revert "- test code for sandbox" This reverts commit 06037747197d17ed8e63b086fcfcf048ecb50dc4. --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../tdpservice/email/helpers/account_deactivation_warning.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py b/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py index 53b67bbc5..2a184e93a 100644 --- a/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py +++ b/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py @@ -2,6 +2,7 @@ from tdpservice.email.email_enums import EmailType from tdpservice.email.email import automated_email from datetime import datetime, timedelta, timezone +from django.conf import settings def send_deactivation_warning_email(users, days): @@ -16,7 +17,8 @@ def send_deactivation_warning_email(users, days): context = { 'first_name': user.first_name, 'days': days, - 'deactivation_date': deactivation_date + 'deactivation_date': deactivation_date, + 'url': f'{settings.FRONTEND_BASE_URL}/login/' } logger_context = { From c26703ea836d2fdd17739e5dd5d779fa4710a514 Mon Sep 17 00:00:00 2001 From: Lauren Frohlich <61251539+lfrohlich@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:05:42 -0400 Subject: [PATCH 045/120] Update README.md (#2577) Add ATO Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Security-Compliance/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Security-Compliance/README.md b/docs/Security-Compliance/README.md index 9cb4c4f4e..c55f72986 100644 --- a/docs/Security-Compliance/README.md +++ b/docs/Security-Compliance/README.md @@ -2,6 +2,8 @@ This subdirectory contains documentation that describes our practices as it relates to keeping the TDP system in compliance with security requirements for [FISMA moderate systems](https://csrc.nist.gov/CSRC/media/Projects/risk-management/documents/02-Categorize%20Step/NIST%20RMF%20Categorize%20Step-FAQs.pdf) (*Note: see section 18 for definition*) and other related federal system regulations. +The TANF Data Portal received its Authority to Operate (ATO) from the ACF Chief Information Officer on May 18, 2021. The ATO was extended on March 29, 2023 through May 24, 2024. + ## Table of Contents * **[Security Controls](./Security-Controls)** - Herein you will find information about TDP's security controls (security requirements laid out by the [National Institute of Standards and Technology (NIST)](https://www.nist.gov/)), documented as part of the authority to operate (ATO) process in coordination with ACF's Office of the Chief Information Officer (OCIO). * **[White House Cybersecurity Executive Order 14208](./WH_CybersecurityEO.md)** - includes information about the status of TANF Data Portal's compliance with White House Cybersecurity Executive Order 14208 issued on May 12, 2021. From 075ff86e5af9b55089291204e92cbdd27d289579 Mon Sep 17 00:00:00 2001 From: Miles Reiter Date: Wed, 21 Jun 2023 02:30:38 -0400 Subject: [PATCH 046/120] Create 2023, Spring - Testing CSV & Excel-based error reports.md --- ...Testing CSV & Excel-based error reports.md | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 docs/User-Experience/Research-Syntheses/2023, Spring - Testing CSV & Excel-based error reports.md diff --git a/docs/User-Experience/Research-Syntheses/2023, Spring - Testing CSV & Excel-based error reports.md b/docs/User-Experience/Research-Syntheses/2023, Spring - Testing CSV & Excel-based error reports.md new file mode 100644 index 000000000..c6ca83066 --- /dev/null +++ b/docs/User-Experience/Research-Syntheses/2023, Spring - Testing CSV & Excel-based error reports.md @@ -0,0 +1,209 @@ +# Spring 2023 - Testing CSV & Excel-based error reports + +## Contents: +* [Key insights](#Key-insights) +* [Who we talked to](#Who-we-talked-to) +* [What we did](#What-we-did) +* [What we tested](#What-we-tested) +* [What we learned](#What-we-learned) +* [What's next](#Whats-next) +___ + +### Key insights +As research conducted in [January 2023 Errors Research](https://hackmd.io/WxtMOPCUQiO-JHjahVMPDQ) focused on error messaging and specific internal workflows occuring after data submission, our research in this cycle explored the end-to-end error report experience and simulated it in the direct context of the file formats that the parsing engine will ultimately produce. + + + +- All participants were able to automatically open all variations of the error report prototypes that we tested in Excel. Participants were also able to expand collapsed columns themselves, but we found that it decreased time and effort on the part of the grantee when we provided versions of the prototype with columns that were pre-expanded to fit their content. + + - **Impact**: As a consequence of this finding and conversations with the development team we shifted from CSV format to XLSX to take advantage of AutoFit column widths to enhance the immediate readability of reports. + + +- We identified which columns should be included—and in what order—to maximize the actionable : extraneous data ratio of reports. + - **Impact**: Case Number, Month, and Error Message columns proved most critical for participants with Item Number and Field Name frequently assessed as a "nice to have". A final prototype reflecting our complete findings in this area will be produced for [#2527](https://github.com/raft-tech/TANF-app/issues/2527) and include the following columns: + + | Case Number | Month | Error Message | Item Number | Field Name | + | ----------- | ----- | ------------- | ----------- | ---------- | + +- Item number and field name represent an area where the language used in TANF documentation has shifted over time. Participants frequently appeared more familiar with the term "Data Element" than "Item" or "Field" but were ultimately all able to make the correct association with the relevant content of the data reporting instructions. + - **Impact** After discussions with ACF regarding upcoming revisions to the instructions "Item number" does reflect the preferred terminology going forward. This will be captured in [#2527](https://github.com/raft-tech/TANF-app/issues/2527) and additionally justify future knowledge center content to address any resulting confusion for TDP users. + +**Overall the feedback on the post data file submission was positive and encouraging to continue developing the email communication notice and Error Report.** + +> “This [Excel file] is helpful too [...] after further review, I’m able to understand the spreadsheet more. I’m excited for this project!" - Tribe + +> "I think this excel sheet is more user friendly than the [legacy system report] table. I think this has more information than compared to what I get now" - State + +> "[...]I think this is perfect. Once I knew what I was looking at it’s very easy to follow." - Tribe + +> “I’m excited about [having this in] an online portal. One of the things that was an issue prior was they would ask for proof that we submitted and we didn’t have it.” - Tribe + +> "I like the words and not just the [item] numbers. This saves me a step." - State +___ + + + +### Who we talked to + +We recruited based primarily on the following criteria: +* Participants from past error message research sessions +* TANF programs which had previously encountered errors on their Section 1 data files +* Programs and past participants who had expressed specific interest in participating in research sessions + +Research sessions were conducted with: +* 5 States +* 4 Tribes +___ + +### What we did + +Our research was conducted in order to: + +* Test the usability of an Excel-based error report located in Submission History to evaluate whether participants would be successful in downloading, opening, and ultimately acting on the reports. +* Explore optimal column structures and data needed to help guide error correction and cut down on the time required to understand the report itself. + +___ + +### What we tested + +During these sessions, we wanted to simulate the future experience of the error reports with our participants. We asked participants to: + +* Speak about their expectations at each step of the process including the error report email, the submission history page (including acceptance status and the link to the error report), and the downloadable error report prototype itself. +* Open the error report prototype once they downloaded it. +* Identify the meaning in their own words of key columns. +* Rank the columns in order of their priority in regard to error correction. +* Use the Error Report as if they were using it to resolve the errors listed and expand on how it might fit into their current workflows. + +**Supporting Documentation:** +- [A variation of our HTML/XLSX prototype](https://reitermb.github.io/HTML-Table-example/xlsx-login) +___ + +### What we learned + +Jump to: +* [Collapsed columns in the Error Report created extra steps for participants when first opening the CSV-based prototype](#Collapsed-columns-in-the-error-report-created-extra-steps-for-participants-when-first-opening-the-csv-based-prototype) +* [We identified which columns should be included—and in what order—to maximize the actionable : extraneous data ratio of reports](#We-identified-which-columns-should-be-included-and-in-what-order-to-maximize-the-actionable-extraneous-data-ratio-of-reports) +* [Participants use the coding instructions in conjunction with the Error Report for error definition, but it can be challenging to know whether all information is available](#Participants-use-the-coding-instructions-in-conjunction-with-the-error-report-for-error-definition-but-it-can-be-challenging-to-know-whether-all-information-is-available) +* [Item number and field name represent an area where the language used in TANF documentation has shifted over time](#Item-number-and-field-name-represent-an-area-where-the-language-used-in-TANF-documentation-has-shifted-over-time) +* [We further validated the utility of submission history and notification emails as ways for TANF programs to show proof of transmission for audit purposes](#We-further-validated-the-utility-of-submission-history-and-notification-emails-as-ways-for-tanf-programs-to-show-proof-of-transmission-for-audit-purposes) + +--- + +#### Collapsed columns in the Error Report created extra steps for participants when first opening the CSV-based prototype + +The following observations were made of the first few participants when opening the initial file format of .csv (comma-separated values) + +- Participant expanded the columns one-by-one manually +- Participant began to expand the first few columns, but then stopped as the screen size hid the final column from view +- One participant did not expand any of the columns and moved from cell to cell to read the content in the formula bar +- The Case Number field was truncated by Excel and could not be easily read (e.g. 1.111+08) + +After this initial round of review we determined (via conversations with the product and development teams) that we could move forward with a .xlsx file format which could take advantage of column AutoFit. All further research sessions were conducted with the .xlsx-based prototypes, which eliminated the need for participants to adjust columns and allowed more time to be spent dedicated to the content of the error report. + +--- + +#### We identified which columns should be included—and in what order—to maximize the actionable : extraneous data ratio of reports. + +We started the first few research sessions with a version of the column headers and iterated as we went on to further sessions based on early participant feedback. + +Our first version used the following columns/orderings: + + +| Case Number | Month | Item Number | Field Name | Error Message | Error Type | +| ----------- | ----- | ----------- | ---------- | ------------- | ---------- | + + + +> “I’m not sure about the item number - what does this represent?” - Tribe + +> “I don’t know what item number and field name are.” Initial reaction was that the Error Report did not provide the information they needed - State + +> “I’d suggest removing item number and field name but leave the rest” - Tribe + +> “I would prefer the Error Message come before the Item Number and Field Name” - Tribe + +After the first three research sessions we reevaluated our hypothesis that that Item Number and Field Name near the beginning was the most digestible way to present an error report. Those columns coming before the Error Message seemed to be at odds with participant's priorities and expectations. + +Our subsequent version—used for the remaining research sessions—had the following columns/orderings: + + +| Case Number | Month | Error Message | Item Number | Field Name | Error Type | +| ----------- | ----- | ------------- | ----------- | ---------- | ---------- | + + +> “I like the flow of the columns - Tribe + +> “I think the sequence is good for me” - State + +> “I am strictly looking at Case Numbers, Month and Error Message here. The rest is noise to me” - State + +> “I think these columns are pretty good at showing me where things are at” - Tribe + +We received a wide range of feedback around the utility of Item Number and Field Name. Participants who had many years of experience with TANF data tended to feel they were redundant to the Error Message—but might be useful for those new to correcting TANF data. Others seemed to appreciate the degree to which those columns reinforced key parts of the Error Message. + +While this research did not directly content test Error Types, some participants were able to correctly interpret their meanings. However, it seemed that the primary way Data Analysts categorize different types of errors among themselves is by classification under their Item/Field names. For example, a Data Analyst receiving the error, "If Item 49 (Work Participation Status) = 99 Or Blank, Item 30 (Family Affiliation - Adult) Must = 2-5" would consider its type to be a "Work Participation Status Error". + + +--- + +#### Participants use the coding instructions in conjunction with the Error Report for error definition, but it can be challenging to know whether all information is available + +When we asked participants to use the error report prototype to resolve the errors they consistently understood how to pair it with the coding instructions. + +> “I want to cheat here” (Participant referring to the Tribal Coding Instructions as a cheat sheet while responding to a particular error) - Tribe + +> “I will look at this line and then go to the guide to find more information for Item 7” - State + +> “The way the error report is… it is kind of like a maze where we have to go here and then back” - Tribe + +While all had access to at least some of the coding instructions, it wasn't always clear whether the participants had the latest or even complete versions of them. We observed one participant who had printed a few pages to keep in their workspace to help handle common errors. + +> “Is there any updated code instructions book so that we have the latest?” - State + +> “Can you get to the Instructions from [the excel error report] or no?” - Tribe + +Linking to the coding instructions; and particularly linking directly to relevant items may help reduce lookup times and make the whole process easier to navigate for those new to TANF data coding. + +> “A link to the coding instructions would not be helpful because it is 97 pages long”. - State + +--- + +#### Item number and field name represent an area where the language used in TANF documentation has shifted over time. + +The term “Item Number” originated from the fTANF system where each field is labeled as “Item [x]”. However, in the current coding instructions, it is referred to as a "Data Element". + +> “Are Item Numbers related to Data Elements?” - State + +> “What is it called… it’s not item number…” (Participant flips through a printed excerpt of the Coding Instructions) - Tribe + +After discussions with OFA regarding upcoming revisions to the coding instructions "Item number" does reflect the preferred terminology going forward. All participants ultimately managed to connect the two terms, but this is still a likely area for future knowledge center content such as an FAQ item to reassure TDP users that they're making the correct connection of terms. + +--- + +#### We further validated the utility of submission history and notification emails as ways for TANF programs to show proof of transmission for audit purposes + +The error report email and submission history page would serve to help answer two common questions that Data Analysts—and/or their auditors—have after data file submission. "When did we transmit the data files and has ACF received them?"" + +> “Referenced the transmission file as what they still gets for confirmation on when files are transmitted to ACF.” - State + +> “Transmission report is what the auditors get to show that I’ve met the timeline” - State + +> “The date and time that I submitted is critical for the auditors” - State + +> “Auditors want proof” - State + +Participants also spoke to the usefulness of TDP's notification emails for fulfilling elements of these audits. + +A common subsequent question is "Are my files without errors or do they need correction?". While evaluating TDP's acceptance statuses were not a primary goal of this research, multiple participants did respond to its presence in our prototype—commentating on acceptance status and its lack of clarity in TDRS. + +> “[After I submit] has it been accepted or is the data okay? Is it acceptable data? Do I need correct anything?” - Tribe + +> “[I] wasn’t too clear whether the current transmission report provided [all] errors or files were transmitted successfully” - State + + + +--- + +### What's next +- We plan to capture final prototype encompassing all changes informed by this research in [#2527](https://github.com/raft-tech/TANF-app/issues/2527). +- Further errors research with participants from this round who submitted files with errors for the May 15th deadline to repeat many elements of this test but directly in the context of their real data (rather than example data). \ No newline at end of file From 2ff7d2ec2f6160634e02063cf9f133483461f4f7 Mon Sep 17 00:00:00 2001 From: Miles Reiter Date: Wed, 21 Jun 2023 12:13:39 -0400 Subject: [PATCH 047/120] Update README.md --- docs/User-Experience/Research-Syntheses/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/User-Experience/Research-Syntheses/README.md b/docs/User-Experience/Research-Syntheses/README.md index 261a30fe2..43496f75a 100644 --- a/docs/User-Experience/Research-Syntheses/README.md +++ b/docs/User-Experience/Research-Syntheses/README.md @@ -5,9 +5,15 @@ With a few exceptions, we've tended to publish markdown research syntheses to su The syntheses included herein are organized reverse-chronologically from newest to oldest: +### [2023, Sprint - TDP 3.0 Pilot Program](https://github.com/raft-tech/TANF-app/blob/develop/docs/User-Experience/Research-Syntheses/2023%2C%20Spring%20-%20Testing%20CSV%20%26%20Excel-based%20error%20reports.md#spring-2023---testing-csv--excel-based-error-reports) + +- Research sessions conducted with 5 states and 4 Tribes with a focus on programs that had errors on their Section 1 Data Files. +- Tested the usability of an Excel-based error report located in Submission History to evaluate whether participants would be successful in downloading, opening, and ultimately acting on the reports. +* Explored optimal column structures and data needed to help guide error correction and cut down on the time required to understand the report itself. + ### [2023, Winter - TDP 3.0 Pilot Program](https://github.com/raft-tech/TANF-app/blob/develop/docs/User-Experience/Research-Syntheses/2023%2C%20Winter%20-%20TDP%203.0%20Pilot%20Program.md) -- A total of 17 states, 17 tribes and two territories participated in the pilot expansion using the production version of TDP to submit their FY2023 Q1 data files. +- A total of 17 states, 17 Tribes and two territories participated in the pilot expansion using the production version of TDP to submit their FY2023 Q1 data files. - Conducted a retrospective on the usability of Django (system admin) as the TDP user base continues to expand and what might the System Admins need to keep up with onboarding, and managing users and logs at scale. ### [2022, Winter - Understanding How STTs Would Use the Transmission Report](https://github.com/raft-tech/TANF-app/blob/develop/docs/User-Experience/Research-Syntheses/2022%2C%20Winter%20-%20Understanding%20How%20STTs%20Would%20Use%20the%20Transmission%20Report.md) From 093f67f220204020630b5c822bc4b8410af495bb Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Wed, 21 Jun 2023 13:54:57 -0400 Subject: [PATCH 048/120] Updating deliverable links (#2584) --- .github/pull_request_template.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 895d9cd92..625fe7f9c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,15 +16,15 @@ cd tdrs-backend && docker-compose -f docker-compose.yml -f docker-compose.local. > *Demo GIF(s) and screenshots for testing procedure* ## Deliverables -_More details on how deliverables herein are assessed included [here](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverables)._ +_More details on how deliverables herein are assessed included [here](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverables)._ -### [Deliverable 1: Accepted Features](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-1-Accepted-Features) +### [Deliverable 1: Accepted Features](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-1-Accepted-Features) Checklist of ACs: + [ ] [**_insert ACs here_**] + [ ] **`lfrohlich`** and/or **`adpennington`** confirmed that ACs are met. -### [Deliverable 2: Tested Code](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-2-Tested-Code) +### [Deliverable 2: Tested Code](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-2-Tested-Code) + Are all areas of code introduced in this PR meaningfully tested? + [ ] If this PR introduces backend code changes, are they meaningfully tested? @@ -33,13 +33,13 @@ Checklist of ACs: + [ ] Frontend coverage: [_insert coverage %_] (see `CodeCov Report` comment in PR) + [ ] Backend coverage: [_insert coverage %_] (see `CodeCov Report` comment in PR) -### [Deliverable 3: Properly Styled Code](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-3-Properly-Styled-Code) +### [Deliverable 3: Properly Styled Code](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-3-Properly-Styled-Code) + [ ] Are backend code style checks passing on CircleCI? + [ ] Are frontend code style checks passing on CircleCI? + [ ] Are code maintainability principles being followed? -### [Deliverable 4: Accessible](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-4-Accessibility) +### [Deliverable 4: Accessible](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-4-Accessibility) + [ ] Does this PR complete the epic? + [ ] Are links included to any other gov-approved PRs associated with epic? @@ -47,11 +47,11 @@ Checklist of ACs: + [ ] Did automated and manual testing with `iamjolly` and `ttran-hub` using Accessibility Insights reveal any errors introduced in this PR? -### [Deliverable 5: Deployed](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-5-Deployed) +### [Deliverable 5: Deployed](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-5-Deployed) + [ ] Was the code successfully deployed via automated CircleCI process to development on Cloud.gov? -### [Deliverable 6: Documented](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-6-Code-documentation) +### [Deliverable 6: Documented](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-6-Code-documentation) + [ ] Does this PR provide background for why coding decisions were made? + [ ] If this PR introduces backend code, is that code easy to understand and sufficiently documented, both inline and overall? @@ -59,13 +59,13 @@ Checklist of ACs: + [ ] If this PR introduces dependencies, are their licenses documented? + [ ] Can reviewer explain and take ownership of these elements presented in this code review? -### [Deliverable 7: Secure](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-7-Secure) +### [Deliverable 7: Secure](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-7-Secure) + [ ] Does the OWASP Scan pass on CircleCI? + [ ] Do manual code review and manual testing detect any new security issues? + [ ] If new issues detected, is investigation and/or remediation plan documented? -### [Deliverable 8: User Research](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-8-User-Research) +### [Deliverable 8: User Research](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-8-User-Research) Research product(s) clearly articulate(s): + [ ] the purpose of the research From 9534d255d071719c5a42ba13aeb99a6b8133b2d7 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Wed, 21 Jun 2023 12:16:21 -0600 Subject: [PATCH 049/120] User viewset not returning/duplicating users (#2573) * - Fixed issue not allowing pagination to work locally with nginx - Added ordering to user field to fix duplicates issue * - fix lint error * - Removing ID check since we cannot guarantee that the uuid that is generated per test run will be lexigraphically consistent --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- tdrs-backend/tdpservice/users/models.py | 5 +++++ .../tdpservice/users/test/test_api/test_endpoints.py | 1 - tdrs-frontend/nginx/local/locations.conf | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/users/models.py b/tdrs-backend/tdpservice/users/models.py index 7d9a9507c..d0a9c924d 100644 --- a/tdrs-backend/tdpservice/users/models.py +++ b/tdrs-backend/tdpservice/users/models.py @@ -33,6 +33,11 @@ class AccountApprovalStatusChoices(models.TextChoices): class User(AbstractUser): """Define user fields and methods.""" + class Meta: + """Define meta user model attributes.""" + + ordering = ['pk'] + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) stt = models.ForeignKey( diff --git a/tdrs-backend/tdpservice/users/test/test_api/test_endpoints.py b/tdrs-backend/tdpservice/users/test/test_api/test_endpoints.py index a9265b74f..a20244538 100644 --- a/tdrs-backend/tdpservice/users/test/test_api/test_endpoints.py +++ b/tdrs-backend/tdpservice/users/test/test_api/test_endpoints.py @@ -231,7 +231,6 @@ def test_list_users(self, api_client, user): results = response.data['results'] assert response.status_code == status.HTTP_200_OK assert len(results) == 2 - assert results[1]['id'] == user.id def test_get_user(self, api_client, user): """Get a specific user, expect 200 and ability to get any user.""" diff --git a/tdrs-frontend/nginx/local/locations.conf b/tdrs-frontend/nginx/local/locations.conf index 4f53dd27b..9859f5fd1 100644 --- a/tdrs-frontend/nginx/local/locations.conf +++ b/tdrs-frontend/nginx/local/locations.conf @@ -7,7 +7,7 @@ location = /nginx_status { location ~ ^/(v1|admin|static/admin|swagger|redocs) { limit_req zone=limitreqsbyaddr delay=5; proxy_pass http://${BACK_END}:8080$request_uri; - proxy_set_header Host $host; + proxy_set_header Host $host:3000; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; From 9a392ba49b8033942273bac39628b7441dd4b363 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Thu, 22 Jun 2023 09:17:36 -0600 Subject: [PATCH 050/120] Update cf os (#2523) * - updated nginx buildpack * - specifying different nginx version * - Updating changelog * - added script to update certain apps in cf - added workflow for each environment in circi * - fixed base config * - fixing jobs * - Updated based on feedback in OH * - Updating defaults * - Removing defaults * - Fixing indent * - Adding params to config * test * test * - updating work dir * - Adding checkout * - adding cf check * - logging into cf * - update cf check to install required binary * - removing unnecessary switch * - Forcing plugin installation * - test installing plugin from script also * - Adding new dependencies * - adding package * - fixing broken install * - fixing libs * - using correct command * - gettign correct version of libc * - trying to upgrade libs * - testing * - Updated README and script * Revert "- Updated README and script" This reverts commit 92697b3e53d1fd87b8d3e7995abb9093aa26e307. * - Removed unnecessary circi stuff - Removed script - Updated docs to callout updating secondary apps * - Correct spelling error --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .circleci/README.md | 14 ++++++++++++++ .circleci/deployment/commands.yml | 4 ++-- .circleci/deployment/jobs.yml | 2 +- .circleci/deployment/workflows.yml | 3 ++- .../Technical-Documentation/buildpack-changelog.md | 4 +++- scripts/cf-check.sh | 1 + tdrs-frontend/manifest.buildpack.yml | 2 +- 7 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.circleci/README.md b/.circleci/README.md index f08a5f0e0..39e484871 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -71,3 +71,17 @@ You want to set the branch to be the branch you want this scan to be run on. You want to add a Pipeline Parameter with `run_nightly_owasp_scan`to be a boolean and set to `true`. You want Attribution set to Scheduled Actor (Scheduling System) + +## Updating Cloud Foundry App OS +Cloud Foundry (CF) occasionally releases OS updates. In doing so they deprecate the previous OS and after a short time +do not allow any apps to run/deploy on anything but the latest OS. The steps below describe how the main TDP apps are +updated along with the secondary apps running in CF. + +### Frontend/Backend + - Before updating, make sure the current buildpacks that these apps use are supported by the latest OS. If they aren't you can update the manifest to point them to the correct buildpacks. + - To update the apps you can either deploy each of the environments (sandbox, raft, qasp, etc) from CircleCi or you can use the `tdrs-deploy ` command from `commands.sh`. Assuming the buildpacks are up to date, that is all you need to do. + +### Secondary apps + - Before you can make the update, you need to ensure you have the CF plugin that allows you to do so. Download the binary for your respective OS [HERE](https://github.com/cloudfoundry/stack-auditor/releases) and follow the installation instructions [HERE](https://docs.cloudfoundry.org/adminguide/stack-auditor.html#install). + - Verify the installation succeeded by running `cf audit-stack`. Note you need to be logged in and have targeted a space via `cf target -o hhs-acf-ofa -s ` + - To update the remaining apps you need to run `cf change-stack ` against every app that is not a frontend/backend app. diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index 65cab6a93..b259fb57f 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -110,10 +110,10 @@ default: tdp-frontend type: string # So the frontend knows what space its in for the banner. -# I am unclear if the domain is a reliable metric to make this function +# I am unclear if the domain is a reliable metric to make this function # It seems like it might not be working cf-space: - default: dev + default: dev type: string steps: - install-nodejs: diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index dc1ee1d5b..a24baef46 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -20,7 +20,7 @@ cf-password: CF_PASSWORD_STAGING cf-space: tanf-staging cf-username: CF_USERNAME_STAGING - + deploy-develop: executor: docker-executor working_directory: ~/tdp-deploy diff --git a/.circleci/deployment/workflows.yml b/.circleci/deployment/workflows.yml index 730a87373..91e5ea56a 100644 --- a/.circleci/deployment/workflows.yml +++ b/.circleci/deployment/workflows.yml @@ -11,7 +11,7 @@ - deploy-infrastructure-dev staging-deployment: - unless: + unless: or: - << pipeline.parameters.run_dev_deployment >> - << pipeline.parameters.run_nightly_owasp_scan >> @@ -71,3 +71,4 @@ branches: only: - master + diff --git a/docs/Technical-Documentation/buildpack-changelog.md b/docs/Technical-Documentation/buildpack-changelog.md index 6208437fa..a53fdcf6a 100644 --- a/docs/Technical-Documentation/buildpack-changelog.md +++ b/docs/Technical-Documentation/buildpack-changelog.md @@ -1,4 +1,6 @@ ## Buildpacks Changelog +- 05/15/2023 [nginx-buildpack v1.2.2](https://github.com/cloudfoundry/nginx-buildpack.git#v1.2.2) + - 11/09/2022 [python-buildpack v1.8.3](https://github.com/cloudfoundry/python-buildpack/releases/tag/v1.8.3) - 11/09/2022 [nginx-buildpack v1.1.45](https://github.com/cloudfoundry/nginx-buildpack/releases/tag/v1.1.45) @@ -10,7 +12,7 @@ - 05/11/2022 [python-buildpack v1.7.53:](https://github.com/cloudfoundry/python-buildpack/releases/tag/v1.7.53) Forced python version to 3.10.4 - 03/10/2022 [python-buildpack v1.7.51:](https://github.com/cloudfoundry/python-buildpack/releases/tag/v1.7.51) - + - 03/10/2022 [nginx-buildpack v1.1.36:](https://github.com/cloudfoundry/nginx-buildpack/releases/tag/v1.1.36) - 01/03/2022 [python-buildpack v1.7.48:](https://github.com/cloudfoundry/python-buildpack/releases/tag/v1.7.48) diff --git a/scripts/cf-check.sh b/scripts/cf-check.sh index c5f513ed7..5eacbd952 100755 --- a/scripts/cf-check.sh +++ b/scripts/cf-check.sh @@ -13,4 +13,5 @@ else apt-get update apt-get install cf7-cli + fi diff --git a/tdrs-frontend/manifest.buildpack.yml b/tdrs-frontend/manifest.buildpack.yml index 097d89323..870321b77 100755 --- a/tdrs-frontend/manifest.buildpack.yml +++ b/tdrs-frontend/manifest.buildpack.yml @@ -3,7 +3,7 @@ version: 1 applications: - name: tdp-frontend buildpacks: - - https://github.com/cloudfoundry/nginx-buildpack.git#v1.1.45 + - https://github.com/cloudfoundry/nginx-buildpack.git#v1.2.2 memory: 128M instances: 1 disk_quota: 256M From f8d75cc32e6459b594098e0083328cef49bd5692 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Fri, 23 Jun 2023 14:24:42 -0600 Subject: [PATCH 051/120] Item Number Mismatch (#2578) * - Updated schemas and models to reflect correct item numbers of fields * - Revert migration * - Updated header/trailer item numbers * - Fixed item numbers off by one errors --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../0006_alter_parsererror_item_number.py | 18 +++ tdrs-backend/tdpservice/parsers/models.py | 2 +- .../tdpservice/parsers/schema_defs/header.py | 20 +-- .../tdpservice/parsers/schema_defs/ssp/m1.py | 86 +++++------ .../tdpservice/parsers/schema_defs/ssp/m2.py | 132 ++++++++--------- .../tdpservice/parsers/schema_defs/ssp/m3.py | 84 +++++------ .../tdpservice/parsers/schema_defs/tanf/t1.py | 92 ++++++------ .../tdpservice/parsers/schema_defs/tanf/t2.py | 138 +++++++++--------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 84 +++++------ .../tdpservice/parsers/schema_defs/trailer.py | 6 +- .../tdpservice/parsers/test/factories.py | 2 +- 11 files changed, 342 insertions(+), 322 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py diff --git a/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py b/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py new file mode 100644 index 000000000..79607377e --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-06-15 18:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0005_auto_20230601_1510'), + ] + + operations = [ + migrations.AlterField( + model_name='parsererror', + name='item_number', + field=models.CharField(max_length=8, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 78b12a163..9ffa8306b 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -35,7 +35,7 @@ class Meta: ) row_number = models.IntegerField(null=False) column_number = models.IntegerField(null=True) - item_number = models.IntegerField(null=True) + item_number = models.CharField(null=True, max_length=8) field_name = models.TextField(null=True, max_length=128) rpt_month_year = models.IntegerField(null=True, blank=False) case_number = models.TextField(null=True, max_length=128) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index b51c433ed..318314252 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -16,34 +16,34 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ + Field(item="2", name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ validators.matches('HEADER'), ]), - Field(item=2, name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ + Field(item="4", name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ validators.between(2000, 2099) ]), - Field(item=3, name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ + Field(item="5", name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ validators.oneOf(['1', '2', '3', '4']) ]), - Field(item=4, name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ + Field(item="6", name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ validators.oneOf(['A', 'C', 'G', 'S']) ]), - Field(item=5, name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ + Field(item="1", name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ validators.between(0, 99) ]), - Field(item=6, name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ + Field(item="3", name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ validators.between(0, 999) ]), - Field(item=7, name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ + Field(item="7", name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ validators.oneOf(['TAN', 'SSP']) ]), - Field(item=8, name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ + Field(item="8", name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ validators.oneOf(['1', '2']) ]), - Field(item=9, name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ + Field(item="9", name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ validators.matches('E') ]), - Field(item=10, name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ + Field(item="10", name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ validators.oneOf(['N', 'D', 'U']) ]), ], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 79b9c8bc8..590ca4ea1 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -13,89 +13,91 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, + Field(item="4", name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[]), - Field(item=7, name='DISPOSITION', type='number', startIndex=29, endIndex=30, + Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30, required=True, validators=[]), - Field(item=8, name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, + Field(item="8", name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, required=True, validators=[]), - Field(item=10, name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, + Field(item="9", name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33, required=True, validators=[]), - Field(item=11, name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, + Field(item="10", name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, required=True, validators=[]), - Field(item=12, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, + Field(item="11", name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, required=True, validators=[]), - Field(item=13, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, + Field(item="12", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, required=True, validators=[]), - Field(item=14, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, + Field(item="13", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, required=True, validators=[]), - Field(item=15, name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, + Field(item="14", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, required=True, validators=[]), - Field(item=16, name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, + Field(item="15", name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=17, name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, + Field(item="16", name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, required=True, validators=[]), - Field(item=18, name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, + Field(item="17", name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, required=True, validators=[]), - Field(item=19, name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, + Field(item="18", name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, required=True, validators=[]), - Field(item=20, name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, + Field(item="19A", name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, required=True, validators=[]), - Field(item=21, name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, + Field(item="19B", name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, required=True, validators=[]), - Field(item=22, name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, + Field(item="20A", name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, required=True, validators=[]), - Field(item=23, name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, + Field(item="20B", name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, required=True, validators=[]), - Field(item=24, name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, + Field(item="20C", name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, required=True, validators=[]), - Field(item=25, name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, + Field(item="21A", name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, required=True, validators=[]), - Field(item=26, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, + Field(item="21B", name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, required=True, validators=[]), - Field(item=27, name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, + Field(item="22A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, required=True, validators=[]), - Field(item=28, name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, + Field(item="22B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, required=True, validators=[]), - Field(item=29, name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, + Field(item="23A", name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, required=True, validators=[]), - Field(item=30, name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, + Field(item="23B", name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, required=True, validators=[]), - Field(item=31, name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, + Field(item="24AI", name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, required=True, validators=[]), - Field(item=32, name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, + Field(item="24AII", name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, required=True, validators=[]), - Field(item=33, name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, + Field(item="24AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, required=True, validators=[]), - Field(item=34, name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, + Field(item="24AIV", name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, required=True, validators=[]), - Field(item=35, name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, + Field(item="24AV", name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, required=True, validators=[]), - Field(item=36, name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, + Field(item="24AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, required=True, validators=[]), - Field(item=37, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, + Field(item="24AVII", name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, required=True, validators=[]), - Field(item=38, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, + Field(item="24B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, required=True, validators=[]), - Field(item=39, name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, + Field(item="24CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, required=True, validators=[]), - Field(item=40, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, + Field(item="24CII", name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, required=True, validators=[]), - Field(item=41, name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, + Field(item="24CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, required=True, validators=[]), - Field(item=42, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, + Field(item="24CIV", name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, required=True, validators=[]), - Field(item=43, name='BLANK', type='string', startIndex=113, endIndex=150, + Field(item="25", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, + required=True, validators=[]), + Field(item="-1", name='BLANK', type='string', startIndex=113, endIndex=150, required=False, validators=[]), ] ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index fd855181b..cc80e3431 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -13,137 +13,137 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="26", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, + Field(item="27", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[]), - Field(item=6, name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, + Field(item="28", name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, required=True, validators=[]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, + Field(item="29", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, + Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, + Field(item="30B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, + Field(item="30C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_BLACK', type='number', startIndex=41, endIndex=42, + Field(item="30D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, + Field(item="30E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='RACE_WHITE', type='number', startIndex=43, endIndex=44, + Field(item="30F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, + Field(item="31", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, + Field(item="32A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, + Field(item="32B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, + Field(item="32C", name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, required=True, validators=[]), - Field(item=18, name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, + Field(item="32D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=19, name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, + Field(item="32E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[]), - Field(item=20, name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, + Field(item="33", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, validators=[]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, + Field(item="34", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[]), - Field(item=22, name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, + Field(item="35", name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, validators=[]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, + Field(item="36", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, validators=[]), - Field(item=24, name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, + Field(item="37", name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, required=True, validators=[]), - Field(item=25, name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, + Field(item="38", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, validators=[]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, + Field(item="39", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, validators=[]), - Field(item=27, name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, + Field(item="40", name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, required=True, validators=[]), - Field(item=28, name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, + Field(item="41", name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, required=True, validators=[]), - Field(item=29, name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, + Field(item="42", name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, required=True, validators=[]), - Field(item=30, name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, + Field(item="43", name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, required=True, validators=[]), - Field(item=31, name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, + Field(item="44", name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=32, name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, + Field(item="45", name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, required=True, validators=[]), - Field(item=33, name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, + Field(item="46A", name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, required=True, validators=[]), - Field(item=34, name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, + Field(item="46B", name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, required=True, validators=[]), - Field(item=35, name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, + Field(item="46C", name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, required=True, validators=[]), - Field(item=36, name='OJT', type='number', startIndex=76, endIndex=78, + Field(item="47", name='OJT', type='number', startIndex=76, endIndex=78, required=True, validators=[]), - Field(item=37, name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, + Field(item="48A", name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, required=True, validators=[]), - Field(item=38, name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, + Field(item="48B", name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, required=True, validators=[]), - Field(item=39, name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, + Field(item="48C", name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, required=True, validators=[]), - Field(item=40, name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, + Field(item="49A", name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, required=True, validators=[]), - Field(item=41, name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, + Field(item="49B", name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, required=True, validators=[]), - Field(item=42, name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, + Field(item="49C", name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, required=True, validators=[]), - Field(item=43, name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, + Field(item="50A", name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=44, name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, + Field(item="50B", name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, required=True, validators=[]), - Field(item=45, name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, + Field(item="50C", name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, required=True, validators=[]), - Field(item=46, name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, + Field(item="51A", name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, required=True, validators=[]), - Field(item=47, name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, + Field(item="51B", name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, required=True, validators=[]), - Field(item=48, name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, + Field(item="51C", name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, required=True, validators=[]), - Field(item=49, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, + Field(item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, required=True, validators=[]), - Field(item=50, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, + Field(item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, required=True, validators=[]), - Field(item=51, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, + Field(item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, required=True, validators=[]), - Field(item=52, name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, + Field(item="53A", name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, required=True, validators=[]), - Field(item=53, name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, + Field(item="53B", name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, required=True, validators=[]), - Field(item=54, name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, + Field(item="53C", name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, required=True, validators=[]), - Field(item=55, name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, + Field(item="54A", name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, required=True, validators=[]), - Field(item=56, name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, + Field(item="54B", name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, required=True, validators=[]), - Field(item=57, name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, + Field(item="54C", name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, required=True, validators=[]), - Field(item=58, name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, + Field(item="55", name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, required=True, validators=[]), - Field(item=59, name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, + Field(item="56", name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, required=True, validators=[]), - Field(item=60, name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, + Field(item="57", name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, required=True, validators=[]), - Field(item=61, name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, + Field(item="58", name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, required=True, validators=[]), - Field(item=62, name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, + Field(item="59A", name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, required=True, validators=[]), - Field(item=63, name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, + Field(item="59B", name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, required=True, validators=[]), - Field(item=64, name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, + Field(item="59C", name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, required=True, validators=[]), - Field(item=65, name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, + Field(item="59D", name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, required=True, validators=[]), - Field(item=66, name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, + Field(item="59E", name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, required=True, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index c269861ff..c2f1551cc 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -13,47 +13,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, + Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, required=True, validators=[]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, + Field(item="62", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[]), - Field(item=7, name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, + Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, + Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, + Field(item="63C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_BLACK', type='number', startIndex=40, endIndex=41, + Field(item="63D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, + Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_WHITE', type='number', startIndex=42, endIndex=43, + Field(item="63F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, + Field(item="64", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, + Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, + Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, + Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[]), - Field(item=17, name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, + Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=18, name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, + Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, required=True, validators=[]), - Field(item=19, name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, + Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[]), - Field(item=20, name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, + Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, required=True, validators=[]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, + Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, required=True, validators=[]) ] ) @@ -67,47 +67,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, + Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[]), - Field(item=23, name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, + Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, required=True, validators=[]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, + Field(item="62", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[]), - Field(item=25, name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, + Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[]), - Field(item=26, name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, + Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[]), - Field(item=27, name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, + Field(item="63C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[]), - Field(item=28, name='RACE_BLACK', type='number', startIndex=81, endIndex=82, + Field(item="63D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[]), - Field(item=29, name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, + Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[]), - Field(item=30, name='RACE_WHITE', type='number', startIndex=83, endIndex=84, + Field(item="63F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, + Field(item="64", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[]), - Field(item=32, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, + Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, required=True, validators=[]), - Field(item=33, name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, + Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, required=True, validators=[]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, + Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[]), - Field(item=35, name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, + Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[]), - Field(item=36, name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, + Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=37, name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, + Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[]), - Field(item=38, name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, + Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, required=True, validators=[]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, + Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, required=True, validators=[]) ] ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 483950ce8..25f0b9181 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -13,96 +13,96 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, + Field(item="5", name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[]), - Field(item=7, name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, + Field(item="8", name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[]), - Field(item=8, name='DISPOSITION', type='number', startIndex=30, endIndex=31, + Field(item="9", name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[]), - Field(item=9, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, + Field(item="10", name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[]), - Field(item=10, name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, + Field(item="11", name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[]), - Field(item=11, name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, + Field(item="12", name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[]), - Field(item=12, name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, + Field(item="13", name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[]), - Field(item=13, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, + Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[]), - Field(item=14, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, + Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=15, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, + Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[]), - Field(item=16, name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, + Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=17, name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, + Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[]), - Field(item=18, name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, + Field(item="19", name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[]), - Field(item=19, name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, + Field(item="20", name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[]), - Field(item=20, name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, + Field(item="21A", name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[]), - Field(item=21, name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, + Field(item="21B", name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[]), - Field(item=22, name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, + Field(item="22A", name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[]), - Field(item=23, name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, + Field(item="22B", name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=24, name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, + Field(item="22C", name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[]), - Field(item=25, name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, + Field(item="23A", name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[]), - Field(item=26, name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, + Field(item="23B", name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[]), - Field(item=27, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, + Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[]), - Field(item=28, name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, + Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[]), - Field(item=29, name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, + Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[]), - Field(item=30, name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, + Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[]), - Field(item=31, name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, + Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[]), - Field(item=32, name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, + Field(item="26AII", name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[]), - Field(item=33, name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, + Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[]), - Field(item=34, name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, + Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[]), - Field(item=35, name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, + Field(item="26AV", name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[]), - Field(item=36, name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, + Field(item="26AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[]), - Field(item=37, name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, + Field(item="26AVII", name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[]), - Field(item=38, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, + Field(item="26B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[]), - Field(item=39, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, + Field(item="26CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[]), - Field(item=40, name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, + Field(item="26CII", name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[]), - Field(item=41, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, + Field(item="26CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[]), - Field(item=42, name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, + Field(item="26CIV", name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[]), - Field(item=43, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, + Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[]), - Field(item=44, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, + Field(item="28", name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[]), - Field(item=45, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, + Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[]), - Field(item=46, name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), + Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index dc94263d4..49ec6eff1 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -13,143 +13,143 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="30", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, + Field(item="31", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[]), - Field(item=6, name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, + Field(item="32", name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, required=True, validators=[]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, + Field(item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_HISPANIC', type='string', startIndex=38, endIndex=39, + Field(item="34A", name='RACE_HISPANIC', type='string', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_AMER_INDIAN', type='string', startIndex=39, endIndex=40, + Field(item="34B", name='RACE_AMER_INDIAN', type='string', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_ASIAN', type='string', startIndex=40, endIndex=41, + Field(item="34C", name='RACE_ASIAN', type='string', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_BLACK', type='string', startIndex=41, endIndex=42, + Field(item="34D", name='RACE_BLACK', type='string', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_HAWAIIAN', type='string', startIndex=42, endIndex=43, + Field(item="34E", name='RACE_HAWAIIAN', type='string', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='RACE_WHITE', type='string', startIndex=43, endIndex=44, + Field(item="34F", name='RACE_WHITE', type='string', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, + Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='FED_OASDI_PROGRAM', type='string', startIndex=45, endIndex=46, + Field(item="36A", name='FED_OASDI_PROGRAM', type='string', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='FED_DISABILITY_STATUS', type='string', startIndex=46, endIndex=47, + Field(item="36B", name='FED_DISABILITY_STATUS', type='string', startIndex=46, endIndex=47, required=True, validators=[]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, validators=[]), - Field(item=18, name='AID_AGED_BLIND', type='string', startIndex=48, endIndex=49, + Field(item="36D", name='AID_AGED_BLIND', type='string', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=19, name='RECEIVE_SSI', type='string', startIndex=49, endIndex=50, + Field(item="36E", name='RECEIVE_SSI', type='string', startIndex=49, endIndex=50, required=True, validators=[]), - Field(item=20, name='MARITAL_STATUS', type='string', startIndex=50, endIndex=51, + Field(item="37", name='MARITAL_STATUS', type='string', startIndex=50, endIndex=51, required=True, validators=[]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, + Field(item="38", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[]), - Field(item=22, name='PARENT_WITH_MINOR_CHILD', type='string', startIndex=53, endIndex=54, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='string', startIndex=53, endIndex=54, required=True, validators=[]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='string', startIndex=54, endIndex=55, + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='string', startIndex=54, endIndex=55, required=True, validators=[]), - Field(item=24, name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, validators=[]), - Field(item=25, name='CITIZENSHIP_STATUS', type='string', startIndex=57, endIndex=58, + Field(item="42", name='CITIZENSHIP_STATUS', type='string', startIndex=57, endIndex=58, required=True, validators=[]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='string', startIndex=58, endIndex=59, + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='string', startIndex=58, endIndex=59, required=True, validators=[]), - Field(item=27, name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, validators=[]), - Field(item=28, name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, validators=[]), - Field(item=29, name='CURRENT_MONTH_STATE_EXEMPT', type='string', startIndex=64, endIndex=65, + Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='string', startIndex=64, endIndex=65, required=True, validators=[]), - Field(item=30, name='EMPLOYMENT_STATUS', type='string', startIndex=65, endIndex=66, + Field(item="47", name='EMPLOYMENT_STATUS', type='string', startIndex=65, endIndex=66, required=True, validators=[]), - Field(item=31, name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, + Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=32, name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, validators=[]), - Field(item=33, name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, validators=[]), - Field(item=34, name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, validators=[]), - Field(item=35, name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, validators=[]), - Field(item=36, name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, validators=[]), - Field(item=37, name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, validators=[]), - Field(item=38, name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, validators=[]), - Field(item=39, name='OJT', type='string', startIndex=82, endIndex=84, + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[]), - Field(item=40, name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, validators=[]), - Field(item=41, name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, validators=[]), - Field(item=42, name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, validators=[]), - Field(item=43, name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=44, name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, validators=[]), - Field(item=45, name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, validators=[]), - Field(item=46, name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, + Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, required=False, validators=[]), - Field(item=47, name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, + Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, required=False, validators=[]), - Field(item=48, name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, + Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, required=False, validators=[]), - Field(item=49, name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, + Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, required=False, validators=[]), - Field(item=50, name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, + Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, required=False, validators=[]), - Field(item=51, name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, + Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, required=False, validators=[]), - Field(item=52, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, + Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, required=False, validators=[]), - Field(item=53, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, + Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, required=False, validators=[]), - Field(item=54, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, + Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, required=False, validators=[]), - Field(item=55, name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, + Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, required=False, validators=[]), - Field(item=56, name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, + Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, required=False, validators=[]), - Field(item=57, name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, + Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, required=False, validators=[]), - Field(item=58, name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, + Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, required=False, validators=[]), - Field(item=59, name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, + Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, required=False, validators=[]), - Field(item=60, name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, + Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, required=False, validators=[]), - Field(item=61, name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, + Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, required=False, validators=[]), - Field(item=62, name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, + Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, required=False, validators=[]), - Field(item=63, name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, + Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, required=False, validators=[]), - Field(item=64, name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, + Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=False, validators=[]), - Field(item=65, name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, + Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, required=False, validators=[]), - Field(item=67, name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, + Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=False, validators=[]), - Field(item=68, name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, + Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=False, validators=[]), - Field(item=69, name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, + Field(item="66D", name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, required=False, validators=[]), - Field(item=70, name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, + Field(item="66E", name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index c22dddc7a..4da311c98 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -13,47 +13,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="67", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, + Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, required=True, validators=[]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, + Field(item="69", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[]), - Field(item=7, name='RACE_HISPANIC', type='string', startIndex=37, endIndex=38, + Field(item="70A", name='RACE_HISPANIC', type='string', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_AMER_INDIAN', type='string', startIndex=38, endIndex=39, + Field(item="70B", name='RACE_AMER_INDIAN', type='string', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_ASIAN', type='string', startIndex=39, endIndex=40, + Field(item="70C", name='RACE_ASIAN', type='string', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_BLACK', type='string', startIndex=40, endIndex=41, + Field(item="70D", name='RACE_BLACK', type='string', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_HAWAIIAN', type='string', startIndex=41, endIndex=42, + Field(item="70E", name='RACE_HAWAIIAN', type='string', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_WHITE', type='string', startIndex=42, endIndex=43, + Field(item="70F", name='RACE_WHITE', type='string', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, + Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=44, endIndex=45, + Field(item="72A", name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='RECEIVE_SSI', type='string', startIndex=45, endIndex=46, + Field(item="72B", name='RECEIVE_SSI', type='string', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, + Field(item="73", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[]), - Field(item=17, name='PARENT_MINOR_CHILD', type='string', startIndex=48, endIndex=49, + Field(item="74", name='PARENT_MINOR_CHILD', type='string', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=18, name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, + Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[]), - Field(item=19, name='CITIZENSHIP_STATUS', type='string', startIndex=51, endIndex=52, + Field(item="76", name='CITIZENSHIP_STATUS', type='string', startIndex=51, endIndex=52, required=True, validators=[]), - Field(item=20, name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, + Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, required=False, validators=[]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, + Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, required=False, validators=[]), ], ) @@ -66,47 +66,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, + Field(item="67", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[]), - Field(item=23, name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, + Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, required=True, validators=[]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, + Field(item="69", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[]), - Field(item=25, name='RACE_HISPANIC', type='string', startIndex=78, endIndex=79, + Field(item="70A", name='RACE_HISPANIC', type='string', startIndex=78, endIndex=79, required=True, validators=[]), - Field(item=26, name='RACE_AMER_INDIAN', type='string', startIndex=79, endIndex=80, + Field(item="70B", name='RACE_AMER_INDIAN', type='string', startIndex=79, endIndex=80, required=True, validators=[]), - Field(item=27, name='RACE_ASIAN', type='string', startIndex=80, endIndex=81, + Field(item="70C", name='RACE_ASIAN', type='string', startIndex=80, endIndex=81, required=True, validators=[]), - Field(item=28, name='RACE_BLACK', type='string', startIndex=81, endIndex=82, + Field(item="70D", name='RACE_BLACK', type='string', startIndex=81, endIndex=82, required=True, validators=[]), - Field(item=29, name='RACE_HAWAIIAN', type='string', startIndex=82, endIndex=83, + Field(item="70E", name='RACE_HAWAIIAN', type='string', startIndex=82, endIndex=83, required=True, validators=[]), - Field(item=30, name='RACE_WHITE', type='string', startIndex=83, endIndex=84, + Field(item="70F", name='RACE_WHITE', type='string', startIndex=83, endIndex=84, required=True, validators=[]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, + Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[]), - Field(item=32, name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=85, endIndex=86, + Field(item="72A", name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=85, endIndex=86, required=True, validators=[]), - Field(item=33, name='RECEIVE_SSI', type='string', startIndex=86, endIndex=87, + Field(item="72B", name='RECEIVE_SSI', type='string', startIndex=86, endIndex=87, required=True, validators=[]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, + Field(item="73", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[]), - Field(item=35, name='PARENT_MINOR_CHILD', type='string', startIndex=89, endIndex=90, + Field(item="74", name='PARENT_MINOR_CHILD', type='string', startIndex=89, endIndex=90, required=True, validators=[]), - Field(item=36, name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, + Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=37, name='CITIZENSHIP_STATUS', type='string', startIndex=92, endIndex=93, + Field(item="76", name='CITIZENSHIP_STATUS', type='string', startIndex=92, endIndex=93, required=True, validators=[]), - Field(item=38, name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, + Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, required=False, validators=[]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, + Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 3657cfc51..086c83e85 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -16,13 +16,13 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ + Field(item="1", name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ validators.matches('TRAILER') ]), - Field(item=2, name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ + Field(item="2", name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ validators.between(0, 9999999) ]), - Field(item=3, name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ + Field(item="-1", name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ validators.matches(' ') ]), ], diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index e45e78be2..4af7a6707 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -68,7 +68,7 @@ class Meta: file = factory.SubFactory(DataFileFactory) row_number = 1 column_number = 1 - item_number = 1 + item_number = "1" field_name = "test field name" case_number = '1' rpt_month_year = 202001 From a623e00e326b7b1c8e8e3a5b5bf006f20496b833 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 27 Jun 2023 11:40:22 -0600 Subject: [PATCH 052/120] pipeline filtering (#2538) * pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md --------- Co-authored-by: George Hudson Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .circleci/README.md | 4 +- .circleci/base_config.yml | 18 ++++++ .circleci/build-and-test/workflows.yml | 62 ++++++++++++++++--- .circleci/config.yml | 39 +++++++++++- .circleci/infrastructure/workflows.yml | 8 ++- .github/workflows/build-all.yml | 58 +++++++++++++++++ .github/workflows/build-backend.yml | 44 +++++++++++++ .github/workflows/build-frontend.yml | 44 +++++++++++++ .github/workflows/build-pr.yml | 54 ++++++++++++++++ .github/workflows/deploy-develop-on-merge.yml | 2 +- .github/workflows/deploy-infrastructure.yml | 46 ++++++++++++++ .../020-pipeline-build-flow.md | 44 +++++++++++++ .../Technical-Documentation/github-actions.md | 5 ++ 13 files changed, 414 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/build-all.yml create mode 100644 .github/workflows/build-backend.yml create mode 100644 .github/workflows/build-frontend.yml create mode 100644 .github/workflows/build-pr.yml create mode 100644 .github/workflows/deploy-infrastructure.yml create mode 100644 docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md create mode 100644 docs/Technical-Documentation/github-actions.md diff --git a/.circleci/README.md b/.circleci/README.md index 39e484871..1b0c4c031 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -14,10 +14,10 @@ This script will generate a complete config for building, testing, and deploying ### Directory structure #### build-and-test -Contains workflows, jobs, and commands for building and testing the application. +Contains workflows, jobs, and commands for building and testing the application. For all development side builds, these are now triggered by GitHub Actions that serve as a filter so only the code that's changed is tested. See [build-all](../.github/workflows/build-all.yml), [build-backend](../.github/workflows/build-backend.yml), and [build-frontend](../.github/workflows/build-frontend.yml) #### infrastructure -Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. +Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. For all development side builds, this is now triggered by GitHub Actions that serve as a filter so only runs when infrastructure code is changed. See [deploy-infrastructure](../.github/workflows/deploy-infrastructure.yml) #### deployment Contains workflows, jobs, and commands for deploying the application on Cloud gov. Note: merges to develop now automatically trigger a develop deploy using [deploy-develop-on-merge](../.github/workflows/deploy-develop-on-merge.yml) and deploys to dev environments happen when a label is created on the PR using [deploy-on-label](../.github/workflows/deploy-on-label.yml) diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml index eb4b9af15..d722abecf 100644 --- a/.circleci/base_config.yml +++ b/.circleci/base_config.yml @@ -21,6 +21,18 @@ executors: resource_class: large parameters: + build_and_test_all: + type: boolean + default: false + build_and_test_backend: + type: boolean + default: false + build_and_test_frontend: + type: boolean + default: false + deploy_infrastructure: + type: boolean + default: false develop_branch_deploy: type: boolean default: false @@ -36,3 +48,9 @@ parameters: target_env: type: string default: '' + triggered: + type: boolean + default: false + util_make_erd: + type: boolean + default: false diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml index 9122ab166..7c0e559b0 100644 --- a/.circleci/build-and-test/workflows.yml +++ b/.circleci/build-and-test/workflows.yml @@ -1,19 +1,67 @@ # workflows: - build-and-test: - unless: - or: - - << pipeline.parameters.run_dev_deployment >> - - << pipeline.parameters.develop_branch_deploy >> - - << pipeline.parameters.run_owasp_scan >> - - << pipeline.parameters.run_nightly_owasp_scan >> + build-and-test-all: + when: << pipeline.parameters.build_and_test_all >> jobs: - secrets-check + - test-backend: + requires: + - secrets-check - test-frontend: requires: - secrets-check + - test-e2e: + requires: + - secrets-check + + ci-build-and-test-all: + jobs: + - secrets-check: + filters: + branches: + only: + - main + - master + - /^release.*/ - test-backend: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - test-frontend: + filters: + branches: + only: + - main + - master + - /^release.*/ requires: - secrets-check - test-e2e: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + + build-and-test-backend: + when: << pipeline.parameters.build_and_test_backend >> + jobs: + - secrets-check + - test-backend: + requires: + - secrets-check + + build-and-test-frontend: + when: << pipeline.parameters.build_and_test_frontend >> + jobs: + - secrets-check + - test-frontend: requires: - secrets-check diff --git a/.circleci/config.yml b/.circleci/config.yml index ea0ddd1c4..5d0be7af6 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,18 @@ orbs: # parameters from github actions parameters: + build_and_test_all: + type: boolean + default: false + build_and_test_backend: + type: boolean + default: false + build_and_test_frontend: + type: boolean + default: false + deploy_infrastructure: + type: boolean + default: false develop_branch_deploy: type: boolean default: false @@ -28,6 +40,12 @@ parameters: target_env: type: string default: '' + triggered: + type: boolean + default: false + util_make_erd: + type: boolean + default: false jobs: setup: @@ -45,6 +63,23 @@ jobs: # our single workflow, that triggers the setup job defined above workflows: - setup: + github-triggered-setup: + when: << pipeline.parameters.triggered >> + jobs: + - setup: + filters: + branches: + ignore: + - main + - master + - /^release.*/ + circle-ci-setup: jobs: - - setup + - setup: + filters: + branches: + only: + - main + - master + - /^release.*/ + diff --git a/.circleci/infrastructure/workflows.yml b/.circleci/infrastructure/workflows.yml index 46f29aec7..9cb85e8ed 100644 --- a/.circleci/infrastructure/workflows.yml +++ b/.circleci/infrastructure/workflows.yml @@ -1,6 +1,10 @@ -#workflows: +#workflows: enable-versioning-for-s3-buckets: - unless: << pipeline.parameters.run_nightly_owasp_scan >> + when: + or: + - << pipeline.parameters.deploy_infrastructure >> + - equal: [ 'main', << pipeline.git.branch >> ] + - equal: [ 'master', << pipeline.git.branch >> ] jobs: - enable-versioning: filters: diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml new file mode 100644 index 000000000..c03d344ab --- /dev/null +++ b/.github/workflows/build-all.yml @@ -0,0 +1,58 @@ +########################################################################### +# GitHub Action Workflow +# On changes to scripts or changes to the pipeline code to any branch +# besides develop, staging and master triggers the full build and test +# pipeline. +# +# NOTE: develop, staging(main) and master are skipped on the push because this +# would be redundant after running the full set of tests from the PR. +# See build-pr.yml for builds that run on code being merged into develop. +# See deploy-develop-on-merge.yml and make_erd for the workflow +# pipelines that run on merge to develop, staging, and master branches. +# HHS (main and master and release/**) branches build all only +# and are managed in CircleCI +# +# Step 0: Make changes on your branch to files in scripts/ .circleci or .github +# and push changes to your remote branch. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend and +# build_and_test_frontend to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build and test All on push when scripts/commands change +on: + push: + branches-ignore: + - develop + - main + - master + - 'release/**' + paths: + - 'scripts/**' + - '.circleci/**' + - '.github/**' +jobs: + build_and_test_all: + runs-on: ubuntu-latest + name: Initiate deploy job in CircleCI + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }} + payload: '{ + "build_and_test_all": true, + "triggered": true + }' diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 000000000..26ef5c03e --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,44 @@ +########################################################################### +# GitHub Action Workflow +# On push to any branch, triggers the back end build and test pipeline +# if the tdrs-backend has changed. +# +# Step 0: make changes on your branch to non-documentation files in +# tdrs-backend and push changes to your remote branch +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend +# to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build Only Backend When tdrs-backend/ Files Change +on: + push: + paths: 'tdrs-backend/**' + branches-ignore: + - develop + - main + - master +jobs: + build_and_test_backend: + runs-on: ubuntu-latest + name: Build and Test Backend + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "build_and_test_backend": true, + "triggered": true + }' diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml new file mode 100644 index 000000000..b9b60a914 --- /dev/null +++ b/.github/workflows/build-frontend.yml @@ -0,0 +1,44 @@ +########################################################################### +# GitHub Action Workflow +# On push to any branch, triggers the front end build and test pipeline +# if the tdrs-frontend has changed. +# +# Step 0: make changes on your branch to non-documentation files in +# tdrs-frontend and push changes to your remote branch +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_frontend +# to run the workflow/jobs listed here: +# build-and-test:[ +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build Only Frontend When tdrs-frontend Files Change +on: + push: + paths: 'tdrs-frontend/**' + branches-ignore: + - develop + - main + - master +jobs: + build_and_test_frontend: + runs-on: ubuntu-latest + name: Build and Test Frontend + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "build_and_test_frontend": true, + "triggered": true + }' diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml new file mode 100644 index 000000000..fa54a2a09 --- /dev/null +++ b/.github/workflows/build-pr.yml @@ -0,0 +1,54 @@ +########################################################################### +# GitHub Action Workflow +# On pull requests requesting review from individuals, besides staging, +# master, and release branches triggers the full build and test pipeline. +# +# NOTE: release branches, staging(main) and master are skipped because +# these branch builds are managed in CircleCI +# +# Step 0: make PR from your branch into develop, then select reviewers. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend and +# build_and_test_frontend to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build and test All for PRs +on: + pull_request: + branches-ignore: #handled in circleci + - main + - master + - 'release/**' + types: [review_requested, ready_for_review, synchronize] + paths-ignore: + - 'docs/**' + - '**.md' + - '**.txt' + - '.gitattributes' + - '.gitignore' + - 'LICENSE' +jobs: + build_and_test_pr: + runs-on: ubuntu-latest + name: Initiate deploy job in CircleCI + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }} + payload: '{ + "build_and_test_all": true, + "triggered": true + }' \ No newline at end of file diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml index 80ad9b308..a9df82645 100644 --- a/.github/workflows/deploy-develop-on-merge.yml +++ b/.github/workflows/deploy-develop-on-merge.yml @@ -24,7 +24,7 @@ on: push: branches: - develop - paths_ignore: + paths-ignore: - 'docs/**' - '**.md' - '**.txt' diff --git a/.github/workflows/deploy-infrastructure.yml b/.github/workflows/deploy-infrastructure.yml new file mode 100644 index 000000000..e5eeeb611 --- /dev/null +++ b/.github/workflows/deploy-infrastructure.yml @@ -0,0 +1,46 @@ +########################################################################### +# GitHub Action Workflow +# On push changing terraform files or infrastructure pipelines, triggers the +# terraform deploy pipeline for the appropriate cf space. +# +# Step 0: make changes to non-documentation files in terraform/ or +# .circleci/infrastructure/ and push/merge changes. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon terraform: true flag +# to run the workflow/jobs listed here: +# build-and-test:[ +# enable-versioning-for-s3-buckets +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Run Infrastructure Pipeline When Terraform or Infrastructure Files Change +on: + push: + branches-ignore: #handled in CircleCI + - main + - master + - 'release/**' + paths: + - 'terraform/**' + - '.circleci/infrastructure/**' +jobs: + run_infrastructure_deployment: + runs-on: ubuntu-latest + name: Deploy Infrastructure + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "deploy_infrastructure": true, + "triggered": true + }' + \ No newline at end of file diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md new file mode 100644 index 000000000..0e5f9b17c --- /dev/null +++ b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md @@ -0,0 +1,44 @@ +# 20. Pipeline Build Flow + +Date: 2023-06-07 (_Updated 2023-06-13_) + +## Status + +Pending + +## Context + +We use [CircleCI](https://app.circleci.com/pipelines/github/raft-tech/TANF-app)] as our primary pipeline build tool. On the HHS side, the standard pipeline, build all and deploy ran by CircleCI is sufficiently granular to meet all our build needs. However, on the dev Raft-TANF side, where work can often be held up by waiting for build and test pipelines, it's useful to reduce build times by filtering which workflows run based upon which code is changing. In order to do this, GitHub Actions is leveraged to kick off different CircleCI build workflows since Actions has more granular control over what paths are changed. + +These pipelines are closely tied to the [Git workflow](./009-git-workflow.md) and should run build and test as expected based on what code changed, and still deploy to the correct CF spaces based upon the Decision tree in the [Deployment Flow](./008-deployment-flow.md) + +## Build Logic + +For all release branches, the main, and the master branch, CircleCI should run the full build, infrastructure deploy, and app deployment. + +for feature/development branches, only the build and test pertainint to the code changed should be built. +Front end tests should be run if /tdrs-frontent changes +Back end tests should be run if /tdrs-backend changes +the entire build and test all should run if anything pertaining to the pipeline changes +infrastructure deploy should run if /terraform or infrastructure deploy pipeline changes + +Once a pull request is flagged as ready for review and/or has reviewers assigned, a full build and test all should be run (and tests must pass before merge to develop) + +Develop merges trigger a deploy to the develop CF space, and then a special integration end-2-end test that tests against the real development environment instead of a simulated environment on the CircleCI build servers. + +## Consequences + +**Pros** +* reduce time of build and tests by only running the appropriate workflows. +* only run infrastructure deploys when changes are made to infrastructure code. +* only run end-2-end tests when code is ready to be reviewed. +* only run full integration end-2-end testing when the develop environment assets are updated. + +**Risks** +* Increased pipeline logic complexity + +## Notes + +- For the nuanced build triggers, CircleCI documentation recommends using Actions to trigger CircleCI builds and send different flags to tell which workflows should run. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0 + +- This could be an argument for future complete pipeline switchover to GitHub Actions in order to reduce pipeline complexity, while maintaining the desired build granularity. \ No newline at end of file diff --git a/docs/Technical-Documentation/github-actions.md b/docs/Technical-Documentation/github-actions.md new file mode 100644 index 000000000..05839498a --- /dev/null +++ b/docs/Technical-Documentation/github-actions.md @@ -0,0 +1,5 @@ +# How We Use GitHub Actions +For now, the only use case we have for GitHub Actions is to help up trigger CircleCI builds the way we want to. This is actually the preferred method CircleCI advises for branch, path, pull-request, and labelled filtering and job triggering. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0 + +## Path Filtering +We use Actions to filter which workflows are getting run by CircleCI by sending different flags to CircleCI through the promiseofcake CircleCI API trigger. See the individual files in [.github](../../.github/) for detailed instructions for how to use each. From b6d8b777f8b24f1ea4db313daf6fc17050f452cd Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 29 Jun 2023 08:54:40 -0600 Subject: [PATCH 053/120] Hotfix Devops/2457 path filtering for documentation (#2597) * pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md * build and test all on PRs even for documentation --------- Co-authored-by: George Hudson --- .github/workflows/build-pr.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index fa54a2a09..5aef71099 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -28,13 +28,6 @@ on: - master - 'release/**' types: [review_requested, ready_for_review, synchronize] - paths-ignore: - - 'docs/**' - - '**.md' - - '**.txt' - - '.gitattributes' - - '.gitignore' - - 'LICENSE' jobs: build_and_test_pr: runs-on: ubuntu-latest From 0dcd36a026b9f9e51dba12b4c4e5432a5dd87a25 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Thu, 29 Jun 2023 08:05:08 -0700 Subject: [PATCH 054/120] Create sprint-74-summary.md (#2596) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-74-summary.md | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/Sprint-Review/sprint-74-summary.md diff --git a/docs/Sprint-Review/sprint-74-summary.md b/docs/Sprint-Review/sprint-74-summary.md new file mode 100644 index 000000000..9d64430d3 --- /dev/null +++ b/docs/Sprint-Review/sprint-74-summary.md @@ -0,0 +1,54 @@ +# Sprint 74 Summary +05/24/23 - 06/06/23 + +Velocity: 21 + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX to continue STT onboarding (focusing on onboarding CyberFusion users), errors research sessions, provide copy for 2509 (e-mail notification for data submission and errors/transmission report) +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + +## Tickets + +#### Completed/Merged +* [#2439 - [Design] Video Release — Embed videos in knowledge center +](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2439) +* [#2503 [Design] May Knowledge Center Enhancements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2503) +* [#2531 Bug/Max file size](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2531) +* [#2424 Complete TANF section 1 parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) +* [#2368 errors reporting research](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2368) + + + +#### Submitted (QASP Review, OCIO Review) +* N/A + + +#### Closed (not merged) +* N/A + +### Moved to Next Sprint (Blocked, Raft Review, In Progress) + +#### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +#### Raft Review +* [2486 [Spike - Parser Performance]](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2521 - Update to CFlinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2550 - Deactivation warning emails are missing hyperlinks in staging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2550) + +#### In Progress +* [#2116 [DevOps] Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + + + +### Sprint 74 Demo +* Internal: + * UX walkthrough of Knowledge Center enhancements + * [#2424 Complete TANF section 1 parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) +* External: + * N/A From 9aef162acf32c7a8b33a35ad0b2d36957b04b2c8 Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Wed, 5 Jul 2023 09:52:35 -0400 Subject: [PATCH 055/120] added URL filters (#2580) * added URL filters * allow github to trigger owasp and label deploys (#2601) Co-authored-by: George Hudson --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> Co-authored-by: George Hudson Co-authored-by: George Hudson --- .github/workflows/deploy-develop-on-merge.yml | 6 +- .github/workflows/deploy-on-label.yml | 6 +- .github/workflows/qasp-owasp-scan.yml | 3 +- scripts/zap-scanner.sh | 80 ++++++++++++++++++- 4 files changed, 90 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml index a9df82645..ee4eee057 100644 --- a/.github/workflows/deploy-develop-on-merge.yml +++ b/.github/workflows/deploy-develop-on-merge.yml @@ -45,4 +45,8 @@ jobs: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} project-slug: ${{ github.repository }} branch: ${{ github.ref_name }} - payload: '{"develop_branch_deploy": true, "target_env": "develop"}' + payload: '{ + "develop_branch_deploy": true, + "target_env": "develop", + "triggered": true + }' diff --git a/.github/workflows/deploy-on-label.yml b/.github/workflows/deploy-on-label.yml index 857dd3a35..20fbfd8b3 100644 --- a/.github/workflows/deploy-on-label.yml +++ b/.github/workflows/deploy-on-label.yml @@ -66,4 +66,8 @@ jobs: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} project-slug: ${{ github.repository }} branch: ${{ github.head_ref }} - payload: '{"run_dev_deployment": true, "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}"}' + payload: '{ + "run_dev_deployment": true, + "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}", + "triggered": true + }' diff --git a/.github/workflows/qasp-owasp-scan.yml b/.github/workflows/qasp-owasp-scan.yml index 3a4dea99b..963bf430c 100644 --- a/.github/workflows/qasp-owasp-scan.yml +++ b/.github/workflows/qasp-owasp-scan.yml @@ -34,5 +34,6 @@ jobs: branch: ${{ github.head_ref }} payload: | { - "run_owasp_scan": ${{ env.HAS_QASP_LABEL }} + "run_owasp_scan": ${{ env.HAS_QASP_LABEL }}, + "triggered": true } diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh index 4e017eb7e..dbce417a8 100755 --- a/scripts/zap-scanner.sh +++ b/scripts/zap-scanner.sh @@ -59,15 +59,91 @@ ZAP_CLI_OPTIONS="\ -config globalexcludeurl.url_list.url\(0\).regex='.*/robots\.txt.*' \ -config globalexcludeurl.url_list.url\(0\).description='Exclude robots.txt' \ -config globalexcludeurl.url_list.url\(0\).enabled=true \ + -config globalexcludeurl.url_list.url\(1\).regex='^https?://.*\.cdn\.mozilla\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(1\).description='Site - Mozilla CDN (requests such as getpocket)' \ -config globalexcludeurl.url_list.url\(1\).enabled=true \ + -config globalexcludeurl.url_list.url\(2\).regex='^https?://.*\.amazonaws\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(2\).description='TDP S3 buckets' \ -config globalexcludeurl.url_list.url\(2\).enabled=true \ - -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.acf.hhs.gov\/v1\/login\/.*$' \ - -config globalexcludeurl.url_list.url\(3\).description='Site - identity pages' \ + + -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.hhs.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(3\).description='Site - acf.hhs.gov' \ -config globalexcludeurl.url_list.url\(3\).enabled=true \ + + -config globalexcludeurl.url_list.url\(4\).regex='^https:\/\/.*\.google.com\/.*$' \ + -config globalexcludeurl.url_list.url\(4\).description='Site - Google.com' \ + -config globalexcludeurl.url_list.url\(4\).enabled=true \ + + -config globalexcludeurl.url_list.url\(5\).regex='^https:\/\/.*\.youtube.com\/.*$' \ + -config globalexcludeurl.url_list.url\(5\).description='Site - youtube.com' \ + -config globalexcludeurl.url_list.url\(5\).enabled=true \ + + -config globalexcludeurl.url_list.url\(6\).regex='^https:\/\/.*\.monsido.com\/.*$' \ + -config globalexcludeurl.url_list.url\(6\).description='Site - monsido.com' \ + -config globalexcludeurl.url_list.url\(6\).enabled=true \ + + -config globalexcludeurl.url_list.url\(7\).regex='^https:\/\/.*\.crazyegg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(7\).description='Site - crazyegg.com' \ + -config globalexcludeurl.url_list.url\(7\).enabled=true \ + + -config globalexcludeurl.url_list.url\(8\).regex='^https:\/\/.*\.gstatic.com\/.*$' \ + -config globalexcludeurl.url_list.url\(8\).description='Site - gstatic.com' \ + -config globalexcludeurl.url_list.url\(8\).enabled=true \ + + -config globalexcludeurl.url_list.url\(9\).regex='^https:\/\/.*\.googleapis.com\/.*$' \ + -config globalexcludeurl.url_list.url\(9\).description='Site - GoogleAPIs.com' \ + -config globalexcludeurl.url_list.url\(9\).enabled=true \ + + -config globalexcludeurl.url_list.url\(10\).regex='^https:\/\/.*\.crazyegg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(10\).description='Site - CrazyEgg.com' \ + -config globalexcludeurl.url_list.url\(10\).enabled=true \ + + -config globalexcludeurl.url_list.url\(11\).regex='^https:\/\/.*\.doubleclick.net\/.*$' \ + -config globalexcludeurl.url_list.url\(11\).description='Site - DoubleClick.net' \ + -config globalexcludeurl.url_list.url\(11\).enabled=true \ + + -config globalexcludeurl.url_list.url\(12\).regex='^https:\/\/.*unpkg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(12\).description='Site - Unpkg.com' \ + -config globalexcludeurl.url_list.url\(12\).enabled=true \ + + -config globalexcludeurl.url_list.url\(13\).regex='^https:\/\/.*\.readspeaker.com\/.*$' \ + -config globalexcludeurl.url_list.url\(13\).description='Site - ReadSpeaker.com' \ + -config globalexcludeurl.url_list.url\(13\).enabled=true \ + + -config globalexcludeurl.url_list.url\(14\).regex='^https:\/\/.*\.fontawesome.com\/.*$' \ + -config globalexcludeurl.url_list.url\(14\).description='Site - FontAwesome.com' \ + -config globalexcludeurl.url_list.url\(14\).enabled=true \ + + -config globalexcludeurl.url_list.url\(15\).regex='^https:\/\/.*\.cloud.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(15\).description='Site - Cloud.gov' \ + -config globalexcludeurl.url_list.url\(15\).enabled=true \ + + -config globalexcludeurl.url_list.url\(16\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \ + -config globalexcludeurl.url_list.url\(16\).description='Site - googletagmanager.com' \ + -config globalexcludeurl.url_list.url\(16\).enabled=true \ + + -config globalexcludeurl.url_list.url\(17\).regex='^https:\/\/.*\.cloudflare.com\/.*$' \ + -config globalexcludeurl.url_list.url\(17\).description='Site - CloudFlare.com' \ + -config globalexcludeurl.url_list.url\(17\).enabled=true \ + + -config globalexcludeurl.url_list.url\(18\).regex='^https:\/\/.*\.google-analytics.com\/.*$' \ + -config globalexcludeurl.url_list.url\(18\).description='Site - Google-Analytics.com' \ + -config globalexcludeurl.url_list.url\(18\).enabled=true \ + + -config globalexcludeurl.url_list.url\(19\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \ + -config globalexcludeurl.url_list.url\(19\).description='Site - googletagmanager.com' \ + -config globalexcludeurl.url_list.url\(19\).enabled=true \ + + -config globalexcludeurl.url_list.url\(20\).regex='^https:\/\/.*\.digitalgov.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(20\).description='Site - DigitalGov.gov' \ + -config globalexcludeurl.url_list.url\(20\).enabled=true \ + + -config globalexcludeurl.url_list.url\(21\).regex='^https:\/\/.*\.identitysandbox.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(21\).description='Site - IdentitySandbox.gov' \ + -config globalexcludeurl.url_list.url\(21\).enabled=true \ + -config spider.postform=true" # How long ZAP will crawl the app with the spider process From 91cc3313bed30021e316641104adea378ee8ba8c Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Mon, 10 Jul 2023 06:43:09 -0700 Subject: [PATCH 056/120] Create sprint-75-summary.md (#2608) --- docs/Sprint-Review/sprint-75-summary.md | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/Sprint-Review/sprint-75-summary.md diff --git a/docs/Sprint-Review/sprint-75-summary.md b/docs/Sprint-Review/sprint-75-summary.md new file mode 100644 index 000000000..967b0381b --- /dev/null +++ b/docs/Sprint-Review/sprint-75-summary.md @@ -0,0 +1,49 @@ +# Sprint 75 Summary + +06/06/23 - 06/20/23 + +Velocity: Dev (1) + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX to continue STT onboarding (focusing on onboarding CyberFusion users), errors research synthesis, copy for e-mail notification of data submission and errors/transmission reports - 2559 +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [2550 Deactivation warning emails are missing e-mail links in staging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2550) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (In Progress, Blocked, Raft Review) + +### In Progress +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) + +### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +### Raft Review +* [#2563 Assess OWASP scan accuracy and URL filter](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2563) +* [#2551 [Bug] - All users are not returned in API response](https://](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2551)) +* [#2564 Adjust TANF and SSP Section 1 item numbers](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2564) +* [#2457 Path filtering for CI Builds](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2457) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2516 Errors Research Synthesis ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2516) + + From baffaba6c83ba204d56763bef3e394ab69477672 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Mon, 10 Jul 2023 06:54:17 -0700 Subject: [PATCH 057/120] Create sprint-76-summary.md (#2609) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-76-summary.md | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/Sprint-Review/sprint-76-summary.md diff --git a/docs/Sprint-Review/sprint-76-summary.md b/docs/Sprint-Review/sprint-76-summary.md new file mode 100644 index 000000000..0d35a7314 --- /dev/null +++ b/docs/Sprint-Review/sprint-76-summary.md @@ -0,0 +1,54 @@ +# Sprint 76 Summary + +06/21/23 - 07/04/23 + +Velocity: Dev (17) + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX errors template, follow-on research, onboarding +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [#2563 Assess OWASP scan accuracy and URL filter](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2563) +* [#2564 Adjust TANF and SSP Section 1 item numbers](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2564) +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2551 [Bug] - All users are not returned in API response](https://](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2551)) +* [#2457 Path filtering for CI Builds](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2457) +* [#2516 Errors Research Synthesis ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2516) +* [#2527 Error Research informed excel prototype](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2527) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1784 - Email Relay](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1784) +* [#2347 decouple backend apps spike](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2347) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2598 HHS Staging Deployment Failure](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2598) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) + + +### Blocked +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) + + +### Raft Review +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + + + From b9056678ad39ed60f79796d0fd22520fa41fab1a Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 10:44:17 -0600 Subject: [PATCH 058/120] - Resolved failing tests --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index dbab07165..6329fef49 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -64,7 +64,6 @@ def test_parse_small_correct_file(test_datafile, dfs): def test_parse_section_mismatch(test_datafile, dfs): """Test parsing of small_correct_file where the DataFile section doesn't match the rawfile section.""" test_datafile.section = 'Closed Case Data' - test_datafile.save() dfs.datafile = test_datafile dfs.save() @@ -518,6 +517,8 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 + for e in parser_errors: + print(e.error_type, e.error_message) row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE From 29fa8b8953d1e61bf53808d04fdf5e20b1486616 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 11:54:41 -0600 Subject: [PATCH 059/120] - Corrected merge thrash --- tdrs-backend/tdpservice/parsers/parse.py | 33 +++++++++++++------ .../tdpservice/parsers/test/test_parse.py | 1 + 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 66a18a2dc..fba6d4d96 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -152,25 +152,38 @@ def parse_datafile_lines(datafile, program_type, section): def parse_multi_record_line(line, schema, generate_error): """Parse and validate a datafile line using MultiRecordRowSchema.""" - records = schema.parse_and_validate(line, generate_error) + if schema: + records = schema.parse_and_validate(line, generate_error) - for r in records: - record, record_is_valid, record_errors = r + for r in records: + record, record_is_valid, record_errors = r - if record: - record.save() + if record: + record.save() - return records + return records + + return [(None, False, [ + generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Record Type is missing from record.", + record=None, + field=None + ) + ])] def parse_datafile_line(line, schema, generate_error): """Parse and validate a datafile line and save any errors to the model.""" - record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) + if schema: + record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) + + if record: + record.save() - if record: - record.save() + return record_is_valid, record_errors - return record_is_valid, record_errors return (False, [ generate_error( schema=None, diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 6329fef49..4fbe0ae1e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -64,6 +64,7 @@ def test_parse_small_correct_file(test_datafile, dfs): def test_parse_section_mismatch(test_datafile, dfs): """Test parsing of small_correct_file where the DataFile section doesn't match the rawfile section.""" test_datafile.section = 'Closed Case Data' + test_datafile.save() dfs.datafile = test_datafile dfs.save() From ac9b0659a62f64c4114c41faf0baa659a92be07c Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 13:28:45 -0600 Subject: [PATCH 060/120] - Using randbits to generate pk to get around confilcting sequence pks --- tdrs-backend/tdpservice/stts/test/factories.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/stts/test/factories.py b/tdrs-backend/tdpservice/stts/test/factories.py index 2e20b62cf..19f5a279d 100644 --- a/tdrs-backend/tdpservice/stts/test/factories.py +++ b/tdrs-backend/tdpservice/stts/test/factories.py @@ -2,6 +2,7 @@ import factory from ..models import STT, Region +import random class RegionFactory(factory.django.DjangoModelFactory): @@ -12,7 +13,7 @@ class Meta: model = "stts.Region" - id = factory.Sequence(int) + id = random.getrandbits(16) @classmethod def _create(cls, model_class, *args, **kwargs): From 6c8b3c4bbf1eed9f3b11a5da2c5d2d4c0b923483 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 13:42:02 -0600 Subject: [PATCH 061/120] Revert "- Using randbits to generate pk to get around confilcting sequence pks" This reverts commit ac9b0659a62f64c4114c41faf0baa659a92be07c. --- tdrs-backend/tdpservice/stts/test/factories.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/stts/test/factories.py b/tdrs-backend/tdpservice/stts/test/factories.py index 19f5a279d..2e20b62cf 100644 --- a/tdrs-backend/tdpservice/stts/test/factories.py +++ b/tdrs-backend/tdpservice/stts/test/factories.py @@ -2,7 +2,6 @@ import factory from ..models import STT, Region -import random class RegionFactory(factory.django.DjangoModelFactory): @@ -13,7 +12,7 @@ class Meta: model = "stts.Region" - id = random.getrandbits(16) + id = factory.Sequence(int) @classmethod def _create(cls, model_class, *args, **kwargs): From 88d4b62ca57092acd2c4fe717e8fc3bd749f9ea7 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 13:52:46 -0600 Subject: [PATCH 062/120] - Updating region in fixture instead of factory - letting django handle transaction for test --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 2 ++ tdrs-backend/tdpservice/parsers/test/test_summary.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 4fbe0ae1e..d51a1a300 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -29,6 +29,8 @@ def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): @pytest.fixture def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" + stt.region = None + stt.save() return create_test_datafile('small_correct_file', stt_user, stt) @pytest.fixture diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 4f92ab865..14ca122ba 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -24,7 +24,7 @@ def test_dfs_model(dfs): """Test that the model is created and populated correctly.""" assert dfs.case_aggregates['Jan']['accepted'] == 100 -@pytest.mark.django_db(transaction=True) +@pytest.mark.django_db def test_dfs_rejected(test_datafile, dfs): """Ensure that an invalid file generates a rejected status.""" test_datafile.section = 'Closed Case Data' From cb5077c619edf6c12c81f96b55b9e0792d928224 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 14:10:30 -0600 Subject: [PATCH 063/120] - Moved datafile reference to avoid confusion --- .../tdpservice/search_indexes/admin/ssp.py | 6 +++--- .../tdpservice/search_indexes/admin/tanf.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py index 59cc4709d..bfcfb901e 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/ssp.py @@ -6,13 +6,13 @@ class SSP_M1Admin(admin.ModelAdmin): """ModelAdmin class for parsed M1 data files.""" list_display = [ - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', 'COUNTY_FIPS_CODE', 'ZIP_CODE', 'STRATUM', + 'datafile', ] list_filter = [ @@ -26,10 +26,10 @@ class SSP_M2Admin(admin.ModelAdmin): """ModelAdmin class for parsed M2 data files.""" list_display = [ - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', + 'datafile', ] list_filter = [ @@ -41,10 +41,10 @@ class SSP_M3Admin(admin.ModelAdmin): """ModelAdmin class for parsed M3 data files.""" list_display = [ - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', + 'datafile', ] list_filter = [ diff --git a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py index b03313662..54714d3e2 100644 --- a/tdrs-backend/tdpservice/search_indexes/admin/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/admin/tanf.py @@ -7,13 +7,13 @@ class TANF_T1Admin(admin.ModelAdmin): """ModelAdmin class for parsed T1 data files.""" list_display = [ - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', 'COUNTY_FIPS_CODE', 'ZIP_CODE', 'STRATUM', + 'datafile', ] list_filter = [ @@ -28,10 +28,10 @@ class TANF_T2Admin(admin.ModelAdmin): """ModelAdmin class for parsed T2 data files.""" list_display = [ - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', + 'datafile', ] list_filter = [ @@ -44,10 +44,10 @@ class TANF_T3Admin(admin.ModelAdmin): """ModelAdmin class for parsed T3 data files.""" list_display = [ - 'datafile', 'RecordType', 'RPT_MONTH_YEAR', 'CASE_NUMBER', + 'datafile', ] list_filter = [ @@ -60,10 +60,10 @@ class TANF_T4Admin(admin.ModelAdmin): """ModelAdmin class for parsed T4 data files.""" list_display = [ - 'datafile', 'record', 'rpt_month_year', 'case_number', + 'datafile', ] list_filter = [ @@ -75,10 +75,10 @@ class TANF_T5Admin(admin.ModelAdmin): """ModelAdmin class for parsed T5 data files.""" list_display = [ - 'datafile', 'record', 'rpt_month_year', 'case_number', + 'datafile', ] list_filter = [ @@ -90,9 +90,9 @@ class TANF_T6Admin(admin.ModelAdmin): """ModelAdmin class for parsed T6 data files.""" list_display = [ - 'datafile', 'record', 'rpt_month_year', + 'datafile', ] list_filter = [ @@ -104,9 +104,9 @@ class TANF_T7Admin(admin.ModelAdmin): """ModelAdmin class for parsed T7 data files.""" list_display = [ - 'datafile', 'record', 'rpt_month_year', + 'datafile', ] list_filter = [ From 6d3dfbec6b87d47eeb40e79d64135318e7c36218 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 11 Jul 2023 15:12:16 -0400 Subject: [PATCH 064/120] pushing up incomplete codebase --- tdrs-backend/tdpservice/parsers/models.py | 60 ++++++ tdrs-backend/tdpservice/parsers/parse.py | 180 ++++++++++++------ .../tdpservice/parsers/test/test_parse.py | 46 +++++ .../tdpservice/parsers/test/test_summary.py | 11 ++ 4 files changed, 239 insertions(+), 58 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 78b12a163..70f17a365 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -86,6 +86,11 @@ class Status(models.TextChoices): datafile = models.ForeignKey(DataFile, on_delete=models.CASCADE) + # TODO: worth adding more fields here or ??? + #named program schema + #named models schema via rowschema + #named section via datafile + # eventually needs a breakdown of cases (accepted, rejected, total) per month # elif qtr2 jan-mar # elif qtr3 apr-jun @@ -112,6 +117,61 @@ class Status(models.TextChoices): "total": 110 } """ + def transform_to_months(quarter): + """Return a list of months in a quarter.""" + match quarter: + case "Q1": + return ["Jan", "Feb", "Mar"] + case "Q2": + return ["Apr", "May", "Jun"] + case "Q3": + return ["Jul", "Aug", "Sep"] + case "Q4": + return ["Oct", "Nov", "Dec"] + case _: + raise ValueError("Invalid quarter value.") + + ''' + def case_aggregates_by_month(self): + """Return case aggregates by month.""" + df = self.datafile + section = transform_to_text(df.section) + + # from datafile quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' + month_list = transform_to_months(df.quarter) + # or we do upgrade get_schema_options to always take named params vs string text? + + # use or leverage `get_schema_options` to lookup model in RowSchema + models = [model for x in get_schema_options(program_type)[section]] + + + # using a django queryset, filter by datafile to find relevant search_index objects + + # count the number of objects in the queryset and assign to total + # using a queryset of parserError objects, filter by datafile and error_type to get count of rejected cases + # subtract rejected cases from total to get accepted cases + # return a dict of month: {accepted: x, rejected: y, total: z} + for month in month_list: + total = 0 + rejected = 0 + accepted = 0 + + for model in models: + total += model.objects.filter(datafile=df, month=month).count() # todo change to RPT_MONTH_YEAR + rejected += model.objects.filter(datafile=df, error.exists()).count() # todo filter doesn't actually work this way + #ParserError.objects.filter(datafile=df, month=month).count() #TODO filter where field_name != header or trailer ?? + accepted += total - rejected # again look for all objects where generic relation to error is false/empty + + case_aggregates_by_month[month] = { + + # filter by month + # filter by model + # filter by datafile + # count objects + # add to dict + ''' + + def get_status(self, errors): """Set and return the status field based on errors and models associated with datafile.""" diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 3ee7dffad..b45175022 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -44,31 +44,14 @@ def parse_datafile(datafile): if not trailer_is_valid: errors['trailer'] = trailer_errors - # ensure file section matches upload section - section_names = { - 'TAN': { - 'A': DataFile.Section.ACTIVE_CASE_DATA, - 'C': DataFile.Section.CLOSED_CASE_DATA, - 'G': DataFile.Section.AGGREGATE_DATA, - 'S': DataFile.Section.STRATUM_DATA, - }, - 'SSP': { - 'A': DataFile.Section.SSP_ACTIVE_CASE_DATA, - 'C': DataFile.Section.SSP_CLOSED_CASE_DATA, - 'G': DataFile.Section.SSP_AGGREGATE_DATA, - 'S': DataFile.Section.SSP_STRATUM_DATA, - }, - } - # TODO: utility transformations between text to schemas and back - # text > prog > sections > schemas program_type = header['program_type'] section = header['type'] section_is_valid, section_error = validators.validate_header_section_matches_submission( datafile, - section_names.get(program_type, {}).get(section) + get_section_field(program_type, section) ) if not section_is_valid: @@ -113,12 +96,13 @@ def parse_datafile(datafile): def parse_datafile_lines(datafile, program_type, section): """Parse lines with appropriate schema and return errors.""" + #dfs = DataFileSummary.object.create(datafile=datafile) + # and then what, pass in program_type to case_aggregates after loop? errors = {} rawfile = datafile.file rawfile.seek(0) line_number = 0 - schema_options = get_schema_options(program_type) for rawline in rawfile: line_number += 1 @@ -127,7 +111,7 @@ def parse_datafile_lines(datafile, program_type, section): if line.startswith('HEADER') or line.startswith('TRAILER'): continue - schema = get_schema(line, section, schema_options) + schema = get_schema(line, section, program_type) if schema is None: errors[line_number] = [util.generate_parser_error( datafile=datafile, @@ -190,51 +174,131 @@ def parse_datafile_line(line, schema, generate_error): return record_is_valid, record_errors +def get_schema_options(program, section, query=None, model=None, model_name=None): + ''' + Centralized function to return the appropriate schema for a given program, section, and query. + @param program: the abbreviated program type (.e.g, 'TAN') + @param section: the section of the file (.e.g, 'A') + @param query: the query for section_names (.e.g, 'section', 'models', etc.) + @return: the appropriate references (e.g., ACTIVE_CASE_DATA or {t1,t2,t3}) + ''' -def get_schema_options(program_type): - """Return the allowed schema options.""" - match program_type: - case 'TAN': - return { - 'A': { + # ensure file section matches upload section + schema_options = { + 'TAN': { + 'A': { + 'section': DataFile.Section.ACTIVE_CASE_DATA, + 'models': { 'T1': schema_defs.tanf.t1, 'T2': schema_defs.tanf.t2, 'T3': schema_defs.tanf.t3, - }, - 'C': { - # 'T4': schema_options.t4, - # 'T5': schema_options.t5, - }, - 'G': { - # 'T6': schema_options.t6, - }, - 'S': { - # 'T7': schema_options.t7, - }, + } + }, + 'C': { + 'section': DataFile.Section.CLOSED_CASE_DATA, + 'models': { + #'T4': schema_defs.tanf.t4, + #'T5': schema_defs.tanf.t5, + } + }, + 'G': { + 'section':DataFile.Section.AGGREGATE_DATA, + 'models': { + #'T6': schema_defs.tanf.t6, + } + }, + 'S': { + 'section':DataFile.Section.STRATUM_DATA, + 'models': { + #'T7': schema_defs.tanf.t7, + } } - case 'SSP': - return { - 'A': { - 'M1': schema_defs.ssp.m1, - 'M2': schema_defs.ssp.m2, - 'M3': schema_defs.ssp.m3, - }, - 'C': { - # 'M4': schema_options.m4, - # 'M5': schema_options.m5, - }, - 'G': { - # 'M6': schema_options.m6, - }, - 'S': { - # 'M7': schema_options.m7, - }, + }, + 'SSP': { + 'A': { + 'section': DataFile.Section.SSP_ACTIVE_CASE_DATA, + 'models': { + 'S1': schema_defs.ssp.m1, + 'S2': schema_defs.ssp.m2, + 'S3': schema_defs.ssp.m3, + } + }, + 'C': { + 'section':DataFile.Section.SSP_CLOSED_CASE_DATA, + 'models': { + #'S4': schema_defs.ssp.m4, + #'S5': schema_defs.ssp.m5, + } + }, + 'G': { + 'section':DataFile.Section.SSP_AGGREGATE_DATA, + 'models': { + #'S6': schema_defs.ssp.m6, + } + }, + 'S': { + 'section':DataFile.Section.SSP_STRATUM_DATA, + 'models': { + #'S7': schema_defs.ssp.m7, + } } - # case tribal? - return None + }, + # TODO: tribal tanf + } + + #TODO: add error handling for bad inputs -- how does `get` handle bad inputs? + if query == "text": + for prog_name, prog_dict in schema_options.items(): + for sect,val in prog_dict.items(): + if val==model: + return {'program_type':prog_name, 'section': sect} + raise ValueError("Model not found in schema_defs") + elif query == "section": + return schema_options.get(program, {}).get(section, None)[query] + elif query == "models": + links = schema_options.get(program, {}).get(section, None) + + # if query is not chosen or wrong input, return all options + # query = 'models', model = 'T1' + models = links.get(query, links) + + return models.get(model_name, models) + + +#TODO is it more flexible w/o option? we can do filtering in wrapper functions +# if option is empty, null, none, just return more + +''' +text -> section +text -> models{} YES +text -> model YES +datafile -> model + ^ section -> program -> model +datafile -> text +model -> text YES +section -> text + +text**: input string from the header/file +''' + +def get_program_models(str_prog, str_section): + return get_schema_options(program=str_prog, section=str_section, query='models') + +def get_program_model(str_prog, str_section, str_model): + return get_schema_options(program=str_prog, section=str_section, query='models', model_name=str_model) + +def get_section_reference(str_prog, str_section): + return get_schema_options(program=str_prog, section=str_section, query='section') + +def get_text_from_model(model): + get_schema_options() + + + + #TODO: if given a datafile (section), we can reverse back to the program b/c the section string has "tribal/ssp" in it, then process of elimination we have tanf -def get_schema(line, section, schema_options): +def get_schema(line, section, program_type): """Return the appropriate schema for the line.""" line_type = line[0:2] - return schema_options.get(section, {}).get(line_type, None) + return get_schema_options(program_type, section, query='models', model_name=line_type) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 00f2c90ee..65e8b2677 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -9,6 +9,7 @@ from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 from .factories import DataFileSummaryFactory +from .. import schema_defs def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): """Create a test DataFile instance with the given file attached.""" @@ -602,3 +603,48 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel 5: [row_5_error], 'trailer': [trailer_error], } + + +def test_get_schema_options(): + """Test use-cases for translating strings to named object references.""" + + ''' + text -> section + text -> models{} YES + text -> model YES + datafile -> model + ^ section -> program -> model + datafile -> text + model -> text YES + section -> text + + text**: input string from the header/file + ''' + + # from text: + # get schema + schema = parse.get_schema('T3', 'A', 'TAN') + assert schema == schema_defs.tanf.t3 + # get section + # get model + models = parse.get_program_models('TAN', 'A') + assert models == { + 'T1': schema_defs.tanf.t1, + 'T2': schema_defs.tanf.t2, + 'T3': schema_defs.tanf.t3, + } + + model = parse.get_program_model('TAN', 'A', 'T1') + assert model == schema_defs.tanf.t1 + + section = parse.get_section_reference('TAN','C') + assert section == DataFile.Section.CLOSED_CASE_DATA + # from datafile: + # get model(s) + # get section str + + # from model: + # get text + # get section str + # get ref section + diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 814fa2596..edf28c4b5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -54,3 +54,14 @@ def test_dfs_set_status(dfs): dfs.status = dfs.get_status(errors={'document': parser_errors}) assert dfs.status == DataFileSummary.Status.REJECTED + +@pytest.mark.django_db +def test_dfs_set_case_aggregates(test_datafile, dfs): + """Test that the case aggregates are set correctly.""" + test_datafile.section = 'Active Case Data' + test_datafile.save() + error_ast = parse.parse_datafile(test_datafile) + dfs.case_aggregates = dfs.get_case_aggregates(error_ast) + assert dfs.case_aggregates['Jan']['accepted'] == 1 + assert dfs.case_aggregates['Jan']['rejected'] == 0 + assert dfs.case_aggregates['Jan']['total'] == 1 \ No newline at end of file From fc02fd7dbb76690da25d6b0cfe4c820871884279 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 11 Jul 2023 15:55:24 -0400 Subject: [PATCH 065/120] Other unit tests now have passed w/ good error handling --- tdrs-backend/tdpservice/parsers/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 70f17a365..fca5d8498 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -172,7 +172,6 @@ def case_aggregates_by_month(self): ''' - def get_status(self, errors): """Set and return the status field based on errors and models associated with datafile.""" if errors is None: From e13760967c0ada3f21439e9e31b3ae1edf69233d Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 11 Jul 2023 17:34:24 -0400 Subject: [PATCH 066/120] Working tests, need to get setup for case aggregates populating via DB --- tdrs-backend/tdpservice/parsers/models.py | 53 ----- tdrs-backend/tdpservice/parsers/parse.py | 132 +---------- .../tdpservice/parsers/test/test_parse.py | 20 +- tdrs-backend/tdpservice/parsers/util.py | 217 ++++++++++++++++++ 4 files changed, 232 insertions(+), 190 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index fca5d8498..e8e50b589 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -117,59 +117,6 @@ class Status(models.TextChoices): "total": 110 } """ - def transform_to_months(quarter): - """Return a list of months in a quarter.""" - match quarter: - case "Q1": - return ["Jan", "Feb", "Mar"] - case "Q2": - return ["Apr", "May", "Jun"] - case "Q3": - return ["Jul", "Aug", "Sep"] - case "Q4": - return ["Oct", "Nov", "Dec"] - case _: - raise ValueError("Invalid quarter value.") - - ''' - def case_aggregates_by_month(self): - """Return case aggregates by month.""" - df = self.datafile - section = transform_to_text(df.section) - - # from datafile quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' - month_list = transform_to_months(df.quarter) - # or we do upgrade get_schema_options to always take named params vs string text? - - # use or leverage `get_schema_options` to lookup model in RowSchema - models = [model for x in get_schema_options(program_type)[section]] - - - # using a django queryset, filter by datafile to find relevant search_index objects - - # count the number of objects in the queryset and assign to total - # using a queryset of parserError objects, filter by datafile and error_type to get count of rejected cases - # subtract rejected cases from total to get accepted cases - # return a dict of month: {accepted: x, rejected: y, total: z} - for month in month_list: - total = 0 - rejected = 0 - accepted = 0 - - for model in models: - total += model.objects.filter(datafile=df, month=month).count() # todo change to RPT_MONTH_YEAR - rejected += model.objects.filter(datafile=df, error.exists()).count() # todo filter doesn't actually work this way - #ParserError.objects.filter(datafile=df, month=month).count() #TODO filter where field_name != header or trailer ?? - accepted += total - rejected # again look for all objects where generic relation to error is false/empty - - case_aggregates_by_month[month] = { - - # filter by month - # filter by model - # filter by datafile - # count objects - # add to dict - ''' def get_status(self, errors): diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index b45175022..8603608b7 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -51,7 +51,7 @@ def parse_datafile(datafile): section_is_valid, section_error = validators.validate_header_section_matches_submission( datafile, - get_section_field(program_type, section) + util.get_section_reference(program_type, section) ) if not section_is_valid: @@ -111,7 +111,7 @@ def parse_datafile_lines(datafile, program_type, section): if line.startswith('HEADER') or line.startswith('TRAILER'): continue - schema = get_schema(line, section, program_type) + schema = util.get_schema(line, section, program_type) if schema is None: errors[line_number] = [util.generate_parser_error( datafile=datafile, @@ -174,131 +174,3 @@ def parse_datafile_line(line, schema, generate_error): return record_is_valid, record_errors -def get_schema_options(program, section, query=None, model=None, model_name=None): - ''' - Centralized function to return the appropriate schema for a given program, section, and query. - @param program: the abbreviated program type (.e.g, 'TAN') - @param section: the section of the file (.e.g, 'A') - @param query: the query for section_names (.e.g, 'section', 'models', etc.) - @return: the appropriate references (e.g., ACTIVE_CASE_DATA or {t1,t2,t3}) - ''' - - # ensure file section matches upload section - schema_options = { - 'TAN': { - 'A': { - 'section': DataFile.Section.ACTIVE_CASE_DATA, - 'models': { - 'T1': schema_defs.tanf.t1, - 'T2': schema_defs.tanf.t2, - 'T3': schema_defs.tanf.t3, - } - }, - 'C': { - 'section': DataFile.Section.CLOSED_CASE_DATA, - 'models': { - #'T4': schema_defs.tanf.t4, - #'T5': schema_defs.tanf.t5, - } - }, - 'G': { - 'section':DataFile.Section.AGGREGATE_DATA, - 'models': { - #'T6': schema_defs.tanf.t6, - } - }, - 'S': { - 'section':DataFile.Section.STRATUM_DATA, - 'models': { - #'T7': schema_defs.tanf.t7, - } - } - }, - 'SSP': { - 'A': { - 'section': DataFile.Section.SSP_ACTIVE_CASE_DATA, - 'models': { - 'S1': schema_defs.ssp.m1, - 'S2': schema_defs.ssp.m2, - 'S3': schema_defs.ssp.m3, - } - }, - 'C': { - 'section':DataFile.Section.SSP_CLOSED_CASE_DATA, - 'models': { - #'S4': schema_defs.ssp.m4, - #'S5': schema_defs.ssp.m5, - } - }, - 'G': { - 'section':DataFile.Section.SSP_AGGREGATE_DATA, - 'models': { - #'S6': schema_defs.ssp.m6, - } - }, - 'S': { - 'section':DataFile.Section.SSP_STRATUM_DATA, - 'models': { - #'S7': schema_defs.ssp.m7, - } - } - }, - # TODO: tribal tanf - } - - #TODO: add error handling for bad inputs -- how does `get` handle bad inputs? - if query == "text": - for prog_name, prog_dict in schema_options.items(): - for sect,val in prog_dict.items(): - if val==model: - return {'program_type':prog_name, 'section': sect} - raise ValueError("Model not found in schema_defs") - elif query == "section": - return schema_options.get(program, {}).get(section, None)[query] - elif query == "models": - links = schema_options.get(program, {}).get(section, None) - - # if query is not chosen or wrong input, return all options - # query = 'models', model = 'T1' - models = links.get(query, links) - - return models.get(model_name, models) - - -#TODO is it more flexible w/o option? we can do filtering in wrapper functions -# if option is empty, null, none, just return more - -''' -text -> section -text -> models{} YES -text -> model YES -datafile -> model - ^ section -> program -> model -datafile -> text -model -> text YES -section -> text - -text**: input string from the header/file -''' - -def get_program_models(str_prog, str_section): - return get_schema_options(program=str_prog, section=str_section, query='models') - -def get_program_model(str_prog, str_section, str_model): - return get_schema_options(program=str_prog, section=str_section, query='models', model_name=str_model) - -def get_section_reference(str_prog, str_section): - return get_schema_options(program=str_prog, section=str_section, query='section') - -def get_text_from_model(model): - get_schema_options() - - - - #TODO: if given a datafile (section), we can reverse back to the program b/c the section string has "tribal/ssp" in it, then process of elimination we have tanf - - -def get_schema(line, section, program_type): - """Return the appropriate schema for the line.""" - line_type = line[0:2] - return get_schema_options(program_type, section, query='models', model_name=line_type) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 65e8b2677..63dee5c4e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -9,7 +9,7 @@ from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 from .factories import DataFileSummaryFactory -from .. import schema_defs +from .. import schema_defs, util def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): """Create a test DataFile instance with the given file attached.""" @@ -605,6 +605,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel } +@pytest.mark.django_db def test_get_schema_options(): """Test use-cases for translating strings to named object references.""" @@ -623,22 +624,27 @@ def test_get_schema_options(): # from text: # get schema - schema = parse.get_schema('T3', 'A', 'TAN') + schema = util.get_schema('T3', 'A', 'TAN') assert schema == schema_defs.tanf.t3 - # get section + # get model - models = parse.get_program_models('TAN', 'A') + models = util.get_program_models('TAN', 'A') assert models == { 'T1': schema_defs.tanf.t1, 'T2': schema_defs.tanf.t2, 'T3': schema_defs.tanf.t3, } - model = parse.get_program_model('TAN', 'A', 'T1') + model = util.get_program_model('TAN', 'A', 'T1') assert model == schema_defs.tanf.t1 - - section = parse.get_section_reference('TAN','C') + # get section + section = util.get_section_reference('TAN','C') assert section == DataFile.Section.CLOSED_CASE_DATA + + dfs = DataFileSummaryFactory() + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + + assert False # from datafile: # get model(s) # get section str diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 2aa6f5422..5bed697cc 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -1,6 +1,8 @@ """Utility file for functions shared between all parsers even preparser.""" from .models import ParserError, ParserErrorCategoryChoices from django.contrib.contenttypes.models import ContentType +from . import schema_defs +from tdpservice.data_files.models import DataFile def value_is_empty(value, length): """Handle 'empty' values as field inputs.""" @@ -262,3 +264,218 @@ def parse_and_validate(self, line, generate_error): records.append(r) return records + + + +def get_schema_options(program, section, query=None, model=None, model_name=None): + ''' + Centralized function to return the appropriate schema for a given program, section, and query. + @param program: the abbreviated program type (.e.g, 'TAN') + @param section: the section of the file (.e.g, 'A');; or ACTIVE_CASE_DATA + @param query: the query for section_names (.e.g, 'section', 'models', etc.) + @return: the appropriate references (e.g., ACTIVE_CASE_DATA or {t1,t2,t3}) ;; returning 'A' + ''' + + # ensure file section matches upload section + schema_options = { + 'TAN': { + 'A': { + 'section': DataFile.Section.ACTIVE_CASE_DATA, + 'models': { + 'T1': schema_defs.tanf.t1, + 'T2': schema_defs.tanf.t2, + 'T3': schema_defs.tanf.t3, + } + }, + 'C': { + 'section': DataFile.Section.CLOSED_CASE_DATA, + 'models': { + #'T4': schema_defs.tanf.t4, + #'T5': schema_defs.tanf.t5, + } + }, + 'G': { + 'section':DataFile.Section.AGGREGATE_DATA, + 'models': { + #'T6': schema_defs.tanf.t6, + } + }, + 'S': { + 'section':DataFile.Section.STRATUM_DATA, + 'models': { + #'T7': schema_defs.tanf.t7, + } + } + }, + 'SSP': { + 'A': { + 'section': DataFile.Section.SSP_ACTIVE_CASE_DATA, + 'models': { + 'M1': schema_defs.ssp.m1, + 'M2': schema_defs.ssp.m2, + 'M3': schema_defs.ssp.m3, + } + }, + 'C': { + 'section':DataFile.Section.SSP_CLOSED_CASE_DATA, + 'models': { + #'S4': schema_defs.ssp.m4, + #'S5': schema_defs.ssp.m5, + } + }, + 'G': { + 'section':DataFile.Section.SSP_AGGREGATE_DATA, + 'models': { + #'S6': schema_defs.ssp.m6, + } + }, + 'S': { + 'section':DataFile.Section.SSP_STRATUM_DATA, + 'models': { + #'S7': schema_defs.ssp.m7, + } + } + }, + # TODO: tribal tanf + } + + #TODO: add error handling for bad inputs -- how does `get` handle bad inputs? + if query == "text": + for prog_name, prog_dict in schema_options.items(): + for sect,val in prog_dict.items(): + if val['section'] == section: + return {'program_type':prog_name, 'section': sect} + raise ValueError("Model not found in schema_defs") + elif query == "section": + return schema_options.get(program, {}).get(section, None)[query] + elif query == "models": + links = schema_options.get(program, {}).get(section, None) + + # if query is not chosen or wrong input, return all options + # query = 'models', model = 'T1' + models = links.get(query, links) + + if model_name is None: + return models + elif model_name not in models.keys(): + return None # intentionally trigger the error_msg for unknown record type + else: + return models.get(model_name, models) + + +#TODO is it more flexible w/o option? we can do filtering in wrapper functions +# if option is empty, null, none, just return more + +''' +text -> section YES +text -> models{} YES +text -> model YES +datafile -> model + ^ section -> program -> model +datafile -> text +model -> text YES +section -> text + +text**: input string from the header/file +''' + +def get_program_models(str_prog, str_section): + return get_schema_options(program=str_prog, section=str_section, query='models') + +def get_program_model(str_prog, str_section, str_model): + return get_schema_options(program=str_prog, section=str_section, query='models', model_name=str_model) + +def get_section_reference(str_prog, str_section): + return get_schema_options(program=str_prog, section=str_section, query='section') + +def get_text_from_model(model): + get_schema_options() + +def get_text_from_df(df): + return get_schema_options("", section=df.section, query='text') + +def get_prog_from_section(str_section): # this is pure, we could use get_schema_options but it's hard + # 'SSP Closed Case Data' + if str_section.startswith('SSP'): + return 'SSP' + elif str_section.startswith('Tribal'): + return 'TAN' # problematic, do we need to infer tribal entirely from tribe/fips code? should we make a new type? + else: + return 'TAN' + + #TODO: if given a datafile (section), we can reverse back to the program b/c the + # section string has "tribal/ssp" in it, then process of elimination we have tanf + +def get_schema(line, section, program_type): + """Return the appropriate schema for the line.""" + line_type = line[0:2] + return get_schema_options(program_type, section, query='models', model_name=line_type) + +def transform_to_months(quarter): + """Return a list of months in a quarter.""" + match quarter: + case "Q1": + return ["Jan", "Feb", "Mar"] + case "Q2": + return ["Apr", "May", "Jun"] + case "Q3": + return ["Jul", "Aug", "Sep"] + case "Q4": + return ["Oct", "Nov", "Dec"] + case _: + raise ValueError("Invalid quarter value.") + +def case_aggregates_by_month(df): + """Return case aggregates by month.""" + section = str(df.section) # section -> text + print("section: ", section) + program_type = get_prog_from_section(section) # section -> program_type -> text + print("program_type: ", program_type) + + # from datafile quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' + month_list = transform_to_months(df.quarter) + print("month_list: ", month_list) + # or we do upgrade get_schema_options to always take named params vs string text? + + short_section = get_text_from_df(df)['section'] + models = get_program_models(program_type, short_section) + print("models: ", models) + + #TODO: convert models from dict to list of only the references + + ''' + section: Active Case Data + program_type: TAN + month_list: ['Jan', 'Feb', 'Mar'] + models: {'T1': , + 'T2': , + 'T3': } + ''' + + + + # using a django queryset, filter by datafile to find relevant search_index objects + + # count the number of objects in the queryset and assign to total + # using a queryset of parserError objects, filter by datafile and error_type to get count of rejected cases + # subtract rejected cases from total to get accepted cases + # return a dict of month: {accepted: x, rejected: y, total: z} + '''for month in month_list: + total = 0 + rejected = 0 + accepted = 0 + + for model in models: + total += model.objects.filter(datafile=df, month=month).count() # todo change to RPT_MONTH_YEAR + rejected += model.objects.filter(datafile=df, error.exists()).count() # todo filter doesn't actually work this way + #ParserError.objects.filter(datafile=df, month=month).count() #TODO filter where field_name != header or trailer ?? + accepted += total - rejected # again look for all objects where generic relation to error is false/empty + + case_aggregates_by_month[month] = {} + + # filter by month + # filter by model + # filter by datafile + # count objects + # add to dict + ''' \ No newline at end of file From 9e4e4316ca65be53f1261c8f2c7c2f711e79244a Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Tue, 11 Jul 2023 20:13:56 -0600 Subject: [PATCH 067/120] - Updated queries - Added helper function - Need to merge in 2579 for queries to work --- tdrs-backend/tdpservice/parsers/util.py | 64 +++++++++++++++++++------ 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 5bed697cc..0fbec4529 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -338,7 +338,7 @@ def get_schema_options(program, section, query=None, model=None, model_name=None }, # TODO: tribal tanf } - + #TODO: add error handling for bad inputs -- how does `get` handle bad inputs? if query == "text": for prog_name, prog_dict in schema_options.items(): @@ -362,7 +362,7 @@ def get_schema_options(program, section, query=None, model=None, model_name=None else: return models.get(model_name, models) - + #TODO is it more flexible w/o option? we can do filtering in wrapper functions # if option is empty, null, none, just return more @@ -402,8 +402,8 @@ def get_prog_from_section(str_section): # this is pure, we could use get_schema return 'TAN' # problematic, do we need to infer tribal entirely from tribe/fips code? should we make a new type? else: return 'TAN' - - #TODO: if given a datafile (section), we can reverse back to the program b/c the + + #TODO: if given a datafile (section), we can reverse back to the program b/c the # section string has "tribal/ssp" in it, then process of elimination we have tanf def get_schema(line, section, program_type): @@ -425,6 +425,33 @@ def transform_to_months(quarter): case _: raise ValueError("Invalid quarter value.") +def month_to_int(month): + match month: + case "Jan": + return 1 + case "Feb": + return 2 + case "Mar": + return 3 + case "Apr": + return 4 + case "May": + return 5 + case "Jun": + return 6 + case "Jul": + return 7 + case "Aug": + return 8 + case "Sep": + return 9 + case "Oct": + return 10 + case "Nov": + return 11 + case "Dec": + return 12 + def case_aggregates_by_month(df): """Return case aggregates by month.""" section = str(df.section) # section -> text @@ -438,7 +465,8 @@ def case_aggregates_by_month(df): # or we do upgrade get_schema_options to always take named params vs string text? short_section = get_text_from_df(df)['section'] - models = get_program_models(program_type, short_section) + models_dict = get_program_models(program_type, short_section) + models = [model for model in models_dict.values()] print("models: ", models) #TODO: convert models from dict to list of only the references @@ -447,11 +475,11 @@ def case_aggregates_by_month(df): section: Active Case Data program_type: TAN month_list: ['Jan', 'Feb', 'Mar'] - models: {'T1': , - 'T2': , + models: {'T1': , + 'T2': , 'T3': } ''' - + # using a django queryset, filter by datafile to find relevant search_index objects @@ -460,22 +488,28 @@ def case_aggregates_by_month(df): # using a queryset of parserError objects, filter by datafile and error_type to get count of rejected cases # subtract rejected cases from total to get accepted cases # return a dict of month: {accepted: x, rejected: y, total: z} - '''for month in month_list: + aggregate_data = {} + for month in month_list: total = 0 rejected = 0 accepted = 0 for model in models: - total += model.objects.filter(datafile=df, month=month).count() # todo change to RPT_MONTH_YEAR - rejected += model.objects.filter(datafile=df, error.exists()).count() # todo filter doesn't actually work this way + # TODO: We need the TANF_T1 and other models to have the FK on datafile which hasnt been merged in yet + total += model.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).count() + print(total) + rejected += model.model.objects.filter(datafile=df, error__isnull=False).count() # todo filter doesn't actually work this way #ParserError.objects.filter(datafile=df, month=month).count() #TODO filter where field_name != header or trailer ?? accepted += total - rejected # again look for all objects where generic relation to error is false/empty - case_aggregates_by_month[month] = {} - + aggregate_data[month] = {"accepted": accepted, "rejected": rejected, "total": total} + + return aggregate_data + + + # filter by month # filter by model # filter by datafile # count objects - # add to dict - ''' \ No newline at end of file + # add to dict \ No newline at end of file From af905657e500c177efbfd9b6ae41a53dea544acd Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 12 Jul 2023 09:24:32 -0400 Subject: [PATCH 068/120] minor improvement to month2int --- tdrs-backend/tdpservice/parsers/util.py | 29 ++++--------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 0fbec4529..b92c15f11 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -3,6 +3,7 @@ from django.contrib.contenttypes.models import ContentType from . import schema_defs from tdpservice.data_files.models import DataFile +from datetime import datetime def value_is_empty(value, length): """Handle 'empty' values as field inputs.""" @@ -426,31 +427,9 @@ def transform_to_months(quarter): raise ValueError("Invalid quarter value.") def month_to_int(month): - match month: - case "Jan": - return 1 - case "Feb": - return 2 - case "Mar": - return 3 - case "Apr": - return 4 - case "May": - return 5 - case "Jun": - return 6 - case "Jul": - return 7 - case "Aug": - return 8 - case "Sep": - return 9 - case "Oct": - return 10 - case "Nov": - return 11 - case "Dec": - return 12 + """Return the integer value of a month.""" + return datetime.strptime(month, '%b').month + def case_aggregates_by_month(df): """Return case aggregates by month.""" From 4fc63ebf7103299ac542fd405b33a551cbaa8fa4 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 12 Jul 2023 08:48:05 -0600 Subject: [PATCH 069/120] - Fixing most merge errors --- tdrs-backend/tdpservice/parsers/parse.py | 47 +++++++++++++------ .../tdpservice/parsers/test/test_parse.py | 30 ++++++------ .../tdpservice/parsers/test/test_summary.py | 2 +- tdrs-backend/tdpservice/parsers/validators.py | 19 +------- 4 files changed, 50 insertions(+), 48 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 0d89f996b..c8fa862ae 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -50,8 +50,6 @@ def parse_datafile(datafile): section_is_valid, section_error = validators.validate_header_section_matches_submission( datafile, util.get_section_reference(program_type, section) - program_type, - section, ) if not section_is_valid: @@ -154,26 +152,47 @@ def parse_datafile_lines(datafile, program_type, section): return errors -def parse_multi_record_line(line, schema, generate_error, datafile): +def parse_multi_record_line(line, schema, generate_error): """Parse and validate a datafile line using MultiRecordRowSchema.""" - records = schema.parse_and_validate(line, generate_error) + if schema: + records = schema.parse_and_validate(line, generate_error) - for r in records: - record, record_is_valid, record_errors = r + for r in records: + record, record_is_valid, record_errors = r - if record: - record.datafile = datafile - record.save() + if record: + record.save() + + return records - return records + return [(None, False, [ + generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Record Type is missing from record.", + record=None, + field=None + ) + ])] def parse_datafile_line(line, schema, generate_error): """Parse and validate a datafile line and save any errors to the model.""" - record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) + if schema: + record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) - if record: - record.save() + if record: + record.save() - return record_is_valid, record_errors + return record_is_valid, record_errors + + return (False, [ + generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Record Type is missing from record.", + record=None, + field=None + ) + ]) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index c04911464..a4f6ea0a0 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -2,7 +2,6 @@ import pytest -from ..util import create_test_datafile from .. import parse from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary from tdpservice.data_files.models import DataFile @@ -10,13 +9,12 @@ from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 from .factories import DataFileSummaryFactory from .. import schema_defs, util -from pathlib import Path @pytest.fixture def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" - return create_test_datafile('small_correct_file', stt_user, stt) + return util.create_test_datafile('small_correct_file', stt_user, stt) @pytest.fixture def dfs(): @@ -100,7 +98,7 @@ def test_parse_wrong_program_type(test_datafile, dfs): @pytest.fixture def test_big_file(stt_user, stt): """Fixture for ADS.E2J.FTP1.TS06.""" - return create_test_datafile('ADS.E2J.FTP1.TS06', stt_user, stt) + return util.create_test_datafile('ADS.E2J.FTP1.TS06', stt_user, stt) @pytest.mark.django_db @@ -132,7 +130,7 @@ def test_parse_big_file(test_big_file, dfs): @pytest.fixture def bad_test_file(stt_user, stt): """Fixture for bad_TANF_S2.""" - return create_test_datafile('bad_TANF_S2.txt', stt_user, stt) + return util.create_test_datafile('bad_TANF_S2.txt', stt_user, stt) @pytest.mark.django_db @@ -160,7 +158,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): @pytest.fixture def bad_file_missing_header(stt_user, stt): """Fixture for bad_missing_header.""" - return create_test_datafile('bad_missing_header.txt', stt_user, stt) + return util.create_test_datafile('bad_missing_header.txt', stt_user, stt) @pytest.mark.django_db @@ -186,7 +184,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): @pytest.fixture def bad_file_multiple_headers(stt_user, stt): """Fixture for bad_two_headers.""" - return create_test_datafile('bad_two_headers.txt', stt_user, stt) + return util.create_test_datafile('bad_two_headers.txt', stt_user, stt) @pytest.mark.django_db @@ -213,7 +211,7 @@ def test_parse_bad_file_multiple_headers(bad_file_multiple_headers, dfs): @pytest.fixture def big_bad_test_file(stt_user, stt): """Fixture for bad_TANF_S1.""" - return create_test_datafile('bad_TANF_S1.txt', stt_user, stt) + return util.create_test_datafile('bad_TANF_S1.txt', stt_user, stt) @pytest.mark.django_db @@ -239,7 +237,7 @@ def test_parse_big_bad_test_file(big_bad_test_file): @pytest.fixture def bad_trailer_file(stt_user, stt): """Fixture for bad_trailer_1.""" - return create_test_datafile('bad_trailer_1.txt', stt_user, stt) + return util.create_test_datafile('bad_trailer_1.txt', stt_user, stt) @pytest.mark.django_db @@ -271,7 +269,7 @@ def test_parse_bad_trailer_file(bad_trailer_file): @pytest.fixture def bad_trailer_file_2(stt_user, stt): """Fixture for bad_trailer_2.""" - return create_test_datafile('bad_trailer_2.txt', stt_user, stt) + return util.create_test_datafile('bad_trailer_2.txt', stt_user, stt) @pytest.mark.django_db @@ -321,7 +319,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2): @pytest.fixture def empty_file(stt_user, stt): """Fixture for empty_file.""" - return create_test_datafile('empty_file', stt_user, stt) + return util.create_test_datafile('empty_file', stt_user, stt) @pytest.mark.django_db @@ -347,7 +345,7 @@ def test_parse_empty_file(empty_file): @pytest.fixture def small_ssp_section1_datafile(stt_user, stt): """Fixture for small_ssp_section1.""" - return create_test_datafile('small_ssp_section1.txt', stt_user, stt, 'SSP Active Case Data') + return util.create_test_datafile('small_ssp_section1.txt', stt_user, stt, 'SSP Active Case Data') @pytest.mark.django_db @@ -380,7 +378,7 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile): @pytest.fixture def ssp_section1_datafile(stt_user, stt): """Fixture for ssp_section1_datafile.""" - return create_test_datafile('ssp_section1_datafile.txt', stt_user, stt, 'SSP Active Case Data') + return util.create_test_datafile('ssp_section1_datafile.txt', stt_user, stt, 'SSP Active Case Data') # @pytest.mark.django_db @@ -434,7 +432,7 @@ def ssp_section1_datafile(stt_user, stt): @pytest.fixture def small_tanf_section1_datafile(stt_user, stt): """Fixture for small_tanf_section1.""" - return create_test_datafile('small_tanf_section1.txt', stt_user, stt) + return util.create_test_datafile('small_tanf_section1.txt', stt_user, stt) @pytest.mark.django_db def test_parse_tanf_section1_datafile(small_tanf_section1_datafile): @@ -495,7 +493,7 @@ def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile): @pytest.fixture def bad_tanf_s1__row_missing_required_field(stt_user, stt): """Fixture for small_tanf_section1.""" - return create_test_datafile('small_bad_tanf_s1', stt_user, stt) + return util.create_test_datafile('small_bad_tanf_s1', stt_user, stt) @pytest.mark.django_db @@ -541,7 +539,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field @pytest.fixture def bad_ssp_s1__row_missing_required_field(stt_user, stt): """Fixture for ssp_section1_datafile.""" - return create_test_datafile('small_bad_ssp_s1', stt_user, stt, 'SSP Active Case Data') + return util.create_test_datafile('small_bad_ssp_s1', stt_user, stt, 'SSP Active Case Data') @pytest.mark.django_db diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index edf28c4b5..db267d95d 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -4,7 +4,7 @@ from tdpservice.parsers import parse from tdpservice.parsers.models import DataFileSummary, ParserErrorCategoryChoices from .factories import DataFileSummaryFactory, ParserErrorFactory -from .test_parse import create_test_datafile +from ..util import create_test_datafile import logging logger = logging.getLogger(__name__) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index c3b58b8d1..2625f3d44 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -117,24 +117,9 @@ def validate_single_header_trailer(datafile): return is_valid, error -def validate_header_section_matches_submission(datafile, program_type, section): +def validate_header_section_matches_submission(datafile, section): """Validate header section matches submission section.""" - section_names = { - 'TAN': { - 'A': DataFile.Section.ACTIVE_CASE_DATA, - 'C': DataFile.Section.CLOSED_CASE_DATA, - 'G': DataFile.Section.AGGREGATE_DATA, - 'S': DataFile.Section.STRATUM_DATA, - }, - 'SSP': { - 'A': DataFile.Section.SSP_ACTIVE_CASE_DATA, - 'C': DataFile.Section.SSP_CLOSED_CASE_DATA, - 'G': DataFile.Section.SSP_AGGREGATE_DATA, - 'S': DataFile.Section.SSP_STRATUM_DATA, - }, - } - - is_valid = datafile.section == section_names.get(program_type, {}).get(section) + is_valid = datafile.section == section error = None if not is_valid: From cc39ddd9dff11ab6219cd50c70a802723c3e76da Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 12 Jul 2023 09:08:43 -0600 Subject: [PATCH 070/120] - Fixing functions --- tdrs-backend/tdpservice/parsers/parse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index c8fa862ae..ecf860dd5 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -152,7 +152,7 @@ def parse_datafile_lines(datafile, program_type, section): return errors -def parse_multi_record_line(line, schema, generate_error): +def parse_multi_record_line(line, schema, generate_error, datafile): """Parse and validate a datafile line using MultiRecordRowSchema.""" if schema: records = schema.parse_and_validate(line, generate_error) @@ -161,6 +161,7 @@ def parse_multi_record_line(line, schema, generate_error): record, record_is_valid, record_errors = r if record: + record.datafile = datafile record.save() return records @@ -176,12 +177,13 @@ def parse_multi_record_line(line, schema, generate_error): ])] -def parse_datafile_line(line, schema, generate_error): +def parse_datafile_line(line, schema, generate_error, datafile): """Parse and validate a datafile line and save any errors to the model.""" if schema: record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) if record: + record.datafile = datafile record.save() return record_is_valid, record_errors From a3239c5297a2fe06e0f5b25b45e35b6f1bf19c7d Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 12 Jul 2023 09:50:39 -0600 Subject: [PATCH 071/120] - Updated queries based on generic relation --- tdrs-backend/tdpservice/parsers/util.py | 26 +++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 78f96affc..86a0329b0 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -464,9 +464,9 @@ def case_aggregates_by_month(df): # or we do upgrade get_schema_options to always take named params vs string text? short_section = get_text_from_df(df)['section'] - models_dict = get_program_models(program_type, short_section) - models = [model for model in models_dict.values()] - print("models: ", models) + schema_models_dict = get_program_models(program_type, short_section) + schema_models = [model for model in schema_models_dict.values()] + print("models: ", schema_models) #TODO: convert models from dict to list of only the references @@ -493,15 +493,21 @@ def case_aggregates_by_month(df): rejected = 0 accepted = 0 - for model in models: - # TODO: We need the TANF_T1 and other models to have the FK on datafile which hasnt been merged in yet - total += model.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).count() - print(total) - rejected += model.model.objects.filter(datafile=df, error__isnull=False).count() # todo filter doesn't actually work this way - #ParserError.objects.filter(datafile=df, month=month).count() #TODO filter where field_name != header or trailer ?? - accepted += total - rejected # again look for all objects where generic relation to error is false/empty + for schema_model in schema_models: + if isinstance(schema_model, MultiRecordRowSchema): + for sm in schema_model.schemas: + total += sm.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).count() + ids = sm.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).values_list('pk', flat=True) + rejected += ParserError.objects.filter(content_type=ContentType.objects.get_for_model(sm.model), object_id__in=ids).count() + accepted += total - rejected + else: + total += schema_model.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).count() + ids = schema_model.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).values_list('pk', flat=True) + rejected += ParserError.objects.filter(content_type=ContentType.objects.get_for_model(schema_model.model), object_id__in=ids).count() + accepted += total - rejected aggregate_data[month] = {"accepted": accepted, "rejected": rejected, "total": total} + print(aggregate_data) return aggregate_data From 29f52e6a596c4660af7bab8d8356d00a81fdd502 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 12 Jul 2023 13:22:59 -0600 Subject: [PATCH 072/120] - Updated queries to count by case number instead of record number --- .../tdpservice/parsers/test/test_parse.py | 89 ++++++++++++++++++- tdrs-backend/tdpservice/parsers/util.py | 33 +++---- 2 files changed, 103 insertions(+), 19 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index a4f6ea0a0..6d05f7fbb 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -9,6 +9,10 @@ from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 from .factories import DataFileSummaryFactory from .. import schema_defs, util +import logging + +es_logger = logging.getLogger('elasticsearch') +es_logger.setLevel(logging.WARNING) @pytest.fixture @@ -24,8 +28,18 @@ def dfs(): @pytest.mark.django_db def test_parse_small_correct_file(test_datafile, dfs): """Test parsing of small_correct_file.""" + dfs.datafile = test_datafile + dfs.save() + errors = parse.parse_datafile(test_datafile) + print(f"OBJ Date: {TANF_T1.objects.all().first().RPT_MONTH_YEAR}") + + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + assert dfs.case_aggregates == {'Oct': {'accepted': 1, 'rejected': 0, 'total': 1}, + 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, + 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} + assert errors == {} assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED assert TANF_T1.objects.count() == 1 @@ -55,6 +69,12 @@ def test_parse_section_mismatch(test_datafile, dfs): dfs.save() errors = parse.parse_datafile(test_datafile) + + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + print(dfs.case_aggregates) + # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that + assert dfs.case_aggregates == False #{'Jan': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Feb': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Mar': {'accepted': 0, 'rejected': 0, 'total': 0}} + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -80,6 +100,11 @@ def test_parse_wrong_program_type(test_datafile, dfs): errors = parse.parse_datafile(test_datafile) assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + print(dfs.case_aggregates) + # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that + assert dfs.case_aggregates == False #{'Jan': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Feb': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Mar': {'accepted': 0, 'rejected': 0, 'total': 0}} + parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -108,7 +133,16 @@ def test_parse_big_file(test_big_file, dfs): expected_t2_record_count = 882 expected_t3_record_count = 1376 + dfs.datafile = test_big_file + dfs.save() + errors = parse.parse_datafile(test_big_file) + + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + assert dfs.case_aggregates == {'Oct': {'accepted': 270, 'rejected': 0, 'total': 270}, + 'Nov': {'accepted': 273, 'rejected': 0, 'total': 273}, + 'Dec': {'accepted': 272, 'rejected': 0, 'total': 272}} + assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS parser_errors = ParserError.objects.filter(file=test_big_file) assert parser_errors.count() == 355 @@ -136,9 +170,20 @@ def bad_test_file(stt_user, stt): @pytest.mark.django_db def test_parse_bad_test_file(bad_test_file, dfs): """Test parsing of bad_TANF_S2.""" + bad_test_file.year = 2021 + bad_test_file.save() + + dfs.datafile = bad_test_file + dfs.save() + errors = parse.parse_datafile(bad_test_file) + print(f"{TANF_T1.objects.all().first().RPT_MONTH_YEAR}") assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + print(dfs.case_aggregates) + # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that + assert dfs.case_aggregates == False parser_errors = ParserError.objects.filter(file=bad_test_file) assert parser_errors.count() == 1 @@ -164,8 +209,17 @@ def bad_file_missing_header(stt_user, stt): @pytest.mark.django_db def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): """Test parsing of bad_missing_header.""" + dfs.datafile = bad_file_missing_header + dfs.save() + errors = parse.parse_datafile(bad_file_missing_header) + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + print(dfs.case_aggregates) + # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that + assert dfs.case_aggregates == False + parser_errors = ParserError.objects.filter(file=bad_file_missing_header) assert parser_errors.count() == 1 @@ -190,8 +244,16 @@ def bad_file_multiple_headers(stt_user, stt): @pytest.mark.django_db def test_parse_bad_file_multiple_headers(bad_file_multiple_headers, dfs): """Test parsing of bad_two_headers.""" + dfs.datafile = bad_file_multiple_headers + dfs.save() + errors = parse.parse_datafile(bad_file_multiple_headers) + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + print(dfs.case_aggregates) + # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that + assert dfs.case_aggregates == False parser_errors = ParserError.objects.filter(file=bad_file_multiple_headers) assert parser_errors.count() == 1 @@ -241,8 +303,11 @@ def bad_trailer_file(stt_user, stt): @pytest.mark.django_db -def test_parse_bad_trailer_file(bad_trailer_file): +def test_parse_bad_trailer_file(bad_trailer_file, dfs): """Test parsing bad_trailer_1.""" + dfs.datafile = bad_trailer_file + dfs.save() + errors = parse.parse_datafile(bad_trailer_file) parser_errors = ParserError.objects.filter(file=bad_trailer_file) @@ -349,14 +414,23 @@ def small_ssp_section1_datafile(stt_user, stt): @pytest.mark.django_db -def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile): +def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): """Test parsing small_ssp_section1_datafile.""" expected_m1_record_count = 5 expected_m2_record_count = 6 expected_m3_record_count = 8 + dfs.datafile = small_ssp_section1_datafile + dfs.save() + errors = parse.parse_datafile(small_ssp_section1_datafile) + assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + print(dfs.case_aggregates) + # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that + assert dfs.case_aggregates == False + parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) assert parser_errors.count() == 1 @@ -435,10 +509,19 @@ def small_tanf_section1_datafile(stt_user, stt): return util.create_test_datafile('small_tanf_section1.txt', stt_user, stt) @pytest.mark.django_db -def test_parse_tanf_section1_datafile(small_tanf_section1_datafile): +def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): """Test parsing of small_tanf_section1_datafile and validate T2 model data.""" + + dfs.datafile = small_tanf_section1_datafile + dfs.save() + errors = parse.parse_datafile(small_tanf_section1_datafile) + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + dfs.case_aggregates = {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, + 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, + 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} + assert errors == {} assert TANF_T2.objects.count() == 5 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 86a0329b0..ad790f449 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -6,14 +6,15 @@ from datetime import datetime from tdpservice.data_files.models import DataFile from pathlib import Path +from itertools import chain def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): """Create a test DataFile instance with the given file attached.""" path = str(Path(__file__).parent.joinpath('test/data')) + f'/{filename}' datafile = DataFile.create_new_version({ - 'quarter': '4', - 'year': 2022, + 'quarter': 'Q4', + 'year': 2020, 'section': section, 'user': stt_user, 'stt': stt @@ -454,19 +455,15 @@ def month_to_int(month): def case_aggregates_by_month(df): """Return case aggregates by month.""" section = str(df.section) # section -> text - print("section: ", section) program_type = get_prog_from_section(section) # section -> program_type -> text - print("program_type: ", program_type) # from datafile quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' month_list = transform_to_months(df.quarter) - print("month_list: ", month_list) # or we do upgrade get_schema_options to always take named params vs string text? short_section = get_text_from_df(df)['section'] schema_models_dict = get_program_models(program_type, short_section) schema_models = [model for model in schema_models_dict.values()] - print("models: ", schema_models) #TODO: convert models from dict to list of only the references @@ -492,21 +489,25 @@ def case_aggregates_by_month(df): total = 0 rejected = 0 accepted = 0 + month_int = month_to_int(month) + rpt_month_year = int(f"{df.year}{month_int}") + qset = set() for schema_model in schema_models: if isinstance(schema_model, MultiRecordRowSchema): - for sm in schema_model.schemas: - total += sm.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).count() - ids = sm.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).values_list('pk', flat=True) - rejected += ParserError.objects.filter(content_type=ContentType.objects.get_for_model(sm.model), object_id__in=ids).count() - accepted += total - rejected - else: - total += schema_model.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).count() - ids = schema_model.model.objects.filter(datafile=df, RPT_MONTH_YEAR=month_to_int(month)).values_list('pk', flat=True) - rejected += ParserError.objects.filter(content_type=ContentType.objects.get_for_model(schema_model.model), object_id__in=ids).count() - accepted += total - rejected + schema_model = schema_model.schemas[0] + + next = set(schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year).distinct("CASE_NUMBER").values_list("CASE_NUMBER", flat=True)) + qset = qset.union(next) + + total += len(qset) + rejected += ParserError.objects.filter(content_type=ContentType.objects.get_for_model(schema_model.model), + case_number__in=qset).count() + accepted = total - rejected aggregate_data[month] = {"accepted": accepted, "rejected": rejected, "total": total} + + print(f"Num Errors: {ParserError.objects.filter(file=df).count()}") print(aggregate_data) return aggregate_data From 89278dbb1e55cb9d86c54ed50a662cfb98af47a7 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 12 Jul 2023 14:10:26 -0600 Subject: [PATCH 073/120] - Added route - Updated task to create dfs --- tdrs-backend/tdpservice/parsers/admin.py | 5 +++++ tdrs-backend/tdpservice/parsers/models.py | 10 +++++----- tdrs-backend/tdpservice/parsers/serializers.py | 12 +++++++++++- tdrs-backend/tdpservice/parsers/urls.py | 7 ++++--- tdrs-backend/tdpservice/parsers/views.py | 12 ++++++++++-- tdrs-backend/tdpservice/scheduling/parser_task.py | 9 +++++++++ 6 files changed, 44 insertions(+), 11 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/admin.py b/tdrs-backend/tdpservice/parsers/admin.py index c98ef5d70..be672b3c8 100644 --- a/tdrs-backend/tdpservice/parsers/admin.py +++ b/tdrs-backend/tdpservice/parsers/admin.py @@ -14,5 +14,10 @@ class ParserErrorAdmin(admin.ModelAdmin): 'error_message', ] +class DataFileSummaryAdmin(admin.ModelAdmin): + """ModelAdmin class for DataFileSummary objects generated in parsing.""" + + list_display = ['status', 'case_aggregates', 'datafile'] admin.site.register(models.ParserError, ParserErrorAdmin) +admin.site.register(models.DataFileSummary, DataFileSummaryAdmin) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index e8e50b589..69a6fcfb4 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -119,20 +119,20 @@ class Status(models.TextChoices): """ - def get_status(self, errors): + def get_status(errors): """Set and return the status field based on errors and models associated with datafile.""" if errors is None: - return self.status # aka PENDING + return DataFileSummary.Status.PENDING if type(errors) != dict: raise TypeError("errors parameter must be a dictionary.") if errors == {}: - return self.Status.ACCEPTED + return DataFileSummary.Status.ACCEPTED elif DataFileSummary.find_precheck(errors): - return self.Status.REJECTED + return DataFileSummary.Status.REJECTED else: - return self.Status.ACCEPTED_WITH_ERRORS + return DataFileSummary.Status.ACCEPTED_WITH_ERRORS def find_precheck(errors): """Check for pre-parsing errors. diff --git a/tdrs-backend/tdpservice/parsers/serializers.py b/tdrs-backend/tdpservice/parsers/serializers.py index 05a4e0d07..9b4ad734d 100644 --- a/tdrs-backend/tdpservice/parsers/serializers.py +++ b/tdrs-backend/tdpservice/parsers/serializers.py @@ -1,7 +1,7 @@ """Serializers for parsing errors.""" from rest_framework import serializers -from .models import ParserError +from .models import ParserError, DataFileSummary class ParsingErrorSerializer(serializers.ModelSerializer): @@ -23,3 +23,13 @@ class Meta: model = ParserError fields = '__all__' + + +class DataFileSummarySerializer(serializers.ModelSerializer): + """Serializer for Parsing Errors.""" + + class Meta: + """Metadata.""" + + model = DataFileSummary + fields = ['status', 'case_aggregates', 'datafile'] diff --git a/tdrs-backend/tdpservice/parsers/urls.py b/tdrs-backend/tdpservice/parsers/urls.py index f2226e0ab..625b84d04 100644 --- a/tdrs-backend/tdpservice/parsers/urls.py +++ b/tdrs-backend/tdpservice/parsers/urls.py @@ -1,12 +1,13 @@ """Routing for DataFiles.""" from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import ParsingErrorViewSet +from .views import ParsingErrorViewSet, DataFileSummaryViewSet router = DefaultRouter() -router.register("", ParsingErrorViewSet) +router.register("parsing_errors/", ParsingErrorViewSet) +router.register("dfs/", DataFileSummaryViewSet) urlpatterns = [ - path('parsing_errors/', include(router.urls)), + path('', include(router.urls)), ] diff --git a/tdrs-backend/tdpservice/parsers/views.py b/tdrs-backend/tdpservice/parsers/views.py index d39965ee3..8e40b79e4 100644 --- a/tdrs-backend/tdpservice/parsers/views.py +++ b/tdrs-backend/tdpservice/parsers/views.py @@ -2,8 +2,8 @@ from tdpservice.users.permissions import IsApprovedPermission from rest_framework.viewsets import ModelViewSet from rest_framework.response import Response -from .serializers import ParsingErrorSerializer -from .models import ParserError +from .serializers import ParsingErrorSerializer, DataFileSummarySerializer +from .models import ParserError, DataFileSummary import logging import base64 from io import BytesIO @@ -69,3 +69,11 @@ def _get_xls_serialized_file(self, data): col += 1 workbook.close() return {"data": data, "xls_report": base64.b64encode(output.getvalue()).decode("utf-8")} + + +class DataFileSummaryViewSet(ModelViewSet): + """DataFileSummary file views.""" + + queryset = DataFileSummary.objects.all() + serializer_class = DataFileSummarySerializer + permission_classes = [IsApprovedPermission] diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index 649ae0723..95e7a711e 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -4,6 +4,9 @@ import logging from tdpservice.data_files.models import DataFile from tdpservice.parsers.parse import parse_datafile +from tdpservice.parsers.models import DataFileSummary +from tdpservice.parsers.util import case_aggregates_by_month + logger = logging.getLogger(__name__) @@ -18,4 +21,10 @@ def parse(data_file_id): logger.info(f"DataFile parsing started for file {data_file.filename}") errors = parse_datafile(data_file) + print(errors) + dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.get_status(errors=errors)) + print(f"\n\n{dfs}\n\n") + dfs.case_aggregates = case_aggregates_by_month(data_file) + dfs.save() + print(f"\n\n{dfs.case_aggregates}\n\n") logger.info(f"DataFile parsing finished with {len(errors)} errors: {errors}") From e078f50c35eee27268365c8d56633f9df47d2f2b Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Wed, 12 Jul 2023 15:31:47 -0600 Subject: [PATCH 074/120] - updated tests to include dfs --- tdrs-backend/tdpservice/parsers/models.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 79 ++++++------------- 2 files changed, 25 insertions(+), 56 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 69a6fcfb4..064993b2d 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -152,6 +152,6 @@ def find_precheck(errors): if key == 'trailer': continue for parserError in errors[key]: - if parserError.error_type == ParserErrorCategoryChoices.PRE_CHECK: + if type(parserError) is ParserError and parserError.error_type == ParserErrorCategoryChoices.PRE_CHECK: return True return False diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 6d05f7fbb..e3a373bd5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -41,7 +41,7 @@ def test_parse_small_correct_file(test_datafile, dfs): 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} assert errors == {} - assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.ACCEPTED assert TANF_T1.objects.count() == 1 assert ParserError.objects.filter(file=test_datafile).count() == 0 @@ -65,17 +65,9 @@ def test_parse_section_mismatch(test_datafile, dfs): test_datafile.section = 'Closed Case Data' test_datafile.save() - dfs.datafile = test_datafile - dfs.save() - errors = parse.parse_datafile(test_datafile) - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - print(dfs.case_aggregates) - # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that - assert dfs.case_aggregates == False #{'Jan': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Feb': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Mar': {'accepted': 0, 'rejected': 0, 'total': 0}} - - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -98,12 +90,7 @@ def test_parse_wrong_program_type(test_datafile, dfs): test_datafile.save() errors = parse.parse_datafile(test_datafile) - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED - - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - print(dfs.case_aggregates) - # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that - assert dfs.case_aggregates == False #{'Jan': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Feb': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Mar': {'accepted': 0, 'rejected': 0, 'total': 0}} + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -138,12 +125,12 @@ def test_parse_big_file(test_big_file, dfs): errors = parse.parse_datafile(test_big_file) + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) assert dfs.case_aggregates == {'Oct': {'accepted': 270, 'rejected': 0, 'total': 270}, 'Nov': {'accepted': 273, 'rejected': 0, 'total': 273}, 'Dec': {'accepted': 272, 'rejected': 0, 'total': 272}} - assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS parser_errors = ParserError.objects.filter(file=test_big_file) assert parser_errors.count() == 355 assert len(errors) == 334 @@ -170,20 +157,7 @@ def bad_test_file(stt_user, stt): @pytest.mark.django_db def test_parse_bad_test_file(bad_test_file, dfs): """Test parsing of bad_TANF_S2.""" - bad_test_file.year = 2021 - bad_test_file.save() - - dfs.datafile = bad_test_file - dfs.save() - errors = parse.parse_datafile(bad_test_file) - print(f"{TANF_T1.objects.all().first().RPT_MONTH_YEAR}") - - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - print(dfs.case_aggregates) - # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that - assert dfs.case_aggregates == False parser_errors = ParserError.objects.filter(file=bad_test_file) assert parser_errors.count() == 1 @@ -209,16 +183,9 @@ def bad_file_missing_header(stt_user, stt): @pytest.mark.django_db def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): """Test parsing of bad_missing_header.""" - dfs.datafile = bad_file_missing_header - dfs.save() - errors = parse.parse_datafile(bad_file_missing_header) - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - print(dfs.case_aggregates) - # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that - assert dfs.case_aggregates == False + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_missing_header) assert parser_errors.count() == 1 @@ -244,16 +211,9 @@ def bad_file_multiple_headers(stt_user, stt): @pytest.mark.django_db def test_parse_bad_file_multiple_headers(bad_file_multiple_headers, dfs): """Test parsing of bad_two_headers.""" - dfs.datafile = bad_file_multiple_headers - dfs.save() - errors = parse.parse_datafile(bad_file_multiple_headers) - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - print(dfs.case_aggregates) - # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that - assert dfs.case_aggregates == False + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_multiple_headers) assert parser_errors.count() == 1 @@ -277,7 +237,7 @@ def big_bad_test_file(stt_user, stt): @pytest.mark.django_db -def test_parse_big_bad_test_file(big_bad_test_file): +def test_parse_big_bad_test_file(big_bad_test_file, dfs): """Test parsing of bad_TANF_S1.""" errors = parse.parse_datafile(big_bad_test_file) @@ -420,16 +380,19 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): expected_m2_record_count = 6 expected_m3_record_count = 8 + small_ssp_section1_datafile.year = 2018 + small_ssp_section1_datafile.save() + dfs.datafile = small_ssp_section1_datafile dfs.save() errors = parse.parse_datafile(small_ssp_section1_datafile) - assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - print(dfs.case_aggregates) - # I think in the case where no records are generated because of the type of error, we need a check in the util function to handle that - assert dfs.case_aggregates == False + assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, + 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, + 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) assert parser_errors.count() == 1 @@ -517,10 +480,11 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): errors = parse.parse_datafile(small_tanf_section1_datafile) + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.ACCEPTED dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - dfs.case_aggregates = {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, - 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, - 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} + assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, + 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, + 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} assert errors == {} assert TANF_T2.objects.count() == 5 @@ -580,10 +544,15 @@ def bad_tanf_s1__row_missing_required_field(stt_user, stt): @pytest.mark.django_db -def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field): +def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field, dfs): """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" + dfs.datafile = bad_tanf_s1__row_missing_required_field + dfs.save() + errors = parse.parse_datafile(bad_tanf_s1__row_missing_required_field) + assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED + parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 From 0fbaf77753b1544f51673fe78fe8bd341e2347b9 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 13 Jul 2023 17:02:54 -0400 Subject: [PATCH 075/120] Cleaning up most comments that are no longer necessary and fixed lint issues. --- tdrs-backend/tdpservice/parsers/admin.py | 2 + tdrs-backend/tdpservice/parsers/models.py | 31 ------ tdrs-backend/tdpservice/parsers/parse.py | 34 +------ .../tdpservice/parsers/test/test_parse.py | 5 +- .../tdpservice/parsers/test/test_summary.py | 2 +- tdrs-backend/tdpservice/parsers/util.py | 95 +++++++------------ 6 files changed, 37 insertions(+), 132 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/admin.py b/tdrs-backend/tdpservice/parsers/admin.py index be672b3c8..266fb5b26 100644 --- a/tdrs-backend/tdpservice/parsers/admin.py +++ b/tdrs-backend/tdpservice/parsers/admin.py @@ -14,10 +14,12 @@ class ParserErrorAdmin(admin.ModelAdmin): 'error_message', ] + class DataFileSummaryAdmin(admin.ModelAdmin): """ModelAdmin class for DataFileSummary objects generated in parsing.""" list_display = ['status', 'case_aggregates', 'datafile'] + admin.site.register(models.ParserError, ParserErrorAdmin) admin.site.register(models.DataFileSummary, DataFileSummaryAdmin) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 0adeb96d3..c4dd5aa3c 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -86,38 +86,7 @@ class Status(models.TextChoices): datafile = models.ForeignKey(DataFile, on_delete=models.CASCADE) - # TODO: worth adding more fields here or ??? - #named program schema - #named models schema via rowschema - #named section via datafile - - # eventually needs a breakdown of cases (accepted, rejected, total) per month - # elif qtr2 jan-mar - # elif qtr3 apr-jun - # elif qtr4 jul-sept - case_aggregates = models.JSONField(null=True, blank=False) - """ - # Do these queries only once, save result during creation of this model - # or do we grab this data during parsing and bubble it up during create call? - { - "Jan": { - "accepted": 100, - "rejected": 10, - "total": 110 - }, - "Feb": { - "accepted": 100, - "rejected": 10, - "total": 110 - }, - "Mar": { - "accepted": 100, - "rejected": 10, - "total": 110 - } - """ - def get_status(errors): """Set and return the status field based on errors and models associated with datafile.""" diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index f224da2b6..454189fff 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -60,42 +60,11 @@ def parse_datafile(datafile): errors = errors | line_errors - # errors['summary'] = DataFileSummary.objects.create( - # datafile=datafile, - # status=DataFileSummary.get_status(errors) - # ) - - # or perhaps just invert this? - # what does it look like having the errors dict as a field of the summary? - # summary.errors = errors --- but I don't want/need to store this in DB - # divesting that storage and just using my FK to datafile so I can run querysets later - # perserves the ability to use the summary object to generate the errors dict - - # perhaps just formalize the entire errors struct? - # pros: - # - can be used to generate error report - # - can be used to generate summary - # - can be used to generate error count - # - can be used to generate error count by type - # - can be used to generate error count by record type - # - can be used to generate error count by field - # - can be used to generate error count by field type - # - has a consistent structure between differing file types - # - has testable functions for each of the above - # - has agreed-upon inputs/outputs - # cons: - # - requires boilerplate to generate - # - different structures may be needed for different purposes - # - built-in dict may be easier to reference ala Cameron - # - built-in dict is freer-form and complete already - return errors def parse_datafile_lines(datafile, program_type, section): - """Parse lines with appropriate schema and return errors.""" - #dfs = DataFileSummary.object.create(datafile=datafile) - # and then what, pass in program_type to case_aggregates after loop? + """Parse and validate all lines in a datafile.""" errors = {} rawfile = datafile.file @@ -188,7 +157,6 @@ def parse_datafile_line(line, schema, generate_error, datafile): return record_is_valid, record_errors - return (False, [ generate_error( schema=None, diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index d583a2d4c..bbc655232 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -474,7 +474,6 @@ def small_tanf_section1_datafile(stt_user, stt): @pytest.mark.django_db def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): """Test parsing of small_tanf_section1_datafile and validate T2 model data.""" - dfs.datafile = small_tanf_section1_datafile dfs.save() @@ -646,7 +645,6 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel @pytest.mark.django_db def test_get_schema_options(): """Test use-cases for translating strings to named object references.""" - ''' text -> section text -> models{} YES @@ -676,7 +674,7 @@ def test_get_schema_options(): model = util.get_program_model('TAN', 'A', 'T1') assert model == schema_defs.tanf.t1 # get section - section = util.get_section_reference('TAN','C') + section = util.get_section_reference('TAN', 'C') assert section == DataFile.Section.CLOSED_CASE_DATA dfs = DataFileSummaryFactory() @@ -691,4 +689,3 @@ def test_get_schema_options(): # get text # get section str # get ref section - diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index b4d96c746..896dd8dba 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -64,4 +64,4 @@ def test_dfs_set_case_aggregates(test_datafile, dfs): dfs.case_aggregates = dfs.get_case_aggregates(error_ast) assert dfs.case_aggregates['Jan']['accepted'] == 1 assert dfs.case_aggregates['Jan']['rejected'] == 0 - assert dfs.case_aggregates['Jan']['total'] == 1 \ No newline at end of file + assert dfs.case_aggregates['Jan']['total'] == 1 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index ad790f449..2f1c1603f 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -4,9 +4,7 @@ from . import schema_defs from tdpservice.data_files.models import DataFile from datetime import datetime -from tdpservice.data_files.models import DataFile from pathlib import Path -from itertools import chain def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): @@ -287,18 +285,16 @@ def parse_and_validate(self, line, generate_error): return records +def get_schema_options(program, section, query=None, model=None, model_name=None): + """Centralized function to return the appropriate schema for a given program, section, and query. + TODO: need to rework this docstring as it is outdated hence the weird ';;' for some of them. -def get_schema_options(program, section, query=None, model=None, model_name=None): - ''' - Centralized function to return the appropriate schema for a given program, section, and query. @param program: the abbreviated program type (.e.g, 'TAN') @param section: the section of the file (.e.g, 'A');; or ACTIVE_CASE_DATA @param query: the query for section_names (.e.g, 'section', 'models', etc.) @return: the appropriate references (e.g., ACTIVE_CASE_DATA or {t1,t2,t3}) ;; returning 'A' - ''' - - # ensure file section matches upload section + """ schema_options = { 'TAN': { 'A': { @@ -312,20 +308,20 @@ def get_schema_options(program, section, query=None, model=None, model_name=None 'C': { 'section': DataFile.Section.CLOSED_CASE_DATA, 'models': { - #'T4': schema_defs.tanf.t4, - #'T5': schema_defs.tanf.t5, + # 'T4': schema_defs.tanf.t4, + # 'T5': schema_defs.tanf.t5, } }, 'G': { - 'section':DataFile.Section.AGGREGATE_DATA, + 'section': DataFile.Section.AGGREGATE_DATA, 'models': { - #'T6': schema_defs.tanf.t6, + # 'T6': schema_defs.tanf.t6, } }, 'S': { - 'section':DataFile.Section.STRATUM_DATA, + 'section': DataFile.Section.STRATUM_DATA, 'models': { - #'T7': schema_defs.tanf.t7, + # 'T7': schema_defs.tanf.t7, } } }, @@ -339,34 +335,33 @@ def get_schema_options(program, section, query=None, model=None, model_name=None } }, 'C': { - 'section':DataFile.Section.SSP_CLOSED_CASE_DATA, + 'section': DataFile.Section.SSP_CLOSED_CASE_DATA, 'models': { - #'S4': schema_defs.ssp.m4, - #'S5': schema_defs.ssp.m5, + # 'S4': schema_defs.ssp.m4, + # 'S5': schema_defs.ssp.m5, } }, 'G': { - 'section':DataFile.Section.SSP_AGGREGATE_DATA, + 'section': DataFile.Section.SSP_AGGREGATE_DATA, 'models': { - #'S6': schema_defs.ssp.m6, + # 'S6': schema_defs.ssp.m6, } }, 'S': { - 'section':DataFile.Section.SSP_STRATUM_DATA, + 'section': DataFile.Section.SSP_STRATUM_DATA, 'models': { - #'S7': schema_defs.ssp.m7, + # 'S7': schema_defs.ssp.m7, } } }, # TODO: tribal tanf } - #TODO: add error handling for bad inputs -- how does `get` handle bad inputs? if query == "text": for prog_name, prog_dict in schema_options.items(): - for sect,val in prog_dict.items(): + for sect, val in prog_dict.items(): if val['section'] == section: - return {'program_type':prog_name, 'section': sect} + return {'program_type': prog_name, 'section': sect} raise ValueError("Model not found in schema_defs") elif query == "section": return schema_options.get(program, {}).get(section, None)[query] @@ -385,9 +380,6 @@ def get_schema_options(program, section, query=None, model=None, model_name=None return models.get(model_name, models) -#TODO is it more flexible w/o option? we can do filtering in wrapper functions -# if option is empty, null, none, just return more - ''' text -> section YES text -> models{} YES @@ -402,30 +394,32 @@ def get_schema_options(program, section, query=None, model=None, model_name=None ''' def get_program_models(str_prog, str_section): + """Return the models dict for a given program and section.""" return get_schema_options(program=str_prog, section=str_section, query='models') def get_program_model(str_prog, str_section, str_model): + """Return singular model for a given program, section, and name.""" return get_schema_options(program=str_prog, section=str_section, query='models', model_name=str_model) def get_section_reference(str_prog, str_section): + """Return the named section reference for a given program and section.""" return get_schema_options(program=str_prog, section=str_section, query='section') -def get_text_from_model(model): - get_schema_options() - def get_text_from_df(df): + """Return the short-hand text for program, section for a given datafile.""" return get_schema_options("", section=df.section, query='text') -def get_prog_from_section(str_section): # this is pure, we could use get_schema_options but it's hard - # 'SSP Closed Case Data' +def get_prog_from_section(str_section): + """Return the program type for a given section.""" + # e.g., 'SSP Closed Case Data' if str_section.startswith('SSP'): return 'SSP' elif str_section.startswith('Tribal'): - return 'TAN' # problematic, do we need to infer tribal entirely from tribe/fips code? should we make a new type? + return 'TAN' # problematic, do we need to infer tribal entirely from tribe/fips code? else: return 'TAN' - #TODO: if given a datafile (section), we can reverse back to the program b/c the + # TODO: if given a datafile (section), we can reverse back to the program b/c the # section string has "tribal/ssp" in it, then process of elimination we have tanf def get_schema(line, section, program_type): @@ -459,31 +453,13 @@ def case_aggregates_by_month(df): # from datafile quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' month_list = transform_to_months(df.quarter) - # or we do upgrade get_schema_options to always take named params vs string text? short_section = get_text_from_df(df)['section'] schema_models_dict = get_program_models(program_type, short_section) schema_models = [model for model in schema_models_dict.values()] - #TODO: convert models from dict to list of only the references - - ''' - section: Active Case Data - program_type: TAN - month_list: ['Jan', 'Feb', 'Mar'] - models: {'T1': , - 'T2': , - 'T3': } - ''' + # TODO: convert models from dict to list of only the references - - - # using a django queryset, filter by datafile to find relevant search_index objects - - # count the number of objects in the queryset and assign to total - # using a queryset of parserError objects, filter by datafile and error_type to get count of rejected cases - # subtract rejected cases from total to get accepted cases - # return a dict of month: {accepted: x, rejected: y, total: z} aggregate_data = {} for month in month_list: total = 0 @@ -497,12 +473,13 @@ def case_aggregates_by_month(df): if isinstance(schema_model, MultiRecordRowSchema): schema_model = schema_model.schemas[0] - next = set(schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year).distinct("CASE_NUMBER").values_list("CASE_NUMBER", flat=True)) + next = set(schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year) + .distinct("CASE_NUMBER").values_list("CASE_NUMBER", flat=True)) qset = qset.union(next) total += len(qset) rejected += ParserError.objects.filter(content_type=ContentType.objects.get_for_model(schema_model.model), - case_number__in=qset).count() + case_number__in=qset).count() accepted = total - rejected aggregate_data[month] = {"accepted": accepted, "rejected": rejected, "total": total} @@ -511,11 +488,3 @@ def case_aggregates_by_month(df): print(aggregate_data) return aggregate_data - - - - # filter by month - # filter by model - # filter by datafile - # count objects - # add to dict \ No newline at end of file From f7c67725f5ba269c6072966480666ee429054680 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 14 Jul 2023 11:51:36 -0400 Subject: [PATCH 076/120] making minor updates, still broken tests. --- .../tdpservice/parsers/test/test_parse.py | 1 - tdrs-backend/tdpservice/parsers/util.py | 2 -- tdrs-backend/tdpservice/parsers/validators.py | 19 ++----------------- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index bbc655232..c2665e3f5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -680,7 +680,6 @@ def test_get_schema_options(): dfs = DataFileSummaryFactory() dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - assert False # from datafile: # get model(s) # get section str diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 2f1c1603f..067c8a91b 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -458,8 +458,6 @@ def case_aggregates_by_month(df): schema_models_dict = get_program_models(program_type, short_section) schema_models = [model for model in schema_models_dict.values()] - # TODO: convert models from dict to list of only the references - aggregate_data = {} for month in month_list: total = 0 diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index c3b58b8d1..2625f3d44 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -117,24 +117,9 @@ def validate_single_header_trailer(datafile): return is_valid, error -def validate_header_section_matches_submission(datafile, program_type, section): +def validate_header_section_matches_submission(datafile, section): """Validate header section matches submission section.""" - section_names = { - 'TAN': { - 'A': DataFile.Section.ACTIVE_CASE_DATA, - 'C': DataFile.Section.CLOSED_CASE_DATA, - 'G': DataFile.Section.AGGREGATE_DATA, - 'S': DataFile.Section.STRATUM_DATA, - }, - 'SSP': { - 'A': DataFile.Section.SSP_ACTIVE_CASE_DATA, - 'C': DataFile.Section.SSP_CLOSED_CASE_DATA, - 'G': DataFile.Section.SSP_AGGREGATE_DATA, - 'S': DataFile.Section.SSP_STRATUM_DATA, - }, - } - - is_valid = datafile.section == section_names.get(program_type, {}).get(section) + is_valid = datafile.section == section error = None if not is_valid: From 46a6d73df1974b085c4a47caa4a35915499db682 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 14 Jul 2023 12:13:44 -0400 Subject: [PATCH 077/120] updating pipfile.lock and rebuild image resolved test issues --- tdrs-backend/Pipfile.lock | 46 ++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index 214c1e06b..8cf6c23b4 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -198,11 +198,11 @@ }, "click": { "hashes": [ - "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3", - "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37" + "sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367", + "sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548" ], "markers": "python_version >= '3.7'", - "version": "==8.1.4" + "version": "==8.1.5" }, "click-didyoumean": { "hashes": [ @@ -687,6 +687,7 @@ "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5", "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530", "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d", + "sha256:1ce91b6ec08d866b14413d3f0bbdea7e24dfdc8e59f562bb77bc3fe60b6144ca", "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891", "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992", "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7", @@ -716,6 +717,7 @@ "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485", "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf", "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43", + "sha256:bc2ec7c7b5d66b8ec9ce9f720dbb5fa4bace0f545acd34870eff4a369b44bf37", "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2", "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd", "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86", @@ -744,11 +746,11 @@ }, "prometheus-client": { "hashes": [ - "sha256:9c3b26f1535945e85b8934fb374678d263137b78ef85f305b1156c7c881cd11b", - "sha256:a77b708cf083f4d1a3fb3ce5c95b4afa32b9c521ae363354a4a910204ea095ce" + "sha256:21e674f39831ae3f8acde238afd9a27a37d0d2fb5a28ea094f0ce25d2cbf2091", + "sha256:e537f37160f6807b8202a6fc4764cdd19bac5480ddd3e0d463c3002b34462101" ], "markers": "python_version >= '3.6'", - "version": "==0.17.0" + "version": "==0.17.1" }, "prompt-toolkit": { "hashes": [ @@ -886,10 +888,10 @@ }, "python-crontab": { "hashes": [ - "sha256:9c374d1c9d401afdd8dd958f20077f74c158ab3fffb9604296802715e887fe48", - "sha256:b21af4647c7bbb848fef2f020616c6b0289dcb9f94b4f991a55310ff9bec5749" + "sha256:6d5ba3c190ec76e4d252989a1644fcb233dbf53fbc8fceeb9febe1657b9fb1d4", + "sha256:79fb7465039ddfd4fb93d072d6ee0d45c1ac8bf1597f0686ea14fd4361dba379" ], - "version": "==2.7.1" + "version": "==3.0.0" }, "python-dateutil": { "hashes": [ @@ -901,11 +903,11 @@ }, "pytz": { "hashes": [ - "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", - "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", + "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" ], "index": "pypi", - "version": "==2022.1" + "version": "==2023.3" }, "redis": { "hashes": [ @@ -1203,11 +1205,11 @@ "develop": { "awscli": { "hashes": [ - "sha256:0d8670fe3e0c126ab86f01fcef3b7082be88b6dbb1ed2cd1697f28440a289879", - "sha256:7dd8647c53caf496bca5036fa1bdef99dc0307db30019be10f6f34a8d3ed7587" + "sha256:0f288172aa1f4e5e47419bee6ba7f88b94ae95190ff876a8f80c813cedb1201f", + "sha256:9b5067fa31f74c17fc96ccfa2a37dc73ec5f9764ffa948da8907ebcd52d94b2c" ], "index": "pypi", - "version": "==1.27.5" + "version": "==1.27.165" }, "awscli-local": { "hashes": [ @@ -1234,11 +1236,11 @@ }, "click": { "hashes": [ - "sha256:2739815aaa5d2c986a88f1e9230c55e17f0caad3d958a5e13ad0797c166db9e3", - "sha256:b97d0c74955da062a7d4ef92fadb583806a585b2ea81958a81bd72726cbb8e37" + "sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367", + "sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548" ], "markers": "python_version >= '3.7'", - "version": "==8.1.4" + "version": "==8.1.5" }, "colorama": { "hashes": [ @@ -1343,11 +1345,11 @@ }, "faker": { "hashes": [ - "sha256:801d1a2d71f1fc54d332de2ab19de7452454309937233ea2f7485402882d67b3", - "sha256:84bcf92bb725dd7341336eea4685df9a364f16f2470c4d29c1d7e6c5fd5a457d" + "sha256:4b7d5cd0c898f0b64f88fbf0a35aac66762f2273446ba4a4e459985a2e5c8f8c", + "sha256:d1eb772faf4a7c458c90b19d3626c40ae3460bd665ad7f5fb7b089e31d1a6dcf" ], - "markers": "python_version >= '3.7'", - "version": "==18.13.0" + "markers": "python_version >= '3.8'", + "version": "==19.1.0" }, "flake8": { "hashes": [ From f162423858099fc19aeb5ff24814a28636aac9c0 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 14 Jul 2023 15:40:56 -0400 Subject: [PATCH 078/120] Reorganizing tests, still failing in test_parse.py --- tdrs-backend/tdpservice/parsers/models.py | 2 +- .../tdpservice/parsers/test/test_models.py | 36 +++++++++++++++++-- .../tdpservice/parsers/test/test_parse.py | 24 +++++++++---- .../parsers/test/test_serializer.py | 1 - tdrs-backend/tdpservice/parsers/util.py | 3 -- .../tdpservice/scheduling/parser_task.py | 5 +-- 6 files changed, 53 insertions(+), 18 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index c4dd5aa3c..5a98ad9f0 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -88,7 +88,7 @@ class Status(models.TextChoices): case_aggregates = models.JSONField(null=True, blank=False) - def get_status(errors): + def get_status(self, errors): """Set and return the status field based on errors and models associated with datafile.""" if errors is None: return DataFileSummary.Status.PENDING diff --git a/tdrs-backend/tdpservice/parsers/test/test_models.py b/tdrs-backend/tdpservice/parsers/test/test_models.py index c46532ada..e5ef3838a 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_models.py +++ b/tdrs-backend/tdpservice/parsers/test/test_models.py @@ -1,8 +1,8 @@ """Module testing for data file model.""" import pytest -from tdpservice.parsers.models import ParserError -from tdpservice.parsers.test.factories import ParserErrorFactory +from tdpservice.parsers.models import ParserError, DataFileSummary, ParserErrorCategoryChoices +from .factories import DataFileSummaryFactory, ParserErrorFactory @pytest.fixture def parser_error_instance(): @@ -21,3 +21,35 @@ def test_parser_error_rpt_month_name(parser_error_instance): """Test that the parser error instance is created.""" parser_error_instance.rpt_month_year = 202001 assert parser_error_instance.rpt_month_name == "January" + + +@pytest.fixture() +def dfs(): + """Fixture for DataFileSummary.""" + return DataFileSummaryFactory.create() + + +@pytest.mark.django_db +def test_dfs_model(dfs): + """Test that the model is created and populated correctly.""" + assert dfs.case_aggregates['Jan']['accepted'] == 100 + + +@pytest.mark.django_db +def test_dfs_get_status(dfs): + """Test that the initial status is set correctly.""" + assert dfs.status == DataFileSummary.Status.PENDING + + # create empty Error dict to prompt accepted status + assert dfs.get_status(errors={}) == DataFileSummary.Status.ACCEPTED + + # create category 2 ParserError list to prompt accepted with errors status + parser_errors = [] + for i in range(2, 4): + parser_errors.append(ParserErrorFactory(row_number=i, error_type=str(i))) + + assert dfs.get_status(errors={'document': parser_errors}) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + + # create category 1 ParserError list to prompt rejected status + parser_errors.append(ParserErrorFactory(row_number=5, error_type=ParserErrorCategoryChoices.PRE_CHECK)) + assert dfs.get_status(errors={'document': parser_errors}) == DataFileSummary.Status.REJECTED diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index c2665e3f5..69331f127 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -23,7 +23,7 @@ def test_datafile(stt_user, stt): @pytest.fixture def dfs(): """Fixture for DataFileSummary.""" - return DataFileSummaryFactory() + return DataFileSummaryFactory.create() @pytest.mark.django_db def test_parse_small_correct_file(test_datafile, dfs): @@ -33,8 +33,6 @@ def test_parse_small_correct_file(test_datafile, dfs): errors = parse.parse_datafile(test_datafile) - print(f"OBJ Date: {TANF_T1.objects.all().first().RPT_MONTH_YEAR}") - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) assert dfs.case_aggregates == {'Oct': {'accepted': 1, 'rejected': 0, 'total': 1}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, @@ -554,8 +552,8 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 - for e in parser_errors: - print(e.error_type, e.error_message) + # for e in parser_errors: + # print(e.error_type, e.error_message) row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE @@ -641,9 +639,21 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel 'trailer': [trailer_error], } +@pytest.mark.django_db +def test_dfs_set_case_aggregates(test_datafile, dfs): + """Test that the case aggregates are set correctly.""" + test_datafile.section = 'Active Case Data' + test_datafile.save() + error_ast = parse.parse_datafile(test_datafile) + dfs.case_aggregates = util.case_aggregates_by_month(test_datafile) + dfs.save() + + assert dfs.case_aggregates['Oct']['accepted'] == 1 + assert dfs.case_aggregates['Oct']['rejected'] == 0 + assert dfs.case_aggregates['Oct']['total'] == 1 @pytest.mark.django_db -def test_get_schema_options(): +def test_get_schema_options(dfs): """Test use-cases for translating strings to named object references.""" ''' text -> section @@ -677,7 +687,6 @@ def test_get_schema_options(): section = util.get_section_reference('TAN', 'C') assert section == DataFile.Section.CLOSED_CASE_DATA - dfs = DataFileSummaryFactory() dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) # from datafile: @@ -688,3 +697,4 @@ def test_get_schema_options(): # get text # get section str # get ref section + diff --git a/tdrs-backend/tdpservice/parsers/test/test_serializer.py b/tdrs-backend/tdpservice/parsers/test/test_serializer.py index ce90afb48..7ef5b756e 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_serializer.py +++ b/tdrs-backend/tdpservice/parsers/test/test_serializer.py @@ -57,5 +57,4 @@ def test_serializer_with_no_context(parser_error_instance): parser_error_instance, data={}, partial=True) - print(e.value) assert str(e.value) == "'context'" diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 067c8a91b..7b78a4748 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -482,7 +482,4 @@ def case_aggregates_by_month(df): aggregate_data[month] = {"accepted": accepted, "rejected": rejected, "total": total} - print(f"Num Errors: {ParserError.objects.filter(file=df).count()}") - print(aggregate_data) - return aggregate_data diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index 95e7a711e..6f1a1aede 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -21,10 +21,7 @@ def parse(data_file_id): logger.info(f"DataFile parsing started for file {data_file.filename}") errors = parse_datafile(data_file) - print(errors) dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.get_status(errors=errors)) - print(f"\n\n{dfs}\n\n") dfs.case_aggregates = case_aggregates_by_month(data_file) dfs.save() - print(f"\n\n{dfs.case_aggregates}\n\n") - logger.info(f"DataFile parsing finished with {len(errors)} errors: {errors}") + logger.info(f"DataFile parsing finished with status {dfs.status} and {len(errors)} errors: {errors}") From cb01bd393df5d4414cd83e82fe6105d1eba1bffb Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 14 Jul 2023 15:41:24 -0400 Subject: [PATCH 079/120] deleted summary file, split into other test scripts. --- .../tdpservice/parsers/test/test_summary.py | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 tdrs-backend/tdpservice/parsers/test/test_summary.py diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py deleted file mode 100644 index 896dd8dba..000000000 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Test the new model for DataFileSummary.""" - -import pytest -from tdpservice.parsers import parse -from tdpservice.parsers.models import DataFileSummary, ParserErrorCategoryChoices -from .factories import DataFileSummaryFactory, ParserErrorFactory -from ..util import create_test_datafile - -import logging -logger = logging.getLogger(__name__) - -@pytest.fixture -def test_datafile(stt_user, stt): - """Fixture for small_correct_file.""" - return create_test_datafile('small_correct_file', stt_user, stt) - -@pytest.fixture -def dfs(): - """Fixture for DataFileSummary.""" - return DataFileSummaryFactory() - -@pytest.mark.django_db -def test_dfs_model(dfs): - """Test that the model is created and populated correctly.""" - assert dfs.case_aggregates['Jan']['accepted'] == 100 - -@pytest.mark.django_db -def test_dfs_rejected(test_datafile, dfs): - """Ensure that an invalid file generates a rejected status.""" - test_datafile.section = 'Closed Case Data' - test_datafile.save() - - error_ast = parse.parse_datafile(test_datafile) - dfs.status = dfs.get_status(error_ast) - assert DataFileSummary.Status.REJECTED == dfs.status - -@pytest.mark.django_db -def test_dfs_set_status(dfs): - """Test that the status is set correctly.""" - assert dfs.status == DataFileSummary.Status.PENDING - dfs.status = dfs.get_status(errors={}) - assert dfs.status == DataFileSummary.Status.ACCEPTED - # create category 1 ParserError list to prompt rejected status - parser_errors = [] - - for i in range(2, 4): - parser_errors.append(ParserErrorFactory(row_number=i, error_type=str(i))) - - dfs.status = dfs.get_status(errors={'document': parser_errors}) - - assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - - parser_errors.append(ParserErrorFactory(row_number=5, error_type=ParserErrorCategoryChoices.PRE_CHECK)) - dfs.status = dfs.get_status(errors={'document': parser_errors}) - - assert dfs.status == DataFileSummary.Status.REJECTED - -@pytest.mark.django_db -def test_dfs_set_case_aggregates(test_datafile, dfs): - """Test that the case aggregates are set correctly.""" - test_datafile.section = 'Active Case Data' - test_datafile.save() - error_ast = parse.parse_datafile(test_datafile) - dfs.case_aggregates = dfs.get_case_aggregates(error_ast) - assert dfs.case_aggregates['Jan']['accepted'] == 1 - assert dfs.case_aggregates['Jan']['rejected'] == 0 - assert dfs.case_aggregates['Jan']['total'] == 1 From a194462ba957d96d36a2d66670b96929cc62fd4d Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 14 Jul 2023 15:48:36 -0400 Subject: [PATCH 080/120] Fixed missing self reference. --- .../tdpservice/parsers/test/test_parse.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 69331f127..0d38258c1 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -39,7 +39,7 @@ def test_parse_small_correct_file(test_datafile, dfs): 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} assert errors == {} - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.ACCEPTED + assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED assert TANF_T1.objects.count() == 1 assert ParserError.objects.filter(file=test_datafile).count() == 0 @@ -63,9 +63,12 @@ def test_parse_section_mismatch(test_datafile, dfs): test_datafile.section = 'Closed Case Data' test_datafile.save() + dfs.datafile = test_datafile + dfs.save() + errors = parse.parse_datafile(test_datafile) - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -88,7 +91,7 @@ def test_parse_wrong_program_type(test_datafile, dfs): test_datafile.save() errors = parse.parse_datafile(test_datafile) - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -123,7 +126,7 @@ def test_parse_big_file(test_big_file, dfs): errors = parse.parse_datafile(test_big_file) - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) assert dfs.case_aggregates == {'Oct': {'accepted': 270, 'rejected': 0, 'total': 270}, 'Nov': {'accepted': 273, 'rejected': 0, 'total': 273}, @@ -183,7 +186,7 @@ def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): """Test parsing of bad_missing_header.""" errors = parse.parse_datafile(bad_file_missing_header) - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_missing_header) assert parser_errors.count() == 1 @@ -211,7 +214,7 @@ def test_parse_bad_file_multiple_headers(bad_file_multiple_headers, dfs): """Test parsing of bad_two_headers.""" errors = parse.parse_datafile(bad_file_multiple_headers) - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_multiple_headers) assert parser_errors.count() == 1 @@ -386,7 +389,7 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): errors = parse.parse_datafile(small_ssp_section1_datafile) - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, @@ -477,7 +480,7 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): errors = parse.parse_datafile(small_tanf_section1_datafile) - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.ACCEPTED + assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, @@ -548,7 +551,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field errors = parse.parse_datafile(bad_tanf_s1__row_missing_required_field) - assert DataFileSummary.get_status(errors) == DataFileSummary.Status.REJECTED + assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 From c3bfa54f93893adc91a545ed076ac8417d3b4d3b Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 17 Jul 2023 09:16:46 -0400 Subject: [PATCH 081/120] Linting fixes. --- tdrs-backend/tdpservice/parsers/test/test_models.py | 2 +- tdrs-backend/tdpservice/parsers/test/test_parse.py | 3 +-- tdrs-backend/tdpservice/parsers/validators.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_models.py b/tdrs-backend/tdpservice/parsers/test/test_models.py index e5ef3838a..0b607777c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_models.py +++ b/tdrs-backend/tdpservice/parsers/test/test_models.py @@ -39,7 +39,7 @@ def test_dfs_model(dfs): def test_dfs_get_status(dfs): """Test that the initial status is set correctly.""" assert dfs.status == DataFileSummary.Status.PENDING - + # create empty Error dict to prompt accepted status assert dfs.get_status(errors={}) == DataFileSummary.Status.ACCEPTED diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 0d38258c1..ad74833be 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -647,7 +647,7 @@ def test_dfs_set_case_aggregates(test_datafile, dfs): """Test that the case aggregates are set correctly.""" test_datafile.section = 'Active Case Data' test_datafile.save() - error_ast = parse.parse_datafile(test_datafile) + parse.parse_datafile(test_datafile) # this still needs to execute to create db objects to be queried dfs.case_aggregates = util.case_aggregates_by_month(test_datafile) dfs.save() @@ -700,4 +700,3 @@ def test_get_schema_options(dfs): # get text # get section str # get ref section - diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 2625f3d44..0fe3c4c5e 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -2,7 +2,6 @@ from .util import generate_parser_error from .models import ParserErrorCategoryChoices -from tdpservice.data_files.models import DataFile # higher order validator func From 534e635c58ea426a25f11ac1d0a7ed93e2b65870 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 18 Jul 2023 16:23:29 -0400 Subject: [PATCH 082/120] Found reference failure in deployed env. --- tdrs-backend/tdpservice/scheduling/parser_task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index 6f1a1aede..b197ed252 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -21,7 +21,8 @@ def parse(data_file_id): logger.info(f"DataFile parsing started for file {data_file.filename}") errors = parse_datafile(data_file) - dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.get_status(errors=errors)) + dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING) + dfs.status = dfs.get_status(errors=errors) dfs.case_aggregates = case_aggregates_by_month(data_file) dfs.save() logger.info(f"DataFile parsing finished with status {dfs.status} and {len(errors)} errors: {errors}") From 5e79786bb815f199f696d40ed813b8789c7796ab Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 18 Jul 2023 16:35:05 -0400 Subject: [PATCH 083/120] Removing extra returns for missing record type. --- tdrs-backend/tdpservice/parsers/parse.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index fba6d4d96..4267aa900 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -163,16 +163,6 @@ def parse_multi_record_line(line, schema, generate_error): return records - return [(None, False, [ - generate_error( - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="Record Type is missing from record.", - record=None, - field=None - ) - ])] - def parse_datafile_line(line, schema, generate_error): """Parse and validate a datafile line and save any errors to the model.""" @@ -184,16 +174,6 @@ def parse_datafile_line(line, schema, generate_error): return record_is_valid, record_errors - return (False, [ - generate_error( - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="Record Type is missing from record.", - record=None, - field=None - ) - ]) - def get_schema_options(program_type): """Return the allowed schema options.""" From 1eb7e9afde4af6bc48e854bcb3e254c30e0ec6de Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 19 Jul 2023 12:01:00 -0400 Subject: [PATCH 084/120] lint fix --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 8fd289444..b2b69fbce 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -4,9 +4,7 @@ import pytest from ..util import create_test_datafile from .. import parse - from ..models import ParserError, ParserErrorCategoryChoices, DataFileSummary -from tdpservice.data_files.models import DataFile from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 from .factories import DataFileSummaryFactory From a6d528ec7fdf552e84a5f26cd949ae91e30f30a7 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 19 Jul 2023 17:13:56 -0400 Subject: [PATCH 085/120] Addressed invocation of datafile for failing test --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 69cebcfec..0a228155b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -7,6 +7,7 @@ from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3 from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M2, SSP_M3 from .factories import DataFileSummaryFactory +from tdpservice.data_files.models import DataFile from .. import schema_defs, util import logging From ef2793e4b4ff0e0cb713709ecec4acd1b728e7c5 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 19 Jul 2023 18:58:18 -0400 Subject: [PATCH 086/120] lint update for whitespace --- tdrs-backend/tdpservice/parsers/parse.py | 1 - .../tdpservice/search_indexes/test/test_model_mapping.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index c8b8db55f..1f22cebe6 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -136,7 +136,6 @@ def parse_multi_record_line(line, schema, generate_error, datafile): return records - def parse_datafile_line(line, schema, generate_error, datafile): """Parse and validate a datafile line and save any errors to the model.""" if schema: diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index 1870b4bc5..7286536eb 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -373,7 +373,6 @@ def test_can_create_and_index_tanf_t7_submission(test_datafile): submission.save() - # No checks her because t7 records can't be parsed currently. # assert submission.id is not None From 924367252c5a2c07be8ada23dad0b637041deaeb Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 20 Jul 2023 15:05:08 -0400 Subject: [PATCH 087/120] Intermediary commit, broken test --- tdrs-backend/docker-compose.yml | 2 +- tdrs-backend/tdpservice/parsers/test/test_parse.py | 12 +++++++++++- tdrs-backend/tdpservice/parsers/util.py | 8 +++++++- tdrs-backend/tdpservice/scheduling/parser_task.py | 3 ++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 47de77cde..d6748e940 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -118,5 +118,5 @@ volumes: networks: default: - external: name: external-net + external: true \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 0a228155b..ac014f41c 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -349,10 +349,20 @@ def empty_file(stt_user, stt): @pytest.mark.django_db -def test_parse_empty_file(empty_file): +def test_parse_empty_file(empty_file, dfs): """Test parsing of empty_file.""" + dfs.datafile = empty_file + dfs.save() errors = parse.parse_datafile(empty_file) + dfs.status = dfs.get_status(errors) + dfs.case_aggregates = dfs.case_aggregates_by_month(empty_file) + + assert dfs.status == DataFileSummary.Status.REJECTED + assert dfs.case_aggregates[0] == "N/A" + print(dfs.case_aggregates[0]) + assert False + parser_errors = ParserError.objects.filter(file=empty_file) assert parser_errors.count() == 1 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 12b18c0e5..36cff0899 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -445,7 +445,7 @@ def month_to_int(month): return datetime.strptime(month, '%b').month -def case_aggregates_by_month(df): +def case_aggregates_by_month(df, dfs_status): """Return case aggregates by month.""" section = str(df.section) # section -> text program_type = get_prog_from_section(section) # section -> program_type -> text @@ -465,6 +465,12 @@ def case_aggregates_by_month(df): month_int = month_to_int(month) rpt_month_year = int(f"{df.year}{month_int}") + if dfs_status == "Rejected": + # we need to be careful here on examples of bad headers or empty files, since no month will be found + # but we can rely on the frontend submitted year-quarter to still generate the list of months + aggregate_data[month] = {"accepted": "N/A", "rejected": "N/A", "total": "N/A"} + continue + qset = set() for schema_model in schema_models: if isinstance(schema_model, MultiRecordRowSchema): diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index b197ed252..2af0f1493 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -20,8 +20,9 @@ def parse(data_file_id): data_file = DataFile.objects.get(id=data_file_id) logger.info(f"DataFile parsing started for file {data_file.filename}") - errors = parse_datafile(data_file) + dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING) + errors = parse_datafile(data_file) dfs.status = dfs.get_status(errors=errors) dfs.case_aggregates = case_aggregates_by_month(data_file) dfs.save() From bb158e04b8fd845fda1ca9c3b0e3a99f511d3415 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 20 Jul 2023 15:05:52 -0400 Subject: [PATCH 088/120] new assignemnts in util --- tdrs-backend/tdpservice/parsers/util.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 36cff0899..fc4d7170a 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -426,6 +426,17 @@ def get_schema(line, section, program_type): line_type = line[0:2] return get_schema_options(program_type, section, query='models', model_name=line_type) +def fiscal_to_calendar(year, fiscal_quarter): + """Decrement the input quarter text by one.""" + array = [1,2,3,4] # wrapping around an array + int_qtr = int(fiscal_quarter[1:]) # remove the 'Q', e.g., 'Q1' -> '1' + if int_qtr == 1: + year = year - 1 + + ind_qtr = array.index(int_qtr) # get the index so we can easily wrap-around end of array + return year, "Q{}".format(array[ind_qtr - 1]) # return the previous quarter + + def transform_to_months(quarter): """Return a list of months in a quarter.""" match quarter: @@ -450,8 +461,9 @@ def case_aggregates_by_month(df, dfs_status): section = str(df.section) # section -> text program_type = get_prog_from_section(section) # section -> program_type -> text - # from datafile quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' - month_list = transform_to_months(df.quarter) + # from datafile year/quarter, generate short month names for each month in quarter ala 'Jan', 'Feb', 'Mar' + calendar_year, calendar_qtr = fiscal_to_calendar(df.year, df.quarter) + month_list = transform_to_months(calendar_qtr) short_section = get_text_from_df(df)['section'] schema_models_dict = get_program_models(program_type, short_section) @@ -463,7 +475,7 @@ def case_aggregates_by_month(df, dfs_status): rejected = 0 accepted = 0 month_int = month_to_int(month) - rpt_month_year = int(f"{df.year}{month_int}") + rpt_month_year = int(f"{calendar_year}{month_int}") if dfs_status == "Rejected": # we need to be careful here on examples of bad headers or empty files, since no month will be found From dbac14441d5dda25057634c180fe82a0aa4fbd6f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jul 2023 13:06:35 -0600 Subject: [PATCH 089/120] - updated rejected query to correctly count objs --- .../tdpservice/parsers/test/test_parse.py | 66 +++++++++---------- tdrs-backend/tdpservice/parsers/util.py | 14 ++-- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 0a228155b..d9bc86f48 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -128,9 +128,9 @@ def test_parse_big_file(test_big_file, dfs): assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) - assert dfs.case_aggregates == {'Oct': {'accepted': 270, 'rejected': 0, 'total': 270}, - 'Nov': {'accepted': 273, 'rejected': 0, 'total': 273}, - 'Dec': {'accepted': 272, 'rejected': 0, 'total': 272}} + assert dfs.case_aggregates == {'Oct': {'accepted': 169, 'rejected': 104, 'total': 270}, + 'Nov': {'accepted': 171, 'rejected': 99, 'total': 273}, + 'Dec': {'accepted': 166, 'rejected': 106, 'total': 272}} parser_errors = ParserError.objects.filter(file=test_big_file) assert parser_errors.count() == 355 @@ -503,38 +503,38 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): assert t2_2.FAMILY_AFFILIATION == 2 assert t2_2.OTHER_UNEARNED_INCOME == '0000' -@pytest.mark.django_db -def test_parse_tanf_section1_datafile_obj_counts(small_tanf_section1_datafile): - """Test parsing of small_tanf_section1_datafile in general.""" - errors = parse.parse_datafile(small_tanf_section1_datafile) - - assert errors == {} - assert TANF_T1.objects.count() == 5 - assert TANF_T2.objects.count() == 5 - assert TANF_T3.objects.count() == 6 +# @pytest.mark.django_db +# def test_parse_tanf_section1_datafile_obj_counts(small_tanf_section1_datafile): +# """Test parsing of small_tanf_section1_datafile in general.""" +# errors = parse.parse_datafile(small_tanf_section1_datafile) -@pytest.mark.django_db -def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile): - """Test parsing of small_tanf_section1_datafile and validate T3 model data.""" - errors = parse.parse_datafile(small_tanf_section1_datafile) +# assert errors == {} +# assert TANF_T1.objects.count() == 5 +# assert TANF_T2.objects.count() == 5 +# assert TANF_T3.objects.count() == 6 - assert errors == {} - assert TANF_T3.objects.count() == 6 - - t3_models = TANF_T3.objects.all() - t3_1 = t3_models[0] - assert t3_1.RPT_MONTH_YEAR == 202010 - assert t3_1.CASE_NUMBER == '11111111112' - assert t3_1.FAMILY_AFFILIATION == 1 - assert t3_1.GENDER == 2 - assert t3_1.EDUCATION_LEVEL == '98' - - t3_6 = t3_models[5] - assert t3_6.RPT_MONTH_YEAR == 202010 - assert t3_6.CASE_NUMBER == '11111111151' - assert t3_6.FAMILY_AFFILIATION == 1 - assert t3_6.GENDER == 2 - assert t3_6.EDUCATION_LEVEL == '98' +# @pytest.mark.django_db +# def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile): +# """Test parsing of small_tanf_section1_datafile and validate T3 model data.""" +# errors = parse.parse_datafile(small_tanf_section1_datafile) + +# assert errors == {} +# assert TANF_T3.objects.count() == 6 + +# t3_models = TANF_T3.objects.all() +# t3_1 = t3_models[0] +# assert t3_1.RPT_MONTH_YEAR == 202010 +# assert t3_1.CASE_NUMBER == '11111111112' +# assert t3_1.FAMILY_AFFILIATION == 1 +# assert t3_1.GENDER == 2 +# assert t3_1.EDUCATION_LEVEL == '98' + +# t3_6 = t3_models[5] +# assert t3_6.RPT_MONTH_YEAR == 202010 +# assert t3_6.CASE_NUMBER == '11111111151' +# assert t3_6.FAMILY_AFFILIATION == 1 +# assert t3_6.GENDER == 2 +# assert t3_6.EDUCATION_LEVEL == '98' @pytest.fixture diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 12b18c0e5..7adaa60e1 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -465,18 +465,18 @@ def case_aggregates_by_month(df): month_int = month_to_int(month) rpt_month_year = int(f"{df.year}{month_int}") - qset = set() + case_numbers = set() for schema_model in schema_models: if isinstance(schema_model, MultiRecordRowSchema): schema_model = schema_model.schemas[0] - - next = set(schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year) + + curr_case_numbers = set(schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year) .distinct("CASE_NUMBER").values_list("CASE_NUMBER", flat=True)) - qset = qset.union(next) + case_numbers = case_numbers.union(curr_case_numbers) + + total += len(case_numbers) + rejected += ParserError.objects.filter(case_number__in=case_numbers).distinct('case_number').count() - total += len(qset) - rejected += ParserError.objects.filter(content_type=ContentType.objects.get_for_model(schema_model.model), - case_number__in=qset).count() accepted = total - rejected aggregate_data[month] = {"accepted": accepted, "rejected": rejected, "total": total} From 2c7efd6de10448d94159fdfc1cf9290e1eb51230 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jul 2023 13:38:09 -0600 Subject: [PATCH 090/120] - Fixing most tests --- .../tdpservice/data_files/test/factories.py | 2 +- .../tdpservice/parsers/test/test_parse.py | 28 +++++++++---------- tdrs-backend/tdpservice/parsers/util.py | 4 +-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/factories.py b/tdrs-backend/tdpservice/data_files/test/factories.py index 34522154c..88333f7d9 100644 --- a/tdrs-backend/tdpservice/data_files/test/factories.py +++ b/tdrs-backend/tdpservice/data_files/test/factories.py @@ -18,7 +18,7 @@ class Meta: extension = "txt" section = "Active Case Data" quarter = "Q1" - year = "2020" + year = 2020 version = 1 user = factory.SubFactory(UserFactory) stt = factory.SubFactory(STTFactory) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index bd14edfa4..babffb3fb 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -33,7 +33,7 @@ def test_parse_small_correct_file(test_datafile, dfs): errors = parse.parse_datafile(test_datafile) - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 1, 'rejected': 0, 'total': 1}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} @@ -127,7 +127,8 @@ def test_parse_big_file(test_big_file, dfs): errors = parse.parse_datafile(test_big_file) assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) + print(dfs.case_aggregates) assert dfs.case_aggregates == {'Oct': {'accepted': 169, 'rejected': 104, 'total': 270}, 'Nov': {'accepted': 171, 'rejected': 99, 'total': 273}, 'Dec': {'accepted': 166, 'rejected': 106, 'total': 272}} @@ -356,13 +357,13 @@ def test_parse_empty_file(empty_file, dfs): errors = parse.parse_datafile(empty_file) dfs.status = dfs.get_status(errors) - dfs.case_aggregates = dfs.case_aggregates_by_month(empty_file) + dfs.case_aggregates = util.case_aggregates_by_month(empty_file, dfs.status) assert dfs.status == DataFileSummary.Status.REJECTED - assert dfs.case_aggregates[0] == "N/A" - print(dfs.case_aggregates[0]) - assert False - + assert dfs.case_aggregates == {'Oct': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, + 'Nov': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, + 'Dec': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}} + parser_errors = ParserError.objects.filter(file=empty_file) assert parser_errors.count() == 1 @@ -391,7 +392,8 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): expected_m2_record_count = 6 expected_m3_record_count = 8 - small_ssp_section1_datafile.year = 2018 + small_ssp_section1_datafile.year = 2019 + small_ssp_section1_datafile.quarter = 'Q1' small_ssp_section1_datafile.save() dfs.datafile = small_ssp_section1_datafile @@ -400,7 +402,7 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): errors = parse.parse_datafile(small_ssp_section1_datafile) assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} @@ -491,7 +493,7 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): errors = parse.parse_datafile(small_tanf_section1_datafile) assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} @@ -565,8 +567,6 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 - # for e in parser_errors: - # print(e.error_type, e.error_message) row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE @@ -658,7 +658,7 @@ def test_dfs_set_case_aggregates(test_datafile, dfs): test_datafile.section = 'Active Case Data' test_datafile.save() parse.parse_datafile(test_datafile) # this still needs to execute to create db objects to be queried - dfs.case_aggregates = util.case_aggregates_by_month(test_datafile) + dfs.case_aggregates = util.case_aggregates_by_month(test_datafile, dfs.status) dfs.save() assert dfs.case_aggregates['Oct']['accepted'] == 1 @@ -700,7 +700,7 @@ def test_get_schema_options(dfs): section = util.get_section_reference('TAN', 'C') assert section == DataFile.Section.CLOSED_CASE_DATA - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile) + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) # from datafile: # get model(s) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 5e92a7a27..50e50eefc 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -11,8 +11,8 @@ def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): """Create a test DataFile instance with the given file attached.""" path = str(Path(__file__).parent.joinpath('test/data')) + f'/{filename}' datafile = DataFile.create_new_version({ - 'quarter': 'Q4', - 'year': 2020, + 'quarter': 'Q1', + 'year': 2021, 'section': section, 'user': stt_user, 'stt': stt From e5108b2e4ab148063516a170e46cc4c8fd6732c5 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jul 2023 13:46:29 -0600 Subject: [PATCH 091/120] - Fixed user error. Swapped numbers by accident. --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index babffb3fb..5ee6c2bf4 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -129,8 +129,8 @@ def test_parse_big_file(test_big_file, dfs): assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) print(dfs.case_aggregates) - assert dfs.case_aggregates == {'Oct': {'accepted': 169, 'rejected': 104, 'total': 270}, - 'Nov': {'accepted': 171, 'rejected': 99, 'total': 273}, + assert dfs.case_aggregates == {'Oct': {'accepted': 171, 'rejected': 99, 'total': 270}, + 'Nov': {'accepted': 169, 'rejected': 104, 'total': 273}, 'Dec': {'accepted': 166, 'rejected': 106, 'total': 272}} parser_errors = ParserError.objects.filter(file=test_big_file) From ff1e2b88f203ba1dbb54e930249fd0a38114751f Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jul 2023 13:48:23 -0600 Subject: [PATCH 092/120] - make region None to avoid PK collision --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 5ee6c2bf4..6cae36b74 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -18,7 +18,10 @@ @pytest.fixture def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" - return util.create_test_datafile('small_correct_file', stt_user, stt) + df = util.create_test_datafile('small_correct_file', stt_user, stt) + df.region = None + df.save() + return df @pytest.fixture def dfs(): From 496a6513e00478da9e48b41c918781ba8cdea76c Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jul 2023 13:50:45 -0600 Subject: [PATCH 093/120] - Fix lint errors --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 6 +++--- tdrs-backend/tdpservice/parsers/util.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 6cae36b74..6b8b5bf78 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -363,10 +363,10 @@ def test_parse_empty_file(empty_file, dfs): dfs.case_aggregates = util.case_aggregates_by_month(empty_file, dfs.status) assert dfs.status == DataFileSummary.Status.REJECTED - assert dfs.case_aggregates == {'Oct': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, - 'Nov': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, + assert dfs.case_aggregates == {'Oct': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, + 'Nov': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, 'Dec': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}} - + parser_errors = ParserError.objects.filter(file=empty_file) assert parser_errors.count() == 1 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 50e50eefc..a9a144dd1 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -428,11 +428,11 @@ def get_schema(line, section, program_type): def fiscal_to_calendar(year, fiscal_quarter): """Decrement the input quarter text by one.""" - array = [1,2,3,4] # wrapping around an array + array = [1, 2, 3, 4] # wrapping around an array int_qtr = int(fiscal_quarter[1:]) # remove the 'Q', e.g., 'Q1' -> '1' if int_qtr == 1: year = year - 1 - + ind_qtr = array.index(int_qtr) # get the index so we can easily wrap-around end of array return year, "Q{}".format(array[ind_qtr - 1]) # return the previous quarter @@ -487,9 +487,9 @@ def case_aggregates_by_month(df, dfs_status): for schema_model in schema_models: if isinstance(schema_model, MultiRecordRowSchema): schema_model = schema_model.schemas[0] - + curr_case_numbers = set(schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year) - .distinct("CASE_NUMBER").values_list("CASE_NUMBER", flat=True)) + .distinct("CASE_NUMBER").values_list("CASE_NUMBER", flat=True)) case_numbers = case_numbers.union(curr_case_numbers) total += len(case_numbers) From aed64f9757411bb64a7409455f578b2bed343a53 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Thu, 20 Jul 2023 13:59:56 -0600 Subject: [PATCH 094/120] - Updating to avoid warning --- tdrs-backend/docker-compose.local.yml | 6 +++--- tdrs-frontend/docker-compose.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tdrs-backend/docker-compose.local.yml b/tdrs-backend/docker-compose.local.yml index d49c7d699..3e4499df3 100644 --- a/tdrs-backend/docker-compose.local.yml +++ b/tdrs-backend/docker-compose.local.yml @@ -79,7 +79,7 @@ services: build: . command: > bash -c "./wait_for_services.sh && - ./gunicorn_start.sh && + ./gunicorn_start.sh && celery -A tdpservice.settings worker -l info" ports: - "5555:5555" @@ -104,5 +104,5 @@ volumes: networks: default: - external: - name: external-net + name: external-net + external: true diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml index bd7c9a26e..403e72628 100644 --- a/tdrs-frontend/docker-compose.yml +++ b/tdrs-frontend/docker-compose.yml @@ -32,7 +32,7 @@ services: command: > /bin/sh -c "echo 'starting nginx' && - envsubst '$${BACK_END}' < /etc/nginx/locations.conf > /etc/nginx/locations_.conf && + envsubst '$${BACK_END}' < /etc/nginx/locations.conf > /etc/nginx/locations_.conf && rm /etc/nginx/locations.conf && cp /etc/nginx/locations_.conf /etc/nginx/locations.conf && envsubst ' @@ -46,5 +46,5 @@ networks: driver: bridge default: - external: - name: external-net + name: external-net + external: true From e01ac1586522452f98ea3af7e7566a452df75508 Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Fri, 21 Jul 2023 15:43:41 -0400 Subject: [PATCH 095/120] vscode merge conflict resolution (#2623) * auto-create the external network * didn't stage commit properly * checking diffs, matching 1613.2 * doesn't work in pipeline. must be cached local * re-commenting in unit test * lint failures fixed --------- Co-authored-by: andrew-jameson --- tdrs-backend/docker-compose.yml | 2 +- tdrs-backend/tdpservice/parsers/parse.py | 49 ------------------- .../tdpservice/parsers/test/test_parse.py | 33 +++++++------ 3 files changed, 19 insertions(+), 65 deletions(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index d6748e940..89908553a 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -119,4 +119,4 @@ volumes: networks: default: name: external-net - external: true \ No newline at end of file + external: true diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 1f22cebe6..7911e5d16 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -146,52 +146,3 @@ def parse_datafile_line(line, schema, generate_error, datafile): record.save() return record_is_valid, record_errors - - -def get_schema_options(program_type): - """Return the allowed schema options.""" - match program_type: - case 'TAN': - return { - 'A': { - 'T1': schema_defs.tanf.t1, - 'T2': schema_defs.tanf.t2, - 'T3': schema_defs.tanf.t3, - }, - 'C': { - # 'T4': schema_options.t4, - # 'T5': schema_options.t5, - }, - 'G': { - # 'T6': schema_options.t6, - }, - 'S': { - # 'T7': schema_options.t7, - }, - } - case 'SSP': - return { - 'A': { - 'M1': schema_defs.ssp.m1, - 'M2': schema_defs.ssp.m2, - 'M3': schema_defs.ssp.m3, - }, - 'C': { - # 'M4': schema_options.m4, - # 'M5': schema_options.m5, - }, - 'G': { - # 'M6': schema_options.m6, - }, - 'S': { - # 'M7': schema_options.m7, - }, - } - # case tribal? - return None - - -def get_schema(line, section, schema_options): - """Return the appropriate schema for the line.""" - line_type = line[0:2] - return schema_options.get(section, {}).get(line_type, None) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 6b8b5bf78..8ca598905 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -18,10 +18,7 @@ @pytest.fixture def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" - df = util.create_test_datafile('small_correct_file', stt_user, stt) - df.region = None - df.save() - return df + return util.create_test_datafile('small_correct_file', stt_user, stt) @pytest.fixture def dfs(): @@ -35,7 +32,7 @@ def test_parse_small_correct_file(test_datafile, dfs): dfs.save() errors = parse.parse_datafile(test_datafile) - + dfs.status = dfs.get_status(errors) dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 1, 'rejected': 0, 'total': 1}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, @@ -70,9 +67,13 @@ def test_parse_section_mismatch(test_datafile, dfs): dfs.save() errors = parse.parse_datafile(test_datafile) - - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + dfs.status = dfs.get_status(errors) + assert dfs.status == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) + dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == {'Oct': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, + 'Nov': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, + 'Dec': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}} assert parser_errors.count() == 1 err = parser_errors.first() @@ -128,10 +129,9 @@ def test_parse_big_file(test_big_file, dfs): dfs.save() errors = parse.parse_datafile(test_big_file) - - assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + dfs.status = dfs.get_status(errors) + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) - print(dfs.case_aggregates) assert dfs.case_aggregates == {'Oct': {'accepted': 171, 'rejected': 99, 'total': 270}, 'Nov': {'accepted': 169, 'rejected': 104, 'total': 273}, 'Dec': {'accepted': 166, 'rejected': 106, 'total': 272}} @@ -404,7 +404,8 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): errors = parse.parse_datafile(small_ssp_section1_datafile) - assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + dfs.status = dfs.get_status(errors) + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, @@ -495,7 +496,8 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): errors = parse.parse_datafile(small_tanf_section1_datafile) - assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED + dfs.status = dfs.get_status(errors) + assert dfs.status == DataFileSummary.Status.ACCEPTED dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, @@ -570,6 +572,8 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 + # for e in parser_errors: + # print(e.error_type, e.error_message) row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE @@ -660,7 +664,8 @@ def test_dfs_set_case_aggregates(test_datafile, dfs): """Test that the case aggregates are set correctly.""" test_datafile.section = 'Active Case Data' test_datafile.save() - parse.parse_datafile(test_datafile) # this still needs to execute to create db objects to be queried + errors = parse.parse_datafile(test_datafile) # this still needs to execute to create db objects to be queried + dfs.status = dfs.get_status(errors) dfs.case_aggregates = util.case_aggregates_by_month(test_datafile, dfs.status) dfs.save() @@ -703,8 +708,6 @@ def test_get_schema_options(dfs): section = util.get_section_reference('TAN', 'C') assert section == DataFile.Section.CLOSED_CASE_DATA - dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) - # from datafile: # get model(s) # get section str From 3d444a9a7c4d3d98b1e72d5180b058509422b388 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 26 Jul 2023 10:03:32 -0400 Subject: [PATCH 096/120] url change per me, want pipeline to run e2e --- tdrs-backend/tdpservice/parsers/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/urls.py b/tdrs-backend/tdpservice/parsers/urls.py index 625b84d04..cd1d560d3 100644 --- a/tdrs-backend/tdpservice/parsers/urls.py +++ b/tdrs-backend/tdpservice/parsers/urls.py @@ -5,8 +5,8 @@ router = DefaultRouter() -router.register("parsing_errors/", ParsingErrorViewSet) -router.register("dfs/", DataFileSummaryViewSet) +router.register("parsing_errors", ParsingErrorViewSet) +router.register("dfs", DataFileSummaryViewSet) urlpatterns = [ path('', include(router.urls)), From 3cb28b9336c8d76422f0943f7af6eca74cfef70e Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 7 Aug 2023 13:41:40 -0400 Subject: [PATCH 097/120] Upgraded to querysets, fix PR comments, PE str --- tdrs-backend/tdpservice/parsers/models.py | 44 ++++++------------- .../tdpservice/parsers/test/test_parse.py | 36 ++++++++------- tdrs-backend/tdpservice/parsers/util.py | 2 +- 3 files changed, 36 insertions(+), 46 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 5a98ad9f0..7c9342456 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -61,7 +61,7 @@ def __repr__(self): def __str__(self): """Return a string representation of the model.""" - return f"ParserError {self.id}" + return f"ParserError {self.values()}" def _get_error_message(self): """Return the error message.""" @@ -88,39 +88,23 @@ class Status(models.TextChoices): case_aggregates = models.JSONField(null=True, blank=False) - def get_status(self, errors): + def get_status(self): """Set and return the status field based on errors and models associated with datafile.""" - if errors is None: - return DataFileSummary.Status.PENDING + errors = ParserError.objects.filter(file=self.datafile) - if type(errors) != dict: - raise TypeError("errors parameter must be a dictionary.") + # excluding row-level pre-checks and trailer pre-checks. + precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\ + .exclude(field_name="Record")\ + .exclude(error_message__contains="railer") + # The "railer" is not a typo, we see both t and T in the error message. - if errors == {}: + if errors is None: + return DataFileSummary.Status.PENDING + elif errors.count() == 0: return DataFileSummary.Status.ACCEPTED - elif DataFileSummary.find_precheck(errors): + elif precheck_errors.count() > 0: + print(precheck_errors.values()) return DataFileSummary.Status.REJECTED else: + print(errors) return DataFileSummary.Status.ACCEPTED_WITH_ERRORS - - def find_precheck(errors): - """Check for pre-parsing errors. - - @param errors: dict of errors keyed by location in datafile. - e.g. - errors = - { - "trailer": [ParserError, ...], - "header": [ParserError, ...], - "document": [ParserError, ...], - "123": [ParserError, ...], - ... - } - """ - for key in errors.keys(): - if key == 'trailer': - continue - for parserError in errors[key]: - if type(parserError) is ParserError and parserError.error_type == ParserErrorCategoryChoices.PRE_CHECK: - return True - return False diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 8ca598905..8a8e8eca7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -32,14 +32,14 @@ def test_parse_small_correct_file(test_datafile, dfs): dfs.save() errors = parse.parse_datafile(test_datafile) - dfs.status = dfs.get_status(errors) + dfs.status = dfs.get_status() dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 1, 'rejected': 0, 'total': 1}, 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} assert errors == {} - assert dfs.get_status(errors) == DataFileSummary.Status.ACCEPTED + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED assert TANF_T1.objects.count() == 1 assert ParserError.objects.filter(file=test_datafile).count() == 0 @@ -67,7 +67,7 @@ def test_parse_section_mismatch(test_datafile, dfs): dfs.save() errors = parse.parse_datafile(test_datafile) - dfs.status = dfs.get_status(errors) + dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) @@ -94,8 +94,10 @@ def test_parse_wrong_program_type(test_datafile, dfs): test_datafile.section = 'SSP Active Case Data' test_datafile.save() + dfs.datafile = test_datafile + dfs.save() errors = parse.parse_datafile(test_datafile) - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + assert dfs.get_status() == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) assert parser_errors.count() == 1 @@ -129,7 +131,7 @@ def test_parse_big_file(test_big_file, dfs): dfs.save() errors = parse.parse_datafile(test_big_file) - dfs.status = dfs.get_status(errors) + dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 171, 'rejected': 99, 'total': 270}, @@ -189,8 +191,9 @@ def bad_file_missing_header(stt_user, stt): def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): """Test parsing of bad_missing_header.""" errors = parse.parse_datafile(bad_file_missing_header) - - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + dfs.datafile = bad_file_missing_header + dfs.save() + assert dfs.get_status() == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_missing_header) assert parser_errors.count() == 1 @@ -217,8 +220,9 @@ def bad_file_multiple_headers(stt_user, stt): def test_parse_bad_file_multiple_headers(bad_file_multiple_headers, dfs): """Test parsing of bad_two_headers.""" errors = parse.parse_datafile(bad_file_multiple_headers) - - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + dfs.datafile = bad_file_multiple_headers + dfs.save() + assert dfs.get_status() == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_file_multiple_headers) assert parser_errors.count() == 1 @@ -359,7 +363,7 @@ def test_parse_empty_file(empty_file, dfs): dfs.save() errors = parse.parse_datafile(empty_file) - dfs.status = dfs.get_status(errors) + dfs.status = dfs.get_status() dfs.case_aggregates = util.case_aggregates_by_month(empty_file, dfs.status) assert dfs.status == DataFileSummary.Status.REJECTED @@ -404,7 +408,7 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): errors = parse.parse_datafile(small_ssp_section1_datafile) - dfs.status = dfs.get_status(errors) + dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, @@ -496,7 +500,7 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): errors = parse.parse_datafile(small_tanf_section1_datafile) - dfs.status = dfs.get_status(errors) + dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, @@ -568,7 +572,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field errors = parse.parse_datafile(bad_tanf_s1__row_missing_required_field) - assert dfs.get_status(errors) == DataFileSummary.Status.REJECTED + assert dfs.get_status() == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 @@ -664,8 +668,10 @@ def test_dfs_set_case_aggregates(test_datafile, dfs): """Test that the case aggregates are set correctly.""" test_datafile.section = 'Active Case Data' test_datafile.save() - errors = parse.parse_datafile(test_datafile) # this still needs to execute to create db objects to be queried - dfs.status = dfs.get_status(errors) + parse.parse_datafile(test_datafile) # this still needs to execute to create db objects to be queried + dfs.file = test_datafile + dfs.save() + dfs.status = dfs.get_status() dfs.case_aggregates = util.case_aggregates_by_month(test_datafile, dfs.status) dfs.save() diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index a9a144dd1..82aa3be63 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -186,7 +186,7 @@ def run_preparsing_validators(self, line, generate_error): error_category=ParserErrorCategoryChoices.PRE_CHECK, error_message=validator_error, record=None, - field=None + field="Record" ) ) From 90709a3e97d524abcfbcce412e8244f4316d2c1a Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 7 Aug 2023 14:00:30 -0400 Subject: [PATCH 098/120] missing : not caught locally --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index db397b3b0..2b55ae08f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -577,7 +577,7 @@ def bad_tanf_s1__row_missing_required_field(stt_user, stt): @pytest.mark.django_db -def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field, dfs) +def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field, dfs): """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" dfs.datafile = bad_tanf_s1__row_missing_required_field dfs.save() From ad8ab503503657b555bc790d5c6dfe751ab8f18b Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Mon, 7 Aug 2023 17:26:26 -0400 Subject: [PATCH 099/120] Feat/1613 merge 2 (#2650) * Create sprint-78-summary.md (#2645) * Missing/unsaved parser_error for record_type * removing redundant tests * Hopefully resolved on unit tests and lint --------- Co-authored-by: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Co-authored-by: andrew-jameson --- docs/Sprint-Review/sprint-78-summary.md | 48 +++++++++++++++++++ tdrs-backend/tdpservice/parsers/models.py | 6 +-- tdrs-backend/tdpservice/parsers/parse.py | 43 +++++++---------- tdrs-backend/tdpservice/parsers/row_schema.py | 2 +- .../tdpservice/parsers/test/test_models.py | 36 +------------- .../tdpservice/parsers/test/test_parse.py | 12 +++-- tdrs-backend/tdpservice/parsers/util.py | 38 ++++++--------- 7 files changed, 92 insertions(+), 93 deletions(-) create mode 100644 docs/Sprint-Review/sprint-78-summary.md diff --git a/docs/Sprint-Review/sprint-78-summary.md b/docs/Sprint-Review/sprint-78-summary.md new file mode 100644 index 000000000..5e857a704 --- /dev/null +++ b/docs/Sprint-Review/sprint-78-summary.md @@ -0,0 +1,48 @@ +# Sprint 78 Summary +07/19/23 - 08/01/23 + +Velocity: Dev (16) + +## Sprint Goal +* Continue parsing engine development for TANF Section 1 and Section 2, complete decoupling backend application spike and continue integration test epic (310). +* UX errors template, follow-on research, onboarding and maintenance page copy +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [#2618 Down for maintenance page](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2618) +* [#2598 HHS Staging Deployment Failure](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2598) +* [#332 Process encrypted files TANF (01)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/332) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) +* [#1109 TANF (02) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1109) +* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429) +* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612) + +### Blocked +* [#1784 - Email Relay](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1784) + + +### Raft Review +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) + +### Unplanned work +* UX: + * Static down for maintenance copy and banner rework diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 7c9342456..8c00d830b 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -94,17 +94,15 @@ def get_status(self): # excluding row-level pre-checks and trailer pre-checks. precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\ - .exclude(field_name="Record")\ + .exclude(field_name="Record_Type")\ .exclude(error_message__contains="railer") # The "railer" is not a typo, we see both t and T in the error message. - + print(precheck_errors.values()) if errors is None: return DataFileSummary.Status.PENDING elif errors.count() == 0: return DataFileSummary.Status.ACCEPTED elif precheck_errors.count() > 0: - print(precheck_errors.values()) return DataFileSummary.Status.REJECTED else: - print(errors) return DataFileSummary.Status.ACCEPTED_WITH_ERRORS diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 206ad577a..05a96e956 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -1,6 +1,6 @@ """Convert raw uploaded Datafile into a parsed model, and accumulate/return any errors.""" -import os + from django.db import DatabaseError import itertools import logging @@ -129,19 +129,6 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): header_count += int(line.startswith('HEADER')) trailer_count += int(line.startswith('TRAILER')) - schema = util.get_schema(line, section, program_type) - if schema is None: - errors[line_number] = [util.generate_parser_error( - datafile=datafile, - line_number=line_number, - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="Unknown Record_Type was found.", - record=None, - field="Record_Type", - )] - continue - is_last = offset == file_length multiple_trailer_errors, trailer_errors = evaluate_trailer(datafile, trailer_count, multiple_trailer_errors, is_last, line, line_number) @@ -162,7 +149,6 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): record=None, field=None ) - preparse_error = {line_number: [err_obj]} unsaved_parser_errors.update(preparse_error) rollback_records(unsaved_records, datafile) @@ -174,6 +160,22 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): prev_sum = header_count + trailer_count continue + schema = util.get_schema(line, section, program_type) + if schema is None: + err_obj = util.generate_parser_error( + datafile=datafile, + line_number=line_number, + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Unknown Record_Type was found.", + record=None, + field="Record_Type", + ) + preparse_error = {line_number: [err_obj]} + errors[line_number] = [err_obj] + unsaved_parser_errors.update(preparse_error) + continue + schema_manager = get_schema_manager(line, section, schema_manager_options) schema_manager.update_encrypted_fields(is_encrypted) @@ -233,16 +235,6 @@ def manager_parse_line(line, schema_manager, generate_error): records = schema_manager.parse_and_validate(line, generate_error) return records - return [(None, False, [ - generate_error( - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="Record Type is missing from record.", - record=None, - field=None - ) - ])] - def get_schema_manager_options(program_type): """Return the allowed schema options.""" @@ -291,4 +283,3 @@ def get_schema_manager(line, section, schema_options): """Return the appropriate schema for the line.""" line_type = line[0:2] return schema_options.get(section, {}).get(line_type, util.SchemaManager([])) - diff --git a/tdrs-backend/tdpservice/parsers/row_schema.py b/tdrs-backend/tdpservice/parsers/row_schema.py index a15b1a6bc..f516dd37e 100644 --- a/tdrs-backend/tdpservice/parsers/row_schema.py +++ b/tdrs-backend/tdpservice/parsers/row_schema.py @@ -77,7 +77,7 @@ def run_preparsing_validators(self, line, generate_error): error_category=ParserErrorCategoryChoices.PRE_CHECK, error_message=validator_error, record=None, - field=None + field="Record_Type" ) ) diff --git a/tdrs-backend/tdpservice/parsers/test/test_models.py b/tdrs-backend/tdpservice/parsers/test/test_models.py index 0b607777c..783e859e7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_models.py +++ b/tdrs-backend/tdpservice/parsers/test/test_models.py @@ -1,8 +1,8 @@ """Module testing for data file model.""" import pytest -from tdpservice.parsers.models import ParserError, DataFileSummary, ParserErrorCategoryChoices -from .factories import DataFileSummaryFactory, ParserErrorFactory +from tdpservice.parsers.models import ParserError +from .factories import ParserErrorFactory @pytest.fixture def parser_error_instance(): @@ -21,35 +21,3 @@ def test_parser_error_rpt_month_name(parser_error_instance): """Test that the parser error instance is created.""" parser_error_instance.rpt_month_year = 202001 assert parser_error_instance.rpt_month_name == "January" - - -@pytest.fixture() -def dfs(): - """Fixture for DataFileSummary.""" - return DataFileSummaryFactory.create() - - -@pytest.mark.django_db -def test_dfs_model(dfs): - """Test that the model is created and populated correctly.""" - assert dfs.case_aggregates['Jan']['accepted'] == 100 - - -@pytest.mark.django_db -def test_dfs_get_status(dfs): - """Test that the initial status is set correctly.""" - assert dfs.status == DataFileSummary.Status.PENDING - - # create empty Error dict to prompt accepted status - assert dfs.get_status(errors={}) == DataFileSummary.Status.ACCEPTED - - # create category 2 ParserError list to prompt accepted with errors status - parser_errors = [] - for i in range(2, 4): - parser_errors.append(ParserErrorFactory(row_number=i, error_type=str(i))) - - assert dfs.get_status(errors={'document': parser_errors}) == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - - # create category 1 ParserError list to prompt rejected status - parser_errors.append(ParserErrorFactory(row_number=5, error_type=ParserErrorCategoryChoices.PRE_CHECK)) - assert dfs.get_status(errors={'document': parser_errors}) == DataFileSummary.Status.REJECTED diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 2b55ae08f..2e642b6a9 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -494,7 +494,7 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): @pytest.mark.django_db() def test_parse_tanf_section1_datafile_obj_counts(small_tanf_section1_datafile): """Test parsing of small_tanf_section1_datafile in general.""" - errors = parse.parse_datafile(small_tanf_section1_datafile) + parse.parse_datafile(small_tanf_section1_datafile) # assert errors == {} # assert TANF_T1.objects.count() == 5 @@ -504,7 +504,7 @@ def test_parse_tanf_section1_datafile_obj_counts(small_tanf_section1_datafile): @pytest.mark.django_db() def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile): """Test parsing of small_tanf_section1_datafile and validate T3 model data.""" - errors = parse.parse_datafile(small_tanf_section1_datafile) + parse.parse_datafile(small_tanf_section1_datafile) # assert errors == {} # assert TANF_T3.objects.count() == 6 @@ -527,7 +527,8 @@ def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile): @pytest.fixture def super_big_s1_file(stt_user, stt): """Fixture for ADS.E2J.NDM1.TS53_fake.""" - return create_test_datafile('ADS.E2J.NDM1.TS53_fake', stt_user, stt) + return util.create_test_datafile('ADS.E2J.NDM1.TS53_fake', stt_user, stt) + @pytest.mark.django_db() def test_parse_super_big_s1_file(super_big_s1_file): @@ -544,7 +545,7 @@ def test_parse_super_big_s1_file(super_big_s1_file): @pytest.fixture def super_big_s1_rollback_file(stt_user, stt): """Fixture for ADS.E2J.NDM1.TS53_fake.rollback.""" - return create_test_datafile('ADS.E2J.NDM1.TS53_fake.rollback', stt_user, stt) + return util.create_test_datafile('ADS.E2J.NDM1.TS53_fake.rollback', stt_user, stt) @pytest.mark.django_db() def test_parse_super_big_s1_file_with_rollback(super_big_s1_rollback_file): @@ -584,7 +585,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field errors = parse.parse_datafile(bad_tanf_s1__row_missing_required_field) - assert dfs.get_status() == DataFileSummary.Status.REJECTED + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 @@ -635,6 +636,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel errors = parse.parse_datafile(bad_ssp_s1__row_missing_required_field) parser_errors = ParserError.objects.filter(file=bad_ssp_s1__row_missing_required_field) + print(parser_errors.values()) assert parser_errors.count() == 5 row_2_error = parser_errors.get(row_number=2) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 3b65741c9..2f04d89f8 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -23,14 +23,6 @@ def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): return datafile -def value_is_empty(value, length): - """Handle 'empty' values as field inputs.""" - empty_values = [ - ' '*length, # ' ' - '#'*length, # '#####' - ] - - return value is None or value in empty_values def generate_parser_error(datafile, line_number, schema, error_category, error_message, record=None, field=None): """Create and return a ParserError using args.""" @@ -67,9 +59,9 @@ def generate(schema, error_category, error_message, record=None, field=None): return generate - class SchemaManager: """Manages one or more RowSchema's and runs all parsers and validators.""" + def __init__(self, schemas): self.schemas = schemas @@ -83,6 +75,19 @@ def parse_and_validate(self, line, generate_error): return records + def update_encrypted_fields(self, is_encrypted): + """Update whether schema fields are encrypted or not.""" + for schema in self.schemas: + for field in schema.fields: + if type(field) == EncryptedField: + field.is_encrypted = is_encrypted + +def contains_encrypted_indicator(line, encryption_field): + """Determine if line contains encryption indicator.""" + if encryption_field is not None: + return encryption_field.parse_value(line) == "E" + return False + def get_schema_options(program, section, query=None, model=None, model_name=None): """Centralized function to return the appropriate schema for a given program, section, and query. @@ -285,7 +290,7 @@ def case_aggregates_by_month(df, dfs_status): case_numbers = set() for schema_model in schema_models: - if isinstance(schema_model, MultiRecordRowSchema): + if isinstance(schema_model, SchemaManager): schema_model = schema_model.schemas[0] curr_case_numbers = set(schema_model.model.objects.filter(datafile=df).filter(RPT_MONTH_YEAR=rpt_month_year) @@ -300,16 +305,3 @@ def case_aggregates_by_month(df, dfs_status): aggregate_data[month] = {"accepted": accepted, "rejected": rejected, "total": total} return aggregate_data - - def update_encrypted_fields(self, is_encrypted): - """Update whether schema fields are encrypted or not.""" - for schema in self.schemas: - for field in schema.fields: - if type(field) == EncryptedField: - field.is_encrypted = is_encrypted - -def contains_encrypted_indicator(line, encryption_field): - """Determine if line contains encryption indicator.""" - if encryption_field is not None: - return encryption_field.parse_value(line) == "E" - return False From 3af74408624e20b61891e1c6718ee504cdb390f8 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 8 Aug 2023 15:37:41 -0400 Subject: [PATCH 100/120] icontains --- tdrs-backend/tdpservice/parsers/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 8c00d830b..4b48618fc 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -95,7 +95,7 @@ def get_status(self): # excluding row-level pre-checks and trailer pre-checks. precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\ .exclude(field_name="Record_Type")\ - .exclude(error_message__contains="railer") + .exclude(error_message__icontains="trailer") # The "railer" is not a typo, we see both t and T in the error message. print(precheck_errors.values()) if errors is None: From b98739925e09ca3682c8456aeb604dc3b1ca9baa Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 8 Aug 2023 15:57:04 -0400 Subject: [PATCH 101/120] tests --- tdrs-backend/tdpservice/parsers/models.py | 6 +++--- tdrs-backend/tdpservice/scheduling/parser_task.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 4b48618fc..ab41cf02c 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -95,9 +95,9 @@ def get_status(self): # excluding row-level pre-checks and trailer pre-checks. precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\ .exclude(field_name="Record_Type")\ - .exclude(error_message__icontains="trailer") - # The "railer" is not a typo, we see both t and T in the error message. - print(precheck_errors.values()) + .exclude(error_message__icontains="trailer")\ + .exclude(error_message__icontains="Unknown Record_Type was found.") + if errors is None: return DataFileSummary.Status.PENDING elif errors.count() == 0: diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index 2af0f1493..a9c6cf5d8 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -23,7 +23,7 @@ def parse(data_file_id): dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING) errors = parse_datafile(data_file) - dfs.status = dfs.get_status(errors=errors) + dfs.status = dfs.get_status() dfs.case_aggregates = case_aggregates_by_month(data_file) dfs.save() logger.info(f"DataFile parsing finished with status {dfs.status} and {len(errors)} errors: {errors}") From 091c8d97c15266bdb8e194cef743e3228cd831f8 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 8 Aug 2023 17:24:30 -0400 Subject: [PATCH 102/120] Changing dict structure per 1612. --- .../tdpservice/parsers/test/factories.py | 33 +++++----- .../tdpservice/parsers/test/test_parse.py | 63 ++++++++++++------- tdrs-backend/tdpservice/parsers/util.py | 13 ++-- 3 files changed, 67 insertions(+), 42 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 4af7a6707..927b9c94c 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -37,21 +37,24 @@ class Meta: status = DataFileSummary.Status.PENDING case_aggregates = { - "Jan": { - "accepted": 100, - "rejected": 10, - "total": 110 - }, - "Feb": { - "accepted": 100, - "rejected": 10, - "total": 110 - }, - "Mar": { - "accepted": 100, - "rejected": 10, - "total": 110 - } + "rejected": 0, + "months": [ + { + "accepted_without_errors": 100, + "accepted_with_errors": 10, + "month": "Jan", + }, + { + "accepted_without_errors": 100, + "accepted_with_errors": 10, + "month": "Feb", + }, + { + "accepted_without_errors": 100, + "accepted_with_errors": 10, + "month": "Mar", + }, + ] } datafile = factory.SubFactory(DataFileFactory) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 2e642b6a9..8879d96c5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -35,9 +35,12 @@ def test_parse_small_correct_file(test_datafile, dfs): errors = parse.parse_datafile(test_datafile) dfs.status = dfs.get_status() dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == {'Oct': {'accepted': 1, 'rejected': 0, 'total': 1}, - 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, - 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} + assert dfs.case_aggregates == {'rejected': 0, + 'months': [ + {'accepted_without_errors': 1, 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} + ]} assert errors == {} assert dfs.get_status() == DataFileSummary.Status.ACCEPTED @@ -71,9 +74,12 @@ def test_parse_section_mismatch(test_datafile, dfs): assert dfs.status == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == {'Oct': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, - 'Nov': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, - 'Dec': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}} + assert dfs.case_aggregates == {'rejected': 0, + 'months': [ + {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Oct'}, + {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Nov'}, + {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Dec'} + ]} assert parser_errors.count() == 1 err = parser_errors.first() @@ -120,6 +126,7 @@ def test_big_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.FTP1.TS06', stt_user, stt) @pytest.mark.django_db +@pytest.mark.big_files def test_parse_big_file(test_big_file, dfs): """Test parsing of ADS.E2J.FTP1.TS06.""" expected_t1_record_count = 815 @@ -133,9 +140,12 @@ def test_parse_big_file(test_big_file, dfs): dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == {'Oct': {'accepted': 171, 'rejected': 99, 'total': 270}, - 'Nov': {'accepted': 169, 'rejected': 104, 'total': 273}, - 'Dec': {'accepted': 166, 'rejected': 106, 'total': 272}} + assert dfs.case_aggregates == {'rejected': 0, + 'months': [ + {'accepted_without_errors': 171, 'accepted_with_errors': 99, 'month': 'Oct'}, + {'accepted_without_errors': 169, 'accepted_with_errors': 104, 'month': 'Nov'}, + {'accepted_without_errors': 166, 'accepted_with_errors': 106, 'month': 'Dec'} + ]} parser_errors = ParserError.objects.filter(file=test_big_file) assert parser_errors.count() == 355 @@ -357,9 +367,12 @@ def test_parse_empty_file(empty_file, dfs): dfs.case_aggregates = util.case_aggregates_by_month(empty_file, dfs.status) assert dfs.status == DataFileSummary.Status.REJECTED - assert dfs.case_aggregates == {'Oct': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, - 'Nov': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}, - 'Dec': {'accepted': 'N/A', 'rejected': 'N/A', 'total': 'N/A'}} + assert dfs.case_aggregates == {'rejected': 2, + 'months': [ + {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Oct'}, + {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Nov'}, + {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Dec'} + ]} parser_errors = ParserError.objects.filter(file=empty_file) assert parser_errors.count() == 2 @@ -401,9 +414,12 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, - 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, - 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} + assert dfs.case_aggregates == {'rejected': 1, + 'months': [ + {'accepted_without_errors': 5, 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} + ]} parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) assert parser_errors.count() == 1 @@ -469,9 +485,12 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == {'Oct': {'accepted': 5, 'rejected': 0, 'total': 5}, - 'Nov': {'accepted': 0, 'rejected': 0, 'total': 0}, - 'Dec': {'accepted': 0, 'rejected': 0, 'total': 0}} + assert dfs.case_aggregates == {'rejected': 0, + 'months': [ + {'accepted_without_errors': 5, 'accepted_with_errors': 0, 'month': 'Oct'}, + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} + ]} assert errors == {} assert TANF_T2.objects.count() == 5 @@ -548,6 +567,7 @@ def super_big_s1_rollback_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.NDM1.TS53_fake.rollback', stt_user, stt) @pytest.mark.django_db() +@pytest.mark.big_files def test_parse_super_big_s1_file_with_rollback(super_big_s1_rollback_file): """Test parsing of super_big_s1_rollback_file. @@ -689,9 +709,10 @@ def test_dfs_set_case_aggregates(test_datafile, dfs): dfs.case_aggregates = util.case_aggregates_by_month(test_datafile, dfs.status) dfs.save() - assert dfs.case_aggregates['Oct']['accepted'] == 1 - assert dfs.case_aggregates['Oct']['rejected'] == 0 - assert dfs.case_aggregates['Oct']['total'] == 1 + for month in dfs.case_aggregates['months']: + if month['month'] == 'Oct': + assert month['accepted_without_errors'] == 1 + assert month['accepted_with_errors'] == 0 @pytest.mark.django_db def test_get_schema_options(dfs): diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 2f04d89f8..9a1585a8e 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -274,10 +274,10 @@ def case_aggregates_by_month(df, dfs_status): schema_models_dict = get_program_models(program_type, short_section) schema_models = [model for model in schema_models_dict.values()] - aggregate_data = {} + aggregate_data = {"months":[], "rejected": 0} for month in month_list: total = 0 - rejected = 0 + cases_with_errors = 0 accepted = 0 month_int = month_to_int(month) rpt_month_year = int(f"{calendar_year}{month_int}") @@ -285,7 +285,7 @@ def case_aggregates_by_month(df, dfs_status): if dfs_status == "Rejected": # we need to be careful here on examples of bad headers or empty files, since no month will be found # but we can rely on the frontend submitted year-quarter to still generate the list of months - aggregate_data[month] = {"accepted": "N/A", "rejected": "N/A", "total": "N/A"} + aggregate_data["months"].append({"accepted_with_errors": "N/A", "accepted_without_errors": "N/A", "month": month}) continue case_numbers = set() @@ -298,10 +298,11 @@ def case_aggregates_by_month(df, dfs_status): case_numbers = case_numbers.union(curr_case_numbers) total += len(case_numbers) - rejected += ParserError.objects.filter(case_number__in=case_numbers).distinct('case_number').count() + cases_with_errors += ParserError.objects.filter(case_number__in=case_numbers).distinct('case_number').count() + accepted = total - cases_with_errors - accepted = total - rejected + aggregate_data['months'].append({"month": month, "accepted_without_errors": accepted, "accepted_with_errors": cases_with_errors}) - aggregate_data[month] = {"accepted": accepted, "rejected": rejected, "total": total} + aggregate_data['rejected'] = ParserError.objects.filter(file=df).filter(case_number=None).count() return aggregate_data From ebeaec8e3eabe4dac22679693517629b138a81a9 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 9 Aug 2023 09:53:43 -0400 Subject: [PATCH 103/120] fixed tests and lint issues, parse is too complex --- .../tdpservice/parsers/test/test_parse.py | 34 +++++++++++++------ tdrs-backend/tdpservice/parsers/util.py | 10 ++++-- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 8879d96c5..72b6002fd 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -39,7 +39,7 @@ def test_parse_small_correct_file(test_datafile, dfs): 'months': [ {'accepted_without_errors': 1, 'accepted_with_errors': 0, 'month': 'Oct'}, {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} ]} assert errors == {} @@ -74,11 +74,17 @@ def test_parse_section_mismatch(test_datafile, dfs): assert dfs.status == DataFileSummary.Status.REJECTED parser_errors = ParserError.objects.filter(file=test_datafile) dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == {'rejected': 0, + assert dfs.case_aggregates == {'rejected': 1, 'months': [ - {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Oct'}, - {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Nov'}, - {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Dec'} + {'accepted_without_errors': 'N/A', + 'accepted_with_errors': 'N/A', + 'month': 'Oct'}, + {'accepted_without_errors': 'N/A', + 'accepted_with_errors': 'N/A', + 'month': 'Nov'}, + {'accepted_without_errors': 'N/A', + 'accepted_with_errors': 'N/A', + 'month': 'Dec'} ]} assert parser_errors.count() == 1 @@ -144,7 +150,7 @@ def test_parse_big_file(test_big_file, dfs): 'months': [ {'accepted_without_errors': 171, 'accepted_with_errors': 99, 'month': 'Oct'}, {'accepted_without_errors': 169, 'accepted_with_errors': 104, 'month': 'Nov'}, - {'accepted_without_errors': 166, 'accepted_with_errors': 106, 'month': 'Dec'} + {'accepted_without_errors': 166, 'accepted_with_errors': 106, 'month': 'Dec'} ]} parser_errors = ParserError.objects.filter(file=test_big_file) @@ -369,9 +375,15 @@ def test_parse_empty_file(empty_file, dfs): assert dfs.status == DataFileSummary.Status.REJECTED assert dfs.case_aggregates == {'rejected': 2, 'months': [ - {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Oct'}, - {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Nov'}, - {'accepted_without_errors': 'N/A', 'accepted_with_errors': 'N/A', 'month': 'Dec'} + {'accepted_without_errors': 'N/A', + 'accepted_with_errors': 'N/A', + 'month': 'Oct'}, + {'accepted_without_errors': 'N/A', + 'accepted_with_errors': 'N/A', + 'month': 'Nov'}, + {'accepted_without_errors': 'N/A', + 'accepted_with_errors': 'N/A', + 'month': 'Dec'} ]} parser_errors = ParserError.objects.filter(file=empty_file) @@ -418,7 +430,7 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): 'months': [ {'accepted_without_errors': 5, 'accepted_with_errors': 0, 'month': 'Oct'}, {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} ]} parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) @@ -489,7 +501,7 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): 'months': [ {'accepted_without_errors': 5, 'accepted_with_errors': 0, 'month': 'Oct'}, {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Nov'}, - {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} + {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} ]} assert errors == {} diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 9a1585a8e..37e4664f1 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -274,7 +274,7 @@ def case_aggregates_by_month(df, dfs_status): schema_models_dict = get_program_models(program_type, short_section) schema_models = [model for model in schema_models_dict.values()] - aggregate_data = {"months":[], "rejected": 0} + aggregate_data = {"months": [], "rejected": 0} for month in month_list: total = 0 cases_with_errors = 0 @@ -285,7 +285,9 @@ def case_aggregates_by_month(df, dfs_status): if dfs_status == "Rejected": # we need to be careful here on examples of bad headers or empty files, since no month will be found # but we can rely on the frontend submitted year-quarter to still generate the list of months - aggregate_data["months"].append({"accepted_with_errors": "N/A", "accepted_without_errors": "N/A", "month": month}) + aggregate_data["months"].append({"accepted_with_errors": "N/A", + "accepted_without_errors": "N/A", + "month": month}) continue case_numbers = set() @@ -301,7 +303,9 @@ def case_aggregates_by_month(df, dfs_status): cases_with_errors += ParserError.objects.filter(case_number__in=case_numbers).distinct('case_number').count() accepted = total - cases_with_errors - aggregate_data['months'].append({"month": month, "accepted_without_errors": accepted, "accepted_with_errors": cases_with_errors}) + aggregate_data['months'].append({"month": month, + "accepted_without_errors": accepted, + "accepted_with_errors": cases_with_errors}) aggregate_data['rejected'] = ParserError.objects.filter(file=df).filter(case_number=None).count() From 6fbcebf1ecff9f741a7ab8b2325c2b0d1480f35d Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 9 Aug 2023 10:10:38 -0400 Subject: [PATCH 104/120] schema_manager replaces schema check --- tdrs-backend/tdpservice/parsers/parse.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 05a96e956..c032e1215 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -160,22 +160,6 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): prev_sum = header_count + trailer_count continue - schema = util.get_schema(line, section, program_type) - if schema is None: - err_obj = util.generate_parser_error( - datafile=datafile, - line_number=line_number, - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="Unknown Record_Type was found.", - record=None, - field="Record_Type", - ) - preparse_error = {line_number: [err_obj]} - errors[line_number] = [err_obj] - unsaved_parser_errors.update(preparse_error) - continue - schema_manager = get_schema_manager(line, section, schema_manager_options) schema_manager.update_encrypted_fields(is_encrypted) From 66349798e72924d9842562dcc4c2f39cce588503 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 9 Aug 2023 15:00:04 -0400 Subject: [PATCH 105/120] Saving state prior to merge-conflict. --- tdrs-backend/tdpservice/parsers/parse.py | 22 +++++++++++++++++++--- tdrs-backend/tdpservice/parsers/util.py | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index c032e1215..7b79acadd 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -106,7 +106,7 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): errors = {} line_number = 0 - schema_manager_options = get_schema_manager_options(program_type) + schema_manager_options = get_schema_manager_options(program_type) # TODO: make another wrapper to replace and use get_schema_options unsaved_records = {} unsaved_parser_errors = {} @@ -160,6 +160,22 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): prev_sum = header_count + trailer_count continue + schema = util.get_schema(line, section, program_type) + if schema is None: #TODO: this pushes complexity >10, should be handled by the schema manager + err_obj = util.generate_parser_error( + datafile=datafile, + line_number=line_number, + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Unknown Record_Type was found.", + record=None, + field="Record_Type", + ) + preparse_error = {line_number: [err_obj]} + errors[line_number] = [err_obj] + unsaved_parser_errors.update(preparse_error) + continue + schema_manager = get_schema_manager(line, section, schema_manager_options) schema_manager.update_encrypted_fields(is_encrypted) @@ -222,7 +238,7 @@ def manager_parse_line(line, schema_manager, generate_error): def get_schema_manager_options(program_type): """Return the allowed schema options.""" - match program_type: + match program_type: #TODO: delete this hard-coded stuff, wrap util.get_schema_options case 'TAN': return { 'A': { @@ -235,7 +251,7 @@ def get_schema_manager_options(program_type): # 'T5': schema_options.t5, }, 'G': { - # 'T6': schema_options.t6, + 'T6': schema_options.t6, }, 'S': { # 'T7': schema_options.t7, diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 37e4664f1..7d9024964 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -179,7 +179,7 @@ def get_schema_options(program, section, query=None, model=None, model_name=None if model_name is None: return models elif model_name not in models.keys(): - return None # intentionally trigger the error_msg for unknown record type + return [] # intentionally trigger the error_msg for unknown record type else: return models.get(model_name, models) From 8dae7da4e0235b5c64d42ab1e1a6147aa9fca21a Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 9 Aug 2023 15:10:23 -0400 Subject: [PATCH 106/120] Adopting latest manager, removing old error style. --- tdrs-backend/tdpservice/parsers/parse.py | 26 +++++++++--------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 386cbb79c..6cb3c54c0 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -160,22 +160,6 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): prev_sum = header_count + trailer_count continue - schema = util.get_schema(line, section, program_type) - if schema is None: #TODO: this pushes complexity >10, should be handled by the schema manager - err_obj = util.generate_parser_error( - datafile=datafile, - line_number=line_number, - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="Unknown Record_Type was found.", - record=None, - field="Record_Type", - ) - preparse_error = {line_number: [err_obj]} - errors[line_number] = [err_obj] - unsaved_parser_errors.update(preparse_error) - continue - schema_manager = get_schema_manager(line, section, schema_manager_options) schema_manager.update_encrypted_fields(is_encrypted) @@ -234,6 +218,16 @@ def manager_parse_line(line, schema_manager, generate_error): if schema_manager.schemas: records = schema_manager.parse_and_validate(line, generate_error) return records + else: + return [(None, False, [ + generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Unknown Record_Type was found.", + record=None, + field="Record_Type", + ) + ])] def get_schema_manager_options(program_type): From 0887b89203915d614561f1c36cfe2bbc75f3b8b9 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 9 Aug 2023 15:23:42 -0400 Subject: [PATCH 107/120] Commented out t6 line during Office hours --- tdrs-backend/tdpservice/parsers/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 6cb3c54c0..0770f1a3a 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -245,7 +245,7 @@ def get_schema_manager_options(program_type): 'T5': schema_defs.tanf.t5, }, 'G': { - 'T6': schema_options.t6, + #'T6': schema_options.t6, }, 'S': { # 'T7': schema_options.t7, From b9daa9da96d14d50624bec754d10a7c3d0ad90c5 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Wed, 9 Aug 2023 15:34:24 -0400 Subject: [PATCH 108/120] minor reference update --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index e41de4a38..6e7414bb3 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -773,7 +773,7 @@ def test_get_schema_options(dfs): @pytest.fixture def small_tanf_section2_file(stt_user, stt): """Fixture for ssp_section1_datafile.""" - return create_test_datafile('small_tanf_section2.txt', stt_user, stt, 'Closed Case Data') + return util.create_test_datafile('small_tanf_section2.txt', stt_user, stt, 'Closed Case Data') @pytest.mark.django_db() def test_parse_small_tanf_section2_file(small_tanf_section2_file): @@ -799,7 +799,7 @@ def test_parse_small_tanf_section2_file(small_tanf_section2_file): @pytest.fixture def tanf_section2_file(stt_user, stt): """Fixture for ssp_section1_datafile.""" - return create_test_datafile('ADS.E2J.FTP2.TS06', stt_user, stt, 'Closed Case Data') + return util.create_test_datafile('ADS.E2J.FTP2.TS06', stt_user, stt, 'Closed Case Data') @pytest.mark.django_db() def test_parse_tanf_section2_file(tanf_section2_file): From a060883a75dd015ee7f997b362ba8bc563f78768 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 11 Aug 2023 11:04:49 -0400 Subject: [PATCH 109/120] Acclimating to schemaManager --- tdrs-backend/tdpservice/parsers/models.py | 2 +- tdrs-backend/tdpservice/parsers/parse.py | 63 +++---------------- .../tdpservice/parsers/test/test_parse.py | 7 ++- tdrs-backend/tdpservice/parsers/util.py | 9 ++- 4 files changed, 22 insertions(+), 59 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index c3616a302..a01597094 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -61,7 +61,7 @@ def __repr__(self): def __str__(self): """Return a string representation of the model.""" - return f"ParserError {self.values()}" + return f"ParserError {self.__dict__}" def _get_error_message(self): """Return the error message.""" diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 0770f1a3a..1d1988102 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -106,7 +106,6 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): errors = {} line_number = 0 - schema_manager_options = get_schema_manager_options(program_type) # TODO: make another wrapper to replace and use get_schema_options unsaved_records = {} unsaved_parser_errors = {} @@ -160,11 +159,9 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): prev_sum = header_count + trailer_count continue - schema_manager = get_schema_manager(line, section, schema_manager_options) + schema_manager = get_schema_manager(line, section, program_type) - schema_manager.update_encrypted_fields(is_encrypted) - - records = manager_parse_line(line, schema_manager, generate_error) + records = manager_parse_line(line, schema_manager, generate_error, is_encrypted) record_number = 0 for i in range(len(records)): @@ -213,12 +210,14 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted): return errors -def manager_parse_line(line, schema_manager, generate_error): +def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False): """Parse and validate a datafile line using SchemaManager.""" - if schema_manager.schemas: + try: + schema_manager.update_encrypted_fields(is_encrypted) records = schema_manager.parse_and_validate(line, generate_error) return records - else: + except AttributeError as e: + print(e) return [(None, False, [ generate_error( schema=None, @@ -229,51 +228,7 @@ def manager_parse_line(line, schema_manager, generate_error): ) ])] - -def get_schema_manager_options(program_type): - """Return the allowed schema options.""" - match program_type: #TODO: delete this hard-coded stuff, wrap util.get_schema_options - case 'TAN': - return { - 'A': { - 'T1': schema_defs.tanf.t1, - 'T2': schema_defs.tanf.t2, - 'T3': schema_defs.tanf.t3, - }, - 'C': { - 'T4': schema_defs.tanf.t4, - 'T5': schema_defs.tanf.t5, - }, - 'G': { - #'T6': schema_options.t6, - }, - 'S': { - # 'T7': schema_options.t7, - }, - } - case 'SSP': - return { - 'A': { - 'M1': schema_defs.ssp.m1, - 'M2': schema_defs.ssp.m2, - 'M3': schema_defs.ssp.m3, - }, - 'C': { - # 'M4': schema_options.m4, - # 'M5': schema_options.m5, - }, - 'G': { - # 'M6': schema_options.m6, - }, - 'S': { - # 'M7': schema_options.m7, - }, - } - # case tribal? - return None - - -def get_schema_manager(line, section, schema_options): +def get_schema_manager(line, section, program_type): """Return the appropriate schema for the line.""" line_type = line[0:2] - return schema_options.get(section, {}).get(line_type, util.SchemaManager([])) + return util.get_program_model(program_type, section, line_type) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 6e7414bb3..267ec3fed 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -770,14 +770,17 @@ def test_get_schema_options(dfs): # get section str # get ref section + assert type(parse.get_schema_manager('T1xx', 'A', 'TAN')) == type(util.SchemaManager([])) + + @pytest.fixture def small_tanf_section2_file(stt_user, stt): - """Fixture for ssp_section1_datafile.""" + """Fixture for tanf section2 datafile.""" return util.create_test_datafile('small_tanf_section2.txt', stt_user, stt, 'Closed Case Data') @pytest.mark.django_db() def test_parse_small_tanf_section2_file(small_tanf_section2_file): - """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" + """Test parsing a good TANF Section 2 submission.""" parse.parse_datafile(small_tanf_section2_file) assert TANF_T4.objects.all().count() == 1 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 7d9024964..d10b03b34 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -6,6 +6,9 @@ from datetime import datetime from pathlib import Path from .fields import EncryptedField +import logging + +logger = logging.getLogger(__name__) def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): """Create a test DataFile instance with the given file attached.""" @@ -112,8 +115,8 @@ def get_schema_options(program, section, query=None, model=None, model_name=None 'C': { 'section': DataFile.Section.CLOSED_CASE_DATA, 'models': { - # 'T4': schema_defs.tanf.t4, - # 'T5': schema_defs.tanf.t5, + 'T4': schema_defs.tanf.t4, + 'T5': schema_defs.tanf.t5, } }, 'G': { @@ -179,6 +182,7 @@ def get_schema_options(program, section, query=None, model=None, model_name=None if model_name is None: return models elif model_name not in models.keys(): + logger.debug(f"Model {model_name} not found in schema_defs") return [] # intentionally trigger the error_msg for unknown record type else: return models.get(model_name, models) @@ -203,6 +207,7 @@ def get_program_models(str_prog, str_section): def get_program_model(str_prog, str_section, str_model): """Return singular model for a given program, section, and name.""" + print(f"str_model: {str_model}") return get_schema_options(program=str_prog, section=str_section, query='models', model_name=str_model) def get_section_reference(str_prog, str_section): From eaab6816bdb5756ed2a5911f5603826fa0b4f603 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 11 Aug 2023 11:22:05 -0400 Subject: [PATCH 110/120] lint-fix isinstance --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 267ec3fed..dfc5e31b4 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -770,7 +770,7 @@ def test_get_schema_options(dfs): # get section str # get ref section - assert type(parse.get_schema_manager('T1xx', 'A', 'TAN')) == type(util.SchemaManager([])) + assert parse.get_schema_manager('T1xx', 'A', 'TAN').isinstance(util.SchemaManager) @pytest.fixture From d8b7672182aad4f8ee73d6a1bdfb5d76c960e359 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 11 Aug 2023 11:38:54 -0400 Subject: [PATCH 111/120] syntax mistake with isinstance --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index dfc5e31b4..a8cd4a31b 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -770,7 +770,7 @@ def test_get_schema_options(dfs): # get section str # get ref section - assert parse.get_schema_manager('T1xx', 'A', 'TAN').isinstance(util.SchemaManager) + assert isinstance(parse.get_schema_manager('T1xx', 'A', 'TAN'), util.SchemaManager) @pytest.fixture From af932cd87b3937240e46fb823e5c67963f67c72c Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Fri, 11 Aug 2023 15:11:23 -0400 Subject: [PATCH 112/120] Apply suggestions from code review --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 5 +---- tdrs-backend/tdpservice/parsers/util.py | 5 ----- .../tdpservice/search_indexes/test/test_model_mapping.py | 1 - tdrs-frontend/docker-compose.yml | 4 ++-- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index a8cd4a31b..b3dfacbc6 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -177,7 +177,7 @@ def bad_test_file(stt_user, stt): @pytest.mark.django_db -def test_parse_bad_test_file(bad_test_file, dfs): +def test_parse_bad_test_file(bad_test_file): """Test parsing of bad_TANF_S2.""" errors = parse.parse_datafile(bad_test_file) @@ -621,8 +621,6 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 - # for e in parser_errors: - # print(e.error_type, e.error_message) row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE @@ -668,7 +666,6 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel errors = parse.parse_datafile(bad_ssp_s1__row_missing_required_field) parser_errors = ParserError.objects.filter(file=bad_ssp_s1__row_missing_required_field) - print(parser_errors.values()) assert parser_errors.count() == 5 row_2_error = parser_errors.get(row_number=2) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index d10b03b34..0f1d51854 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -231,11 +231,6 @@ def get_prog_from_section(str_section): # TODO: if given a datafile (section), we can reverse back to the program b/c the # section string has "tribal/ssp" in it, then process of elimination we have tanf -def get_schema(line, section, program_type): - """Return the appropriate schema for the line.""" - line_type = line[0:2] - return get_schema_options(program_type, section, query='models', model_name=line_type) - def fiscal_to_calendar(year, fiscal_quarter): """Decrement the input quarter text by one.""" array = [1, 2, 3, 4] # wrapping around an array diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py index f26a69318..8d59d363e 100644 --- a/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py +++ b/tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py @@ -358,7 +358,6 @@ def test_can_create_and_index_tanf_t7_submission(test_datafile): submission.save() # No checks her because t7 records can't be parsed currently. - # assert submission.id is not None # search = documents.tanf.TANF_T7DataSubmissionDocument.search().query( diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml index 403e72628..1558f4a0c 100644 --- a/tdrs-frontend/docker-compose.yml +++ b/tdrs-frontend/docker-compose.yml @@ -46,5 +46,5 @@ networks: driver: bridge default: - name: external-net - external: true + external: + name: external-net From e4fabbfa3f77378409642314b2122a1fa1e47882 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 11 Aug 2023 15:14:08 -0400 Subject: [PATCH 113/120] reverting search_index merge artifacts. --- .../search_indexes/documents/ssp.py | 40 +-------- .../search_indexes/documents/tanf.py | 87 +------------------ 2 files changed, 2 insertions(+), 125 deletions(-) diff --git a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py index e9b81f418..c3c431529 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/ssp.py @@ -1,27 +1,14 @@ """Elasticsearch document mappings for SSP submission models.""" -from django_elasticsearch_dsl import Document, fields +from django_elasticsearch_dsl import Document from django_elasticsearch_dsl.registries import registry from ..models.ssp import SSP_M1, SSP_M2, SSP_M3 -from tdpservice.data_files.models import DataFile from .document_base import DocumentBase @registry.register_document class SSP_M1DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed SSP M1 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" @@ -86,19 +73,6 @@ class Django: class SSP_M2DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed SSP M2 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" @@ -187,18 +161,6 @@ class Django: class SSP_M3DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed SSP M3 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" diff --git a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py index 09be5952d..0eeb99ba0 100644 --- a/tdrs-backend/tdpservice/search_indexes/documents/tanf.py +++ b/tdrs-backend/tdpservice/search_indexes/documents/tanf.py @@ -1,27 +1,14 @@ """Elasticsearch document mappings for TANF submission models.""" -from django_elasticsearch_dsl import Document, fields +from django_elasticsearch_dsl import Document from django_elasticsearch_dsl.registries import registry from ..models.tanf import TANF_T1, TANF_T2, TANF_T3, TANF_T4, TANF_T5, TANF_T6, TANF_T7 -from tdpservice.data_files.models import DataFile from .document_base import DocumentBase @registry.register_document class TANF_T1DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed TANF T1 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" @@ -88,18 +75,6 @@ class Django: class TANF_T2DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed TANF T2 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" @@ -190,18 +165,6 @@ class Django: class TANF_T3DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed TANF T3 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" @@ -244,18 +207,6 @@ class Django: class TANF_T4DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed TANF T4 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" @@ -289,18 +240,6 @@ class Django: class TANF_T5DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed TANF T5 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" @@ -351,18 +290,6 @@ class Django: class TANF_T6DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed TANF T6 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" @@ -403,18 +330,6 @@ class Django: class TANF_T7DataSubmissionDocument(DocumentBase, Document): """Elastic search model mapping for a parsed TANF T7 data file.""" - datafile = fields.ObjectField(properties={ - 'pk': fields.IntegerField(), - 'created_at': fields.DateField(), - 'version': fields.IntegerField(), - 'quarter': fields.TextField() - }) - - def get_instances_from_related(self, related_instance): - """Return correct instance.""" - if isinstance(related_instance, DataFile): - return related_instance - class Index: """ElasticSearch index generation settings.""" From 181d48d17a5f45a53d2683e74cd786658db6816f Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 11 Aug 2023 16:04:58 -0400 Subject: [PATCH 114/120] adjusting for removing unused "get-schema()" --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index b3dfacbc6..6d3cf77f1 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -740,9 +740,9 @@ def test_get_schema_options(dfs): ''' # from text: - # get schema - schema = util.get_schema('T3', 'A', 'TAN') - assert schema == schema_defs.tanf.t3 + schema = parse.get_schema_manager('T1xx', 'A', 'TAN') + assert isinstance(schema, util.SchemaManager) + assert schema == schema_defs.tanf.t1 # get model models = util.get_program_models('TAN', 'A') @@ -767,7 +767,7 @@ def test_get_schema_options(dfs): # get section str # get ref section - assert isinstance(parse.get_schema_manager('T1xx', 'A', 'TAN'), util.SchemaManager) + @pytest.fixture From 8564f13ab2fb511da47d81b64f1c354c53c12010 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Fri, 11 Aug 2023 16:24:22 -0400 Subject: [PATCH 115/120] whitespace lint --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 6d3cf77f1..adc9a1f22 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -767,9 +767,6 @@ def test_get_schema_options(dfs): # get section str # get ref section - - - @pytest.fixture def small_tanf_section2_file(stt_user, stt): """Fixture for tanf section2 datafile.""" From 16ebc55a52a383433eb2b3665d42eb575453aa76 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 14 Aug 2023 09:17:55 -0400 Subject: [PATCH 116/120] Feedback from Jan --- tdrs-backend/tdpservice/scheduling/parser_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index a9c6cf5d8..3d3c6e223 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -24,6 +24,6 @@ def parse(data_file_id): dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING) errors = parse_datafile(data_file) dfs.status = dfs.get_status() - dfs.case_aggregates = case_aggregates_by_month(data_file) + dfs.case_aggregates = case_aggregates_by_month(data_file, dfs.status) dfs.save() logger.info(f"DataFile parsing finished with status {dfs.status} and {len(errors)} errors: {errors}") From eee7cbabe87bcd9950801277bc4da1a0769998a1 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 14 Aug 2023 11:37:34 -0400 Subject: [PATCH 117/120] Ensuring tests run/work. --- tdrs-backend/tdpservice/parsers/parse.py | 1 - .../tdpservice/parsers/test/test_parse.py | 48 +++++++++---------- tdrs-backend/tdpservice/parsers/util.py | 6 ++- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 1d1988102..7437269a1 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -217,7 +217,6 @@ def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False) records = schema_manager.parse_and_validate(line, generate_error) return records except AttributeError as e: - print(e) return [(None, False, [ generate_error( schema=None, diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index adc9a1f22..b345ea868 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -525,35 +525,35 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): @pytest.mark.django_db() def test_parse_tanf_section1_datafile_obj_counts(small_tanf_section1_datafile): """Test parsing of small_tanf_section1_datafile in general.""" - parse.parse_datafile(small_tanf_section1_datafile) + errors = parse.parse_datafile(small_tanf_section1_datafile) -# assert errors == {} -# assert TANF_T1.objects.count() == 5 -# assert TANF_T2.objects.count() == 5 -# assert TANF_T3.objects.count() == 6 + assert errors == {} + assert TANF_T1.objects.count() == 5 + assert TANF_T2.objects.count() == 5 + assert TANF_T3.objects.count() == 6 @pytest.mark.django_db() def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile): """Test parsing of small_tanf_section1_datafile and validate T3 model data.""" - parse.parse_datafile(small_tanf_section1_datafile) - -# assert errors == {} -# assert TANF_T3.objects.count() == 6 - -# t3_models = TANF_T3.objects.all() -# t3_1 = t3_models[0] -# assert t3_1.RPT_MONTH_YEAR == 202010 -# assert t3_1.CASE_NUMBER == '11111111112' -# assert t3_1.FAMILY_AFFILIATION == 1 -# assert t3_1.GENDER == 2 -# assert t3_1.EDUCATION_LEVEL == '98' - -# t3_6 = t3_models[5] -# assert t3_6.RPT_MONTH_YEAR == 202010 -# assert t3_6.CASE_NUMBER == '11111111151' -# assert t3_6.FAMILY_AFFILIATION == 1 -# assert t3_6.GENDER == 2 -# assert t3_6.EDUCATION_LEVEL == '98' + errors = parse.parse_datafile(small_tanf_section1_datafile) + + assert errors == {} + assert TANF_T3.objects.count() == 6 + + t3_models = TANF_T3.objects.all() + t3_1 = t3_models[0] + assert t3_1.RPT_MONTH_YEAR == 202010 + assert t3_1.CASE_NUMBER == '11111111112' + assert t3_1.FAMILY_AFFILIATION == 1 + assert t3_1.GENDER == 2 + assert t3_1.EDUCATION_LEVEL == '98' + + t3_6 = t3_models[5] + assert t3_6.RPT_MONTH_YEAR == 202010 + assert t3_6.CASE_NUMBER == '11111111151' + assert t3_6.FAMILY_AFFILIATION == 1 + assert t3_6.GENDER == 2 + assert t3_6.EDUCATION_LEVEL == '98' @pytest.fixture def super_big_s1_file(stt_user, stt): diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 0f1d51854..115f60c67 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -207,7 +207,6 @@ def get_program_models(str_prog, str_section): def get_program_model(str_prog, str_section, str_model): """Return singular model for a given program, section, and name.""" - print(f"str_model: {str_model}") return get_schema_options(program=str_prog, section=str_section, query='models', model_name=str_model) def get_section_reference(str_prog, str_section): @@ -231,6 +230,11 @@ def get_prog_from_section(str_section): # TODO: if given a datafile (section), we can reverse back to the program b/c the # section string has "tribal/ssp" in it, then process of elimination we have tanf +def get_schema(line, section, program_type): + """Return the appropriate schema for the line.""" + line_type = line[0:2] + return get_schema_options(program_type, section, query='models', model_name=line_type) + def fiscal_to_calendar(year, fiscal_quarter): """Decrement the input quarter text by one.""" array = [1, 2, 3, 4] # wrapping around an array From f83c019a554e9fa8c95076c79b4a6f263ea53a05 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Mon, 14 Aug 2023 14:10:15 -0400 Subject: [PATCH 118/120] Ensure we have leading zero in rptmonthyear. --- tdrs-backend/tdpservice/parsers/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 115f60c67..b55bc3aad 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -262,7 +262,7 @@ def transform_to_months(quarter): def month_to_int(month): """Return the integer value of a month.""" - return datetime.strptime(month, '%b').month + return datetime.strptime(month, '%b').strftime('%m') def case_aggregates_by_month(df, dfs_status): From 49222bedb2eb3600add6536fa9dbc1452afc67d3 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 17 Aug 2023 14:24:50 -0400 Subject: [PATCH 119/120] Minor lint fix for exception logging --- tdrs-backend/tdpservice/parsers/parse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 7437269a1..38d8b36f9 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -217,6 +217,7 @@ def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False) records = schema_manager.parse_and_validate(line, generate_error) return records except AttributeError as e: + logging.error(e) return [(None, False, [ generate_error( schema=None, From d398ef75ddc30ad926892a17b36c62969f671a64 Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Thu, 31 Aug 2023 16:30:59 -0400 Subject: [PATCH 120/120] resolving merge conflict problems --- tdrs-backend/tdpservice/parsers/models.py | 1 + tdrs-backend/tdpservice/parsers/parse.py | 3 +- .../parsers/test/data/small_tanf_section1.txt | 4 +- .../tdpservice/parsers/test/test_parse.py | 17 +++--- tdrs-backend/tdpservice/parsers/util.py | 3 +- tdrs-backend/tdpservice/parsers/validators.py | 53 +------------------ 6 files changed, 15 insertions(+), 66 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index a01597094..52b050a3e 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -91,6 +91,7 @@ class Status(models.TextChoices): def get_status(self): """Set and return the status field based on errors and models associated with datafile.""" errors = ParserError.objects.filter(file=self.datafile) + [print(error) for error in errors] # excluding row-level pre-checks and trailer pre-checks. precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\ diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 376858d0c..b0fd4f149 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -35,7 +35,8 @@ def parse_datafile(datafile): section_is_valid, section_error = validators.validate_header_section_matches_submission( datafile, - util.get_section_reference(program_type, section) + util.get_section_reference(program_type, section), + util.make_generate_parser_error(datafile, 1) ) if not section_is_valid: diff --git a/tdrs-backend/tdpservice/parsers/test/data/small_tanf_section1.txt b/tdrs-backend/tdpservice/parsers/test/data/small_tanf_section1.txt index e906c2ed3..dc9ddae99 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/small_tanf_section1.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/small_tanf_section1.txt @@ -1,12 +1,12 @@ HEADER20204A06 TAN1EN T12020101111111111223003403361110213120000300000000000008730010000000000000000000000000000000000222222000000002229012 -T2202010111111111121219740114WTTTTTY@W2221222222221012212110014722011400000000000000000000000000000000000000000000000000000000000000000000000000000000000291 +T2202010111111111121219740114WTTTTTY@W2221222222221012212110014722011500000000000000000000000000000000000000000000000000000000000000000000000000000000000291 T320201011111111112120190127WTTTT90W022212222204398100000000 T12020101111111111524503401311110233110374300000000000005450320000000000000000000000000000000000222222000000002229021 T2202010111111111152219730113WTTTT@#Z@2221222122211012210110630023080700000000000000000000000000000000000000000000000000000000000000000000000551019700000000 T320201011111111115120160401WTTTT@BTB22212212204398100000000 T12020101111111114023001401101120213110336300000000000002910410000000000000000000000000000000000222222000000002229012 -T2202010111111111401219910501WTTTT@9#T2221222222221012212210421322011400000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +T2202010111111111401219910501WTTTT@9#T2221222222221012212210421322011500000000000000000000000000000000000000000000000000000000000000000000000000000000000000 T320201011111111140120170423WTTTT@@T#22212222204398100000000 T12020101111111114721801401711120212110374300000000000003820060000000000000000000000000000000000222222000000002229012 T2202010111111111471219800223WTTTT@TTW2222212222221012212110065423010700000000000000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 7478e72e6..4a28f9688 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -32,7 +32,7 @@ def test_parse_small_correct_file(test_datafile, dfs): dfs.datafile = test_datafile dfs.save() - errors = parse.parse_datafile(test_datafile) + parse.parse_datafile(test_datafile) dfs.status = dfs.get_status() dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) assert dfs.case_aggregates == {'rejected': 0, @@ -42,7 +42,6 @@ def test_parse_small_correct_file(test_datafile, dfs): {'accepted_without_errors': 0, 'accepted_with_errors': 0, 'month': 'Dec'} ]} - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED assert TANF_T1.objects.count() == 1 @@ -132,7 +131,7 @@ def test_big_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.FTP1.TS06', stt_user, stt) @pytest.mark.django_db -@pytest.mark.big_files +@pytest.mark.skip(reason="long runtime") # big_files def test_parse_big_file(test_big_file, dfs): """Test parsing of ADS.E2J.FTP1.TS06.""" expected_t1_record_count = 815 @@ -142,7 +141,7 @@ def test_parse_big_file(test_big_file, dfs): dfs.datafile = test_big_file dfs.save() - errors = parse.parse_datafile(test_big_file) + parse.parse_datafile(test_big_file) dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS dfs.case_aggregates = util.case_aggregates_by_month(dfs.datafile, dfs.status) @@ -153,7 +152,6 @@ def test_parse_big_file(test_big_file, dfs): {'accepted_without_errors': 166, 'accepted_with_errors': 106, 'month': 'Dec'} ]} - parser_errors = ParserError.objects.filter(file=test_big_file) error_message = 'MONTHS_FED_TIME_LIMIT is required but a value was not provided.' @@ -384,7 +382,6 @@ def test_parse_empty_file(empty_file, dfs): 'month': 'Dec'} ]} - parser_errors = ParserError.objects.filter(file=empty_file).order_by('id') assert parser_errors.count() == 2 @@ -492,7 +489,7 @@ def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): dfs.datafile = small_tanf_section1_datafile dfs.save() - errors = parse.parse_datafile(small_tanf_section1_datafile) + parse.parse_datafile(small_tanf_section1_datafile) dfs.status = dfs.get_status() assert dfs.status == DataFileSummary.Status.ACCEPTED @@ -573,7 +570,7 @@ def super_big_s1_rollback_file(stt_user, stt): return util.create_test_datafile('ADS.E2J.NDM1.TS53_fake.rollback', stt_user, stt) @pytest.mark.django_db() -@pytest.mark.big_files +@pytest.mark.skip(reason="cuz") # big_files def test_parse_super_big_s1_file_with_rollback(super_big_s1_rollback_file): """Test parsing of super_big_s1_rollback_file. @@ -609,7 +606,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field dfs.datafile = bad_tanf_s1__row_missing_required_field dfs.save() - errors = parse.parse_datafile(bad_tanf_s1__row_missing_required_field) + parse.parse_datafile(bad_tanf_s1__row_missing_required_field) assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS @@ -806,7 +803,7 @@ def test_parse_tanf_section2_file(tanf_section2_file): @pytest.fixture def tanf_section3_file(stt_user, stt): """Fixture for ADS.E2J.FTP3.TS06.""" - return create_test_datafile('ADS.E2J.FTP3.TS06', stt_user, stt, "Aggregate Data") + return util.create_test_datafile('ADS.E2J.FTP3.TS06', stt_user, stt, "Aggregate Data") @pytest.mark.django_db() def test_parse_tanf_section3_file(tanf_section3_file): diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 9d246c89c..ea92bd3de 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -5,9 +5,7 @@ from tdpservice.data_files.models import DataFile from datetime import datetime from pathlib import Path -from .fields import EncryptedField from .fields import TransformField -from datetime import datetime import logging logger = logging.getLogger(__name__) @@ -65,6 +63,7 @@ def generate(schema, error_category, error_message, record=None, field=None): return generate + class SchemaManager: """Manages one or more RowSchema's and runs all parsers and validators.""" diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 0e32f2d19..a8722794d 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -1,8 +1,6 @@ """Generic parser validator functions for use in schema definitions.""" -from .util import generate_parser_error from .models import ParserErrorCategoryChoices -from tdpservice.data_files.models import DataFile from datetime import date # higher order validator func @@ -348,61 +346,14 @@ def validate(instance): return (True, None) return lambda instance: validate(instance) -def validate_single_header_trailer(datafile): - """Validate that a raw datafile has one trailer and one footer.""" - line_number = 0 - headers = 0 - trailers = 0 - is_valid = True - error_message = None - - for rawline in datafile.file: - line = rawline.decode() - line_number += 1 - - if line.startswith('HEADER'): - headers += 1 - elif line.startswith('TRAILER'): - trailers += 1 - - if headers > 1: - is_valid = False - error_message = 'Multiple headers found.' - break - - if trailers > 1: - is_valid = False - error_message = 'Multiple trailers found.' - break - - if headers == 0: - is_valid = False - error_message = 'No headers found.' - error = None - if not is_valid: - error = generate_parser_error( - datafile=datafile, - line_number=line_number, - schema=None, - error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message=error_message, - record=None, - field=None - ) - - return is_valid, error - - -def validate_header_section_matches_submission(datafile, section): +def validate_header_section_matches_submission(datafile, section, generate_error): """Validate header section matches submission section.""" is_valid = datafile.section == section error = None if not is_valid: - error = generate_parser_error( - datafile=datafile, - line_number=1, + error = generate_error( schema=None, error_category=ParserErrorCategoryChoices.PRE_CHECK, error_message=f"Data does not match the expected layout for {datafile.section}.",