Skip to content

Commit

Permalink
feat: validate the fund code when importing CANs
Browse files Browse the repository at this point in the history 
Signed-off-by: John DeAngelis <[email protected]>
  • Loading branch information
@johndeange
johndeange committed Nov 13, 2024
1 parent 01ae45e commit ae69330
Show file tree
Hide file tree
Showing 2 changed files with 241 additions and 42 deletions.
121 changes: 80 additions & 41 deletions backend/data_tools/src/load_cans/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
from csv import DictReader
from dataclasses import dataclass, field
from dataclasses import dataclass
from typing import List

from loguru import logger
from sqlalchemy import and_, select
from sqlalchemy.orm import Session

from models import CAN, BaseModel, CANFundingDetails, CANFundingSource, CANMethodOfTransfer, Portfolio, User
from models import CAN, CANFundingDetails, CANFundingSource, CANMethodOfTransfer, Portfolio, User


@dataclass
Expand Down Expand Up @@ -120,52 +120,91 @@ def create_models(data: CANData, sys_user: User, session: Session) -> None:

can.portfolio = portfolio

# get or create funding details
fiscal_year = int(data.FUND[6:10])
fund_code = data.FUND
allowance = data.ALLOWANCE
sub_allowance = data.SUB_ALLOWANCE
allotment = data.ALLOTMENT_ORG
appropriation = "-".join([data.APPROP_PREFIX or "", data.APPROP_YEAR[0:2] or "", data.APPROP_POSTFIX or ""])
method_of_transfer = CANMethodOfTransfer[data.METHOD_OF_TRANSFER]
funding_source = CANFundingSource[data.FUNDING_SOURCE]

existing_funding_details = session.execute(select(CANFundingDetails).where(
and_(
CANFundingDetails.fiscal_year == fiscal_year,
CANFundingDetails.fund_code == fund_code,
CANFundingDetails.allowance == allowance,
CANFundingDetails.sub_allowance == sub_allowance,
CANFundingDetails.allotment == allotment,
CANFundingDetails.appropriation == appropriation,
CANFundingDetails.method_of_transfer == method_of_transfer,
CANFundingDetails.funding_source == funding_source,
))).scalar_one_or_none()

if not existing_funding_details:
funding_details = CANFundingDetails(
fiscal_year=fiscal_year,
fund_code=fund_code,
allowance=allowance,
sub_allowance=sub_allowance,
allotment=allotment,
appropriation=appropriation,
method_of_transfer=method_of_transfer,
funding_source=funding_source,
created_by=sys_user.id,
)
session.add(funding_details)
session.commit()
can.funding_details = funding_details
else:
can.funding_details = existing_funding_details
try:
validate_fund_code(data)
except ValueError as e:
logger.info(f"Skipping creating funding details for {data} due to invalid fund code. {e}")

can.funding_details = get_or_create_funding_details(data, sys_user, session)
session.merge(can)
session.commit()
except Exception as e:
logger.error(f"Error creating models for {data}")
raise e


def get_or_create_funding_details(data: CANData, sys_user: User, session: Session) -> CANFundingDetails:
"""
Get or create a CANFundingDetails instance.
:param data: The CANData instance to use.
:param sys_user: The system user to use.
:param session: The database session to use.
:return: A CANFundingDetails instance.
"""
fiscal_year = int(data.FUND[6:10])
fund_code = data.FUND
allowance = data.ALLOWANCE
sub_allowance = data.SUB_ALLOWANCE
allotment = data.ALLOTMENT_ORG
appropriation = "-".join([data.APPROP_PREFIX or "", data.APPROP_YEAR[0:2] or "", data.APPROP_POSTFIX or ""])
method_of_transfer = CANMethodOfTransfer[data.METHOD_OF_TRANSFER]
funding_source = CANFundingSource[data.FUNDING_SOURCE]
existing_funding_details = session.execute(select(CANFundingDetails).where(
and_(
CANFundingDetails.fiscal_year == fiscal_year,
CANFundingDetails.fund_code == fund_code,
CANFundingDetails.allowance == allowance,
CANFundingDetails.sub_allowance == sub_allowance,
CANFundingDetails.allotment == allotment,
CANFundingDetails.appropriation == appropriation,
CANFundingDetails.method_of_transfer == method_of_transfer,
CANFundingDetails.funding_source == funding_source,
))).scalar_one_or_none()
if not existing_funding_details:
funding_details = CANFundingDetails(
fiscal_year=fiscal_year,
fund_code=fund_code,
allowance=allowance,
sub_allowance=sub_allowance,
allotment=allotment,
appropriation=appropriation,
method_of_transfer=method_of_transfer,
funding_source=funding_source,
created_by=sys_user.id,
)
return funding_details
else:
return existing_funding_details


def validate_fund_code(data: CANData) -> None:
"""
Validate the fund code in a CANData instance.
:param data: The CANData instance to validate.
:return: None
:raises ValueError: If the fund code is invalid.
"""
if len(data.FUND) != 14:
raise ValueError(f"Invalid fund code length {data.FUND}")
int(data.FUND[6:10])
length_of_appropriation = data.FUND[10]
if length_of_appropriation not in ["0", "1", "5"]:
raise ValueError(f"Invalid length of appropriation {length_of_appropriation}")
direct_or_reimbursable = data.FUND[11]
if direct_or_reimbursable not in ["D", "R"]:
raise ValueError(f"Invalid direct or reimbursable {direct_or_reimbursable}")
category = data.FUND[12]
if category not in ["A", "B", "C"]:
raise ValueError(f"Invalid category {category}")
discretionary_or_mandatory = data.FUND[13]
if discretionary_or_mandatory not in ["D", "M"]:
raise ValueError(f"Invalid discretionary or mandatory {discretionary_or_mandatory}")


def create_all_models(data: List[CANData], sys_user: User, session: Session) -> None:
"""
Convert a list of CanData instances to a list of BaseModel instances.
Expand Down
162 changes: 161 additions & 1 deletion backend/data_tools/tests/load_cans/test_load_cans.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,14 @@
from data_tools.src.common.utils import get_or_create_sys_user
from data_tools.src.import_static_data.import_data import get_config
from data_tools.src.load_cans.main import main
from data_tools.src.load_cans.utils import CANData, create_can_data, create_models, validate_all, validate_data
from data_tools.src.load_cans.utils import (
CANData,
create_can_data,
create_models,
validate_all,
validate_data,
validate_fund_code,
)
from sqlalchemy import and_, text

from models import * # noqa: F403, F401
Expand Down Expand Up @@ -370,3 +377,156 @@ def test_create_models_upsert(db_with_portfolios):
db_with_portfolios.execute(text("DELETE FROM can_funding_details_version"))
db_with_portfolios.execute(text("DELETE FROM ops_db_history"))
db_with_portfolios.execute(text("DELETE FROM ops_db_history_version"))

def test_validate_fund_code():
data = CANData(
SYS_CAN_ID=500,
CAN_NBR="G99HRF2",
CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
FUND="AAXXXX20231DAD",
ALLOWANCE="0000000001",
ALLOTMENT_ORG="YZC6S1JUGUN",
SUB_ALLOWANCE="9KRZ2ND",
CURRENT_FY_FUNDING_YTD=880000.0,
APPROP_PREFIX="XX",
APPROP_POSTFIX="XXXX",
APPROP_YEAR="23",
PORTFOLIO="HMRF",
FUNDING_SOURCE="OPRE",
METHOD_OF_TRANSFER="DIRECT",
NICK_NAME="HMRF-OPRE",
)
validate_fund_code(data)

def test_validate_fund_code_length():
data = CANData(
SYS_CAN_ID=500,
CAN_NBR="G99HRF2",
CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
FUND="AAXXXX20231DADDDDDDD",
ALLOWANCE="0000000001",
ALLOTMENT_ORG="YZC6S1JUGUN",
SUB_ALLOWANCE="9KRZ2ND",
CURRENT_FY_FUNDING_YTD=880000.0,
APPROP_PREFIX="XX",
APPROP_POSTFIX="XXXX",
APPROP_YEAR="23",
PORTFOLIO="HMRF",
FUNDING_SOURCE="OPRE",
METHOD_OF_TRANSFER="DIRECT",
NICK_NAME="HMRF-OPRE",
)
with pytest.raises(ValueError) as e_info:
validate_fund_code(data)
assert e_info.value.args[0] == "Invalid fund code length AAXXXX20231DADDDDDDD"

def test_validate_fund_code_fy():
data = CANData(
SYS_CAN_ID=500,
CAN_NBR="G99HRF2",
CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
FUND="AAXXXX20FY1DAD",
ALLOWANCE="0000000001",
ALLOTMENT_ORG="YZC6S1JUGUN",
SUB_ALLOWANCE="9KRZ2ND",
CURRENT_FY_FUNDING_YTD=880000.0,
APPROP_PREFIX="XX",
APPROP_POSTFIX="XXXX",
APPROP_YEAR="23",
PORTFOLIO="HMRF",
FUNDING_SOURCE="OPRE",
METHOD_OF_TRANSFER="DIRECT",
NICK_NAME="HMRF-OPRE",
)
with pytest.raises(ValueError) as e_info:
validate_fund_code(data)
assert e_info.value.args[0] == "invalid literal for int() with base 10: '20FY'"

def test_validate_fund_code_length_of_appropriation():
data = CANData(
SYS_CAN_ID=500,
CAN_NBR="G99HRF2",
CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
FUND="AAXXXX20233DAD",
ALLOWANCE="0000000001",
ALLOTMENT_ORG="YZC6S1JUGUN",
SUB_ALLOWANCE="9KRZ2ND",
CURRENT_FY_FUNDING_YTD=880000.0,
APPROP_PREFIX="XX",
APPROP_POSTFIX="XXXX",
APPROP_YEAR="23",
PORTFOLIO="HMRF",
FUNDING_SOURCE="OPRE",
METHOD_OF_TRANSFER="DIRECT",
NICK_NAME="HMRF-OPRE",
)
with pytest.raises(ValueError) as e_info:
validate_fund_code(data)
assert e_info.value.args[0] == "Invalid length of appropriation 3"


def test_validate_fund_code_direct_or_reimbursable():
data = CANData(
SYS_CAN_ID=500,
CAN_NBR="G99HRF2",
CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
FUND="AAXXXX20231OAD",
ALLOWANCE="0000000001",
ALLOTMENT_ORG="YZC6S1JUGUN",
SUB_ALLOWANCE="9KRZ2ND",
CURRENT_FY_FUNDING_YTD=880000.0,
APPROP_PREFIX="XX",
APPROP_POSTFIX="XXXX",
APPROP_YEAR="23",
PORTFOLIO="HMRF",
FUNDING_SOURCE="OPRE",
METHOD_OF_TRANSFER="DIRECT",
NICK_NAME="HMRF-OPRE",
)
with pytest.raises(ValueError) as e_info:
validate_fund_code(data)
assert e_info.value.args[0] == "Invalid direct or reimbursable O"

def test_validate_fund_code_category():
data = CANData(
SYS_CAN_ID=500,
CAN_NBR="G99HRF2",
CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
FUND="AAXXXX20231DDD",
ALLOWANCE="0000000001",
ALLOTMENT_ORG="YZC6S1JUGUN",
SUB_ALLOWANCE="9KRZ2ND",
CURRENT_FY_FUNDING_YTD=880000.0,
APPROP_PREFIX="XX",
APPROP_POSTFIX="XXXX",
APPROP_YEAR="23",
PORTFOLIO="HMRF",
FUNDING_SOURCE="OPRE",
METHOD_OF_TRANSFER="DIRECT",
NICK_NAME="HMRF-OPRE",
)
with pytest.raises(ValueError) as e_info:
validate_fund_code(data)
assert e_info.value.args[0] == "Invalid category D"

def test_validate_fund_code_discretionary_or_mandatory():
data = CANData(
SYS_CAN_ID=500,
CAN_NBR="G99HRF2",
CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
FUND="AAXXXX20231DAR",
ALLOWANCE="0000000001",
ALLOTMENT_ORG="YZC6S1JUGUN",
SUB_ALLOWANCE="9KRZ2ND",
CURRENT_FY_FUNDING_YTD=880000.0,
APPROP_PREFIX="XX",
APPROP_POSTFIX="XXXX",
APPROP_YEAR="23",
PORTFOLIO="HMRF",
FUNDING_SOURCE="OPRE",
METHOD_OF_TRANSFER="DIRECT",
NICK_NAME="HMRF-OPRE",
)
with pytest.raises(ValueError) as e_info:
validate_fund_code(data)
assert e_info.value.args[0] == "Invalid discretionary or mandatory R"

0 comments on commit ae69330

Please sign in to comment.