From 75a2d3359e2f7b250a43746d3550e616abdec657 Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 10 Sep 2023 19:16:47 -0700 Subject: [PATCH] fix: resolved several gotchas with balance assertion dates with ofx files --- .../libreader/csvreader.py | 2 +- .../libreader/ofxreader.py | 60 ++++++++++++++----- .../smart/transactions.qfx | 4 +- .../smart/transactions.qfx.extract | 2 +- .../smart/transactions.qfx.file_date | 2 +- 5 files changed, 51 insertions(+), 19 deletions(-) diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 825583e..c03974a 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -229,7 +229,7 @@ def get_max_transaction_date(self): except Exception as err: print("ERROR: no end_date. SKIPPING input.") traceback.print_tb(err.__traceback__) - return False + return None return date diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index 7f2c263..76ea929 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -76,26 +76,56 @@ def get_available_cash(self, settlement_fund_balance=0): return available_cash - settlement_fund_balance return None - def get_ofx_end_date(self): - end_date = self.ofx_account.statement.end_date - # convert end_date from utc to local timezone - end_date = end_date.replace(tzinfo=datetime.timezone.utc).astimezone().date() - return end_date + def get_ofx_end_date(self, field='end_date'): + end_date = getattr(self.ofx_account.statement, field, None) + + # Convert end_date from utc to local timezone if needed and return + # This is needed only if there is an actual timestamp other than time(0, 0) + if end_date: + if end_date.time() != datetime.time(0, 0): + end_date = end_date.replace(tzinfo=datetime.timezone.utc).astimezone() + return end_date.date() + + return None def get_smart_date(self): - """ We find the statement's end date from the OFX file. However, banks and credit cards + """ We want the latest date we can assert balance on. Let's consider all the dates we have: + b--------e-------(s-2)----(s)----(d) + + - b: date of first transaction in this ofx file (end_date) + - e: date of last transaction in this ofx file (max_transaction_date) + - s: statement's end date as claimed by the ofx import fileinput (acc.statement.available_balance_date) + - d: date of download + + Ideally, we would assert balance end of the day on (s) above. However, banks and credit cards typically have pending transactions that are not included in downloads. When we download the next statement, new transactions may appear prior to the balance assertion date that we generate for this statement. To attempt to avoid this, we set the balance assertion date to - either two days before the statement's end date or the last transaction's date, whichever - is later. + (s-2), where 2 is the fudge factor (the time where pending transactions not seen in this + statement might occur in the next statement). + + Not all ofx files have all these dates. Hence we compute this date with the best info we + have. """ - end_date = self.get_ofx_end_date() - end_date -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) + ofx_max_transation_date = self.get_ofx_end_date('end_date') + ofx_balance_date1 = self.get_ofx_end_date('available_balance_date') + ofx_balance_date2 = self.get_ofx_end_date('balance_date') max_transaction_date = self.get_max_transaction_date() - max_transaction_date = max_transaction_date if max_transaction_date else datetime.date.min - return_date = max(end_date, max_transaction_date) + + if ofx_balance_date1: + ofx_balance_date1 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) + if ofx_balance_date2: + ofx_balance_date2 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) + + dates = [ofx_max_transation_date, max_transaction_date, ofx_balance_date1, ofx_balance_date2] + if all(v is None for v in dates[:2]): # because ofx_balance_date appears even for closed accounts + return None + + def vd(x): return x if x else datetime.date.min + return_date = max(*[vd(x) for x in dates]) + print("Smart date computation. Dates were: ", dates) + return return_date def get_balance_assertion_date(self): @@ -119,6 +149,8 @@ def get_balance_assertion_date(self): date_type = self.config.get('balance_assertion_date_type', 'smart') return_date = date_type_map[date_type]() + if not return_date: + return None return return_date + datetime.timedelta(days=1) # Next day, as defined by Beancount def get_max_transaction_date(self): @@ -135,7 +167,7 @@ def get_max_transaction_date(self): date = max(ot.tradeDate if hasattr(ot, 'tradeDate') else ot.date for ot in self.get_transactions()).date() except TypeError: - return False + return None except ValueError: - return False + return None return date diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx index 98158b6..0b95265 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx @@ -46,7 +46,7 @@ CHECKING 20190503000000 -20190901000000 +20190109000000 @@ -81,7 +81,7 @@ Check 23400 150.65 -20150101000000.000[+0:UTC] +20190120000000.000[+0:UTC] diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract index ea7098d..bdec785 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract @@ -5,4 +5,4 @@ 2019-01-09 * "Check 23400" Assets:Banks:Checking -1000 USD -2019-08-30 balance Assets:Banks:Checking 150.65 USD +2019-01-19 balance Assets:Banks:Checking 150.65 USD diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date index 5a9abd6..8d1dde9 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date @@ -1 +1 @@ -2019-09-01T00:00:00 +2019-01-09T00:00:00