From 3b572298586ff3cb092896ec88ab930e98d711c1 Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 9 Sep 2023 12:29:14 -0700 Subject: [PATCH] feat: configurable balance assertion dates --- .../importers/schwab/schwab_csv_checking.py | 6 +- .../importers/stanchart/scbbank.py | 7 +- .../importers/stanchart/scbcard.py | 7 +- .../importers/unitedoverseas/uobbank.py | 7 +- .../importers/unitedoverseas/uobcard.py | 8 +- .../libreader/ofxreader.py | 44 ++++++--- .../last_transaction_date_test.py | 15 ++++ .../last_transaction/transactions.qfx | 90 +++++++++++++++++++ .../last_transaction/transactions.qfx.extract | 8 ++ .../transactions.qfx.file_account | 1 + .../transactions.qfx.file_date | 1 + .../transactions.qfx.file_name | 1 + .../ofx_date/ofx_date_test.py | 15 ++++ .../ofx_date/transactions.qfx | 90 +++++++++++++++++++ .../ofx_date/transactions.qfx.extract | 8 ++ .../ofx_date/transactions.qfx.file_account | 1 + .../ofx_date/transactions.qfx.file_date | 1 + .../ofx_date/transactions.qfx.file_name | 1 + .../smart/smart_date_test.py | 15 ++++ .../smart/transactions.qfx | 90 +++++++++++++++++++ .../smart/transactions.qfx.extract | 8 ++ .../smart/transactions.qfx.file_account | 1 + .../smart/transactions.qfx.file_date | 1 + .../smart/transactions.qfx.file_name | 1 + .../libtransactionbuilder/paycheck.py | 3 - 25 files changed, 390 insertions(+), 40 deletions(-) create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.extract create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_account create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_date create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_name create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.extract create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_account create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_date create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_name create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_account create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date create mode 100644 beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_name diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py index c31be3c..1a972e3 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py @@ -2,7 +2,6 @@ from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking -import datetime class Importer(csvreader.Importer, banking.Importer): @@ -50,7 +49,6 @@ def prepare_table(self, rdr): def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" - max_date = self.get_max_transaction_date() - if max_date: - date = max_date + datetime.timedelta(days=1) + date = self.get_balance_assertion_date() + if date: yield banking.Balance(date, self.rdr.namedtuples()[0].balance, self.currency) diff --git a/beancount_reds_importers/importers/stanchart/scbbank.py b/beancount_reds_importers/importers/stanchart/scbbank.py index d29fd6b..00a5899 100644 --- a/beancount_reds_importers/importers/stanchart/scbbank.py +++ b/beancount_reds_importers/importers/stanchart/scbbank.py @@ -2,7 +2,6 @@ from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking -import datetime import re from beancount.core.number import D @@ -51,8 +50,8 @@ def prepare_raw_file(self, rdr): def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" - max_date = self.get_max_transaction_date() - if max_date: + date = self.get_balance_assertion_date + if date: rdr = self.read_raw(file) rdr = self.prepare_raw_file(rdr) col_labels = self.balance_column_labels_line.split(',') @@ -64,8 +63,6 @@ def get_balance_statement(self, file=None): while '' in rdr.header(): rdr = rdr.cutout('') - date = max_date + datetime.timedelta(days=1) - row = rdr.namedtuples()[0] amount = row.Current_Balance units, debitcredit = amount.split() diff --git a/beancount_reds_importers/importers/stanchart/scbcard.py b/beancount_reds_importers/importers/stanchart/scbcard.py index 048fe15..b4ab04d 100644 --- a/beancount_reds_importers/importers/stanchart/scbcard.py +++ b/beancount_reds_importers/importers/stanchart/scbcard.py @@ -2,7 +2,6 @@ from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking -import datetime import re from beancount.core.number import D @@ -67,14 +66,12 @@ def prepare_raw_file(self, rdr): def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" - max_date = self.get_max_transaction_date() - if max_date: + date = self.get_balance_assertion_date() + if date: balance_row = self.get_row_by_label(file, 'Current Balance') currency, amount = balance_row[1], balance_row[2] units, debitcredit = amount.split() if debitcredit != 'CR': units = '-' + units - date = max_date + datetime.timedelta(days=1) - yield banking.Balance(date, D(units), currency) diff --git a/beancount_reds_importers/importers/unitedoverseas/uobbank.py b/beancount_reds_importers/importers/unitedoverseas/uobbank.py index 97ba863..b59fa09 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobbank.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobbank.py @@ -2,7 +2,6 @@ from beancount_reds_importers.libreader import xlsreader from beancount_reds_importers.libtransactionbuilder import banking -import datetime import re from beancount.core.number import D @@ -52,11 +51,9 @@ def prepare_raw_file(self, rdr): def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" - max_date = self.get_max_transaction_date() - if max_date: + date = self.get_balance_assertion_date() + if date: row = self.rdr.namedtuples()[0] - date = max_date + datetime.timedelta(days=1) - # Get currency from input file currency = self.get_row_by_label(file, 'Account Number:')[2] diff --git a/beancount_reds_importers/importers/unitedoverseas/uobcard.py b/beancount_reds_importers/importers/unitedoverseas/uobcard.py index 44603c7..65ec243 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobcard.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobcard.py @@ -2,7 +2,6 @@ from beancount_reds_importers.libreader import xlsreader from beancount_reds_importers.libtransactionbuilder import banking -import datetime import re from beancount.core.number import D @@ -69,11 +68,8 @@ def prepare_raw_file(self, rdr): def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" - max_date = self.get_max_transaction_date() - if max_date: + date = self.get_balance_assertion_date + if date: balance_row = self.get_row_by_label(file, 'Statement Balance:') units, currency = balance_row[1], balance_row[2] - - date = max_date + datetime.timedelta(days=1) - yield banking.Balance(date, -1 * D(str(units)), currency) diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index 9830f30..7f2c263 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -76,31 +76,51 @@ def get_available_cash(self, settlement_fund_balance=0): return available_cash - settlement_fund_balance return None - def get_balance_assertion_date(self): + def get_ofx_end_date(self): + end_date = self.ofx_account.statement.end_date + # convert end_date from utc to local timezone + end_date = end_date.replace(tzinfo=datetime.timezone.utc).astimezone().date() + return end_date + + def get_smart_date(self): """ We find the statement's end date from the OFX file. However, banks and credit cards typically have pending transactions that are not included in downloads. When we download the next statement, new transactions may appear prior to the balance assertion date that we generate for this statement. To attempt to avoid this, we set the balance assertion date to either two days before the statement's end date or the last transaction's date, whichever is later. - - Finally, we add an additional day, since Beancount balance assertions are defined to occur - on the beginning of the assertion date. """ - - end_date = self.ofx_account.statement.end_date - # convert end_date from utc to local timezone - end_date = end_date.replace(tzinfo=datetime.timezone.utc).astimezone().date() - end_date -= datetime.timedelta(days=getattr(self, 'balance_assertion_date_fudge', 2)) + end_date = self.get_ofx_end_date() + end_date -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) max_transaction_date = self.get_max_transaction_date() max_transaction_date = max_transaction_date if max_transaction_date else datetime.date.min return_date = max(end_date, max_transaction_date) - - # As defined by Beancount - return_date += datetime.timedelta(days=1) return return_date + def get_balance_assertion_date(self): + """ Choices for the date of the generated balance assertion can be specified in + self.config['balance_assertion_date_type'], which can be: + - 'smart': smart date (default) + - 'ofx_date': date specified in ofx file + - 'last_transaction': max transaction date + - 'today': today's date + + If you want something else, simply override this method in individual importer + + Finally, we add an additional day, since Beancount balance assertions are defined to occur + on the beginning of the assertion date. + """ + + date_type_map = {'smart': self.get_smart_date, + 'ofx_date': self.get_ofx_end_date, + 'last_transaction': self.get_max_transaction_date, + 'today': datetime.date.today} + date_type = self.config.get('balance_assertion_date_type', 'smart') + return_date = date_type_map[date_type]() + + return return_date + datetime.timedelta(days=1) # Next day, as defined by Beancount + def get_max_transaction_date(self): """ Here, we find the last transaction's date. If we use the ofx download date (if our source is ofx), we diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py new file mode 100644 index 0000000..848e283 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py @@ -0,0 +1,15 @@ +from os import path +from beancount.ingest import regression_pytest as regtest +from beancount_reds_importers.importers import ally + + +@regtest.with_importer( + ally.Importer({ + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + "balance_assertion_date_type": "last_transaction", + }) +) +@regtest.with_testdir(path.dirname(__file__)) +class TestSmart(regtest.ImporterTestBase): + pass diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx new file mode 100644 index 0000000..98158b6 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx @@ -0,0 +1,90 @@ + + + + + +0 + +INFO + + +20150102170000.000[+0:UTC] + +ENG + + +Great Bank of Orangeland + +666 + + + + + + +5678 + + +0 + +INFO + + + +USD + + +123456789 + +23456 + +CHECKING + + + + + +20190503000000 + +20190901000000 + + + +XFER + +20190109121112 + +-55.00 + +21302970191741 + +Transfer to savings account + + + + + +XFER + +20190109121112 + +-1000 + +21302970191741 + +Check 23400 + + + + + + +150.65 + +20150101000000.000[+0:UTC] + + + + + + diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.extract b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.extract new file mode 100644 index 0000000..42dbfb9 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.extract @@ -0,0 +1,8 @@ + +2019-01-09 * "Transfer to savings account" + Assets:Banks:Checking -55.00 USD + +2019-01-09 * "Check 23400" + Assets:Banks:Checking -1000 USD + +2019-01-10 balance Assets:Banks:Checking 150.65 USD diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_account b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_account new file mode 100644 index 0000000..1d20439 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_account @@ -0,0 +1 @@ +Assets:Banks:Checking diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_date b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_date new file mode 100644 index 0000000..5a9abd6 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_date @@ -0,0 +1 @@ +2019-09-01T00:00:00 diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_name b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_name new file mode 100644 index 0000000..9749cb2 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/transactions.qfx.file_name @@ -0,0 +1 @@ +transactions.qfx diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py new file mode 100644 index 0000000..0a1a210 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py @@ -0,0 +1,15 @@ +from os import path +from beancount.ingest import regression_pytest as regtest +from beancount_reds_importers.importers import ally + + +@regtest.with_importer( + ally.Importer({ + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + "balance_assertion_date_type": "ofx_date", + }) +) +@regtest.with_testdir(path.dirname(__file__)) +class TestSmart(regtest.ImporterTestBase): + pass diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx new file mode 100644 index 0000000..98158b6 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx @@ -0,0 +1,90 @@ + + + + + +0 + +INFO + + +20150102170000.000[+0:UTC] + +ENG + + +Great Bank of Orangeland + +666 + + + + + + +5678 + + +0 + +INFO + + + +USD + + +123456789 + +23456 + +CHECKING + + + + + +20190503000000 + +20190901000000 + + + +XFER + +20190109121112 + +-55.00 + +21302970191741 + +Transfer to savings account + + + + + +XFER + +20190109121112 + +-1000 + +21302970191741 + +Check 23400 + + + + + + +150.65 + +20150101000000.000[+0:UTC] + + + + + + diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.extract b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.extract new file mode 100644 index 0000000..d7cfec2 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.extract @@ -0,0 +1,8 @@ + +2019-01-09 * "Transfer to savings account" + Assets:Banks:Checking -55.00 USD + +2019-01-09 * "Check 23400" + Assets:Banks:Checking -1000 USD + +2019-09-01 balance Assets:Banks:Checking 150.65 USD diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_account b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_account new file mode 100644 index 0000000..1d20439 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_account @@ -0,0 +1 @@ +Assets:Banks:Checking diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_date b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_date new file mode 100644 index 0000000..5a9abd6 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_date @@ -0,0 +1 @@ +2019-09-01T00:00:00 diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_name b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_name new file mode 100644 index 0000000..9749cb2 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/transactions.qfx.file_name @@ -0,0 +1 @@ +transactions.qfx diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py new file mode 100644 index 0000000..ab7cecd --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py @@ -0,0 +1,15 @@ +from os import path +from beancount.ingest import regression_pytest as regtest +from beancount_reds_importers.importers import ally + + +# default balance_assertion_date_type is "smart" +@regtest.with_importer( + ally.Importer({ + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + }) +) +@regtest.with_testdir(path.dirname(__file__)) +class TestSmart(regtest.ImporterTestBase): + pass diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx new file mode 100644 index 0000000..98158b6 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx @@ -0,0 +1,90 @@ + + + + + +0 + +INFO + + +20150102170000.000[+0:UTC] + +ENG + + +Great Bank of Orangeland + +666 + + + + + + +5678 + + +0 + +INFO + + + +USD + + +123456789 + +23456 + +CHECKING + + + + + +20190503000000 + +20190901000000 + + + +XFER + +20190109121112 + +-55.00 + +21302970191741 + +Transfer to savings account + + + + + +XFER + +20190109121112 + +-1000 + +21302970191741 + +Check 23400 + + + + + + +150.65 + +20150101000000.000[+0:UTC] + + + + + + diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract new file mode 100644 index 0000000..ea7098d --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract @@ -0,0 +1,8 @@ + +2019-01-09 * "Transfer to savings account" + Assets:Banks:Checking -55.00 USD + +2019-01-09 * "Check 23400" + Assets:Banks:Checking -1000 USD + +2019-08-30 balance Assets:Banks:Checking 150.65 USD diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_account b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_account new file mode 100644 index 0000000..1d20439 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_account @@ -0,0 +1 @@ +Assets:Banks:Checking diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date new file mode 100644 index 0000000..5a9abd6 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date @@ -0,0 +1 @@ +2019-09-01T00:00:00 diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_name b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_name new file mode 100644 index 0000000..9749cb2 --- /dev/null +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_name @@ -0,0 +1 @@ +transactions.qfx diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index b2ff49e..d6709fe 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -62,9 +62,6 @@ class Importer(banking.Importer): def file_date(self, input_file): return self.paycheck_date(input_file) - def get_max_transaction_date(self): - return self.date.date() - def build_postings(self, entry): template = self.config['paycheck_template'] currency = self.config['currency']