diff --git a/.github/workflows/conventionalcommits.yml b/.github/workflows/conventionalcommits.yml index b7fa7a1..add48c6 100644 --- a/.github/workflows/conventionalcommits.yml +++ b/.github/workflows/conventionalcommits.yml @@ -2,7 +2,7 @@ name: Conventional Commits on: pull_request: - branches: [ master ] + branches: [ main ] jobs: build: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 172e3d6..0312cb2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -34,3 +34,7 @@ jobs: - name: Test with pytest run: | pytest + - name: Check formatting is applied + run: | + ruff format --check + isort --profile black --check . diff --git a/.ruff.toml b/.ruff.toml deleted file mode 100644 index 46121f3..0000000 --- a/.ruff.toml +++ /dev/null @@ -1 +0,0 @@ -line-length = 127 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a33af04..2cdb81e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,5 @@ +# Contributing + Contributions welcome. Preferably: - include a test file. I realize this is sometimes a pain to create, but there is no way for me to test external contributions without test files @@ -18,3 +20,30 @@ Contributions welcome. Preferably:          ├── History_for_Account_X8YYYYYYY.csv          └── run_test.bash ``` + +## Setup + +Development setup would typically look something like this: + +```bash +# clone repo, cd to repo + +# create virtual environment +python3 -m venv venv + +# activate virtual environment +source venv/bin/activate + +# install dependencies +pip install -e .[dev] +``` + +## Formatting + +Prior to finalizing a pull request make sure to run the formatting tools and +commit any resulting changes. + +```bash +ruff format +isort --profile black . +``` diff --git a/beancount_reds_importers/example/fund_info.py b/beancount_reds_importers/example/fund_info.py index 807e647..65858d2 100755 --- a/beancount_reds_importers/example/fund_info.py +++ b/beancount_reds_importers/example/fund_info.py @@ -20,15 +20,15 @@ # mutual funds since those are brokerage specific. fund_data = [ - ('SCHF', '808524805', 'Schwab International Equity ETF'), - ('VGTEST', '012345678', 'Vanguard Test Fund'), - ('VMFXX', '922906300', 'Vanguard Federal Money Market Fund'), + ("SCHF", "808524805", "Schwab International Equity ETF"), + ("VGTEST", "012345678", "Vanguard Test Fund"), + ("VMFXX", "922906300", "Vanguard Federal Money Market Fund"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} diff --git a/beancount_reds_importers/example/my-smart.import b/beancount_reds_importers/example/my-smart.import index 66b62da..2ba0b7b 100644 --- a/beancount_reds_importers/example/my-smart.import +++ b/beancount_reds_importers/example/my-smart.import @@ -4,7 +4,8 @@ import sys from os import path -from smart_importer import apply_hooks, PredictPayees, PredictPostings +from smart_importer import PredictPayees, PredictPostings, apply_hooks + sys.path.insert(0, path.join(path.dirname(__file__))) from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/example/my.import b/beancount_reds_importers/example/my.import index 8cc410e..6eec804 100644 --- a/beancount_reds_importers/example/my.import +++ b/beancount_reds_importers/example/my.import @@ -6,9 +6,11 @@ from os import path sys.path.insert(0, path.join(path.dirname(__file__))) +from fund_info import * + from beancount_reds_importers.importers import vanguard from beancount_reds_importers.importers.schwab import schwab_csv_brokerage -from fund_info import * + # For a better solution for fund_info, see: https://reds-rants.netlify.app/personal-finance/tickers-and-identifiers/ # Setting this variable provides a list of importer instances. diff --git a/beancount_reds_importers/importers/alliant/__init__.py b/beancount_reds_importers/importers/alliant/__init__.py index a67681c..dd6c587 100644 --- a/beancount_reds_importers/importers/alliant/__init__.py +++ b/beancount_reds_importers/importers/alliant/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Alliant Credit Union' + IMPORTER_NAME = "Alliant Credit Union" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*alliant' + self.filename_pattern_def = ".*alliant" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/ally/__init__.py b/beancount_reds_importers/importers/ally/__init__.py index 3764d0f..08b1264 100644 --- a/beancount_reds_importers/importers/ally/__init__.py +++ b/beancount_reds_importers/importers/ally/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Ally Bank' + IMPORTER_NAME = "Ally Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*transactions' + self.filename_pattern_def = ".*transactions" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/ally/tests/ally_test.py b/beancount_reds_importers/importers/ally/tests/ally_test.py index 84b2bad..8f7e87f 100644 --- a/beancount_reds_importers/importers/ally/tests/ally_test.py +++ b/beancount_reds_importers/importers/ally/tests/ally_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/importers/amazongc/__init__.py b/beancount_reds_importers/importers/amazongc/__init__.py index f4f486d..031e2f1 100644 --- a/beancount_reds_importers/importers/amazongc/__init__.py +++ b/beancount_reds_importers/importers/amazongc/__init__.py @@ -23,9 +23,10 @@ import datetime import itertools import ntpath + from beancount.core import data -from beancount.ingest import importer from beancount.core.number import D +from beancount.ingest import importer # account flow ingest source # ---------------------------------------------------- @@ -37,25 +38,25 @@ class Importer(importer.ImporterProtocol): def __init__(self, config): self.config = config - self.currency = self.config.get('currency', 'CURRENCY_NOT_CONFIGURED') - self.filename_pattern_def = 'amazon-gift-card.tsv' + self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED") + self.filename_pattern_def = "amazon-gift-card.tsv" def identify(self, file): return self.filename_pattern_def in file.name def file_name(self, file): - return '{}'.format(ntpath.basename(file.name)) + return "{}".format(ntpath.basename(file.name)) def file_account(self, _): - return self.config['main_account'] + return self.config["main_account"] def file_date(self, file): "Get the maximum date from the file." maxdate = datetime.date.min - for line in open(file.name, 'r').readlines()[1:]: - f = line.split('\t') + for line in open(file.name, "r").readlines()[1:]: + f = line.split("\t") f = [i.strip() for i in f] - date = datetime.datetime.strptime(f[0], '%B %d, %Y').date() + date = datetime.datetime.strptime(f[0], "%B %d, %Y").date() maxdate = max(date, maxdate) return maxdate @@ -65,18 +66,26 @@ def extract(self, file, existing_entries=None): new_entries = [] counter = itertools.count() - for line in open(file.name, 'r').readlines()[1:]: - f = line.split('\t') + for line in open(file.name, "r").readlines()[1:]: + f = line.split("\t") f = [i.strip() for i in f] - date = datetime.datetime.strptime(f[0], '%B %d, %Y').date() + date = datetime.datetime.strptime(f[0], "%B %d, %Y").date() description = f[1].encode("ascii", "ignore").decode() - number = D(f[2].replace('$', '')) + number = D(f[2].replace("$", "")) metadata = data.new_metadata(file.name, next(counter)) - entry = data.Transaction(metadata, date, self.FLAG, - None, description, data.EMPTY_SET, data.EMPTY_SET, []) - data.create_simple_posting(entry, config['main_account'], number, self.currency) - data.create_simple_posting(entry, config['target_account'], None, None) + entry = data.Transaction( + metadata, + date, + self.FLAG, + None, + description, + data.EMPTY_SET, + data.EMPTY_SET, + [], + ) + data.create_simple_posting(entry, config["main_account"], number, self.currency) + data.create_simple_posting(entry, config["target_account"], None, None) new_entries.append(entry) return new_entries diff --git a/beancount_reds_importers/importers/amex/__init__.py b/beancount_reds_importers/importers/amex/__init__.py index 8ebae4e..086cb0a 100644 --- a/beancount_reds_importers/importers/amex/__init__.py +++ b/beancount_reds_importers/importers/amex/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'American Express' + IMPORTER_NAME = "American Express" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*amex' + self.filename_pattern_def = ".*amex" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/becu/__init__.py b/beancount_reds_importers/importers/becu/__init__.py index 3f978fe..a0a019a 100644 --- a/beancount_reds_importers/importers/becu/__init__.py +++ b/beancount_reds_importers/importers/becu/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'BECU' + IMPORTER_NAME = "BECU" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*becu' + self.filename_pattern_def = ".*becu" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/capitalonebank/__init__.py b/beancount_reds_importers/importers/capitalonebank/__init__.py index 43a93d2..48c1043 100644 --- a/beancount_reds_importers/importers/capitalonebank/__init__.py +++ b/beancount_reds_importers/importers/capitalonebank/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Capital One Bank' + IMPORTER_NAME = "Capital One Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*360Checking' + self.filename_pattern_def = ".*360Checking" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py b/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py index 6ca6ac9..a7215c0 100644 --- a/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py +++ b/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import capitalonebank diff --git a/beancount_reds_importers/importers/chase/__init__.py b/beancount_reds_importers/importers/chase/__init__.py index 4e70ebf..4a79edc 100644 --- a/beancount_reds_importers/importers/chase/__init__.py +++ b/beancount_reds_importers/importers/chase/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Chase' + IMPORTER_NAME = "Chase" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*[Cc]hase' + self.filename_pattern_def = ".*[Cc]hase" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/citi/__init__.py b/beancount_reds_importers/importers/citi/__init__.py index 4460ae2..c4d210f 100644 --- a/beancount_reds_importers/importers/citi/__init__.py +++ b/beancount_reds_importers/importers/citi/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Citi' + IMPORTER_NAME = "Citi" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*citi' + self.filename_pattern_def = ".*citi" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/dcu/__init__.py b/beancount_reds_importers/importers/dcu/__init__.py index d818a1b..3a31104 100644 --- a/beancount_reds_importers/importers/dcu/__init__.py +++ b/beancount_reds_importers/importers/dcu/__init__.py @@ -12,21 +12,24 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = ".*Account_Transactions" self.header_identifier = "" - self.column_labels_line = '"DATE","TRANSACTION TYPE","DESCRIPTION","AMOUNT","ID","MEMO","CURRENT BALANCE"' + self.column_labels_line = ( + '"DATE","TRANSACTION TYPE","DESCRIPTION","AMOUNT","ID","MEMO","CURRENT BALANCE"' + ) self.date_format = "%m/%d/%Y" + # fmt: off self.header_map = { - "DATE": "date", - "DESCRIPTION": "payee", - "MEMO": "memo", - "AMOUNT": "amount", - "CURRENT BALANCE": "balance", + "DATE": "date", + "DESCRIPTION": "payee", + "MEMO": "memo", + "AMOUNT": "amount", + "CURRENT BALANCE": "balance", "TRANSACTION TYPE": "type", } - self.transaction_type_map = { - "DEBIT": "transfer", - "CREDIT": "transfer", + "DEBIT": "transfer", + "CREDIT": "transfer", } + # fmt: on self.skip_transaction_types = [] def get_balance_statement(self, file=None): @@ -34,6 +37,4 @@ def get_balance_statement(self, file=None): date = self.get_balance_assertion_date() if date: - yield banking.Balance( - date, self.rdr.namedtuples()[0].balance, self.currency - ) + yield banking.Balance(date, self.rdr.namedtuples()[0].balance, self.currency) diff --git a/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py b/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py index dbadc22..c05e35c 100644 --- a/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py +++ b/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import dcu diff --git a/beancount_reds_importers/importers/discover/__init__.py b/beancount_reds_importers/importers/discover/__init__.py index a554e07..d06cdfc 100644 --- a/beancount_reds_importers/importers/discover/__init__.py +++ b/beancount_reds_importers/importers/discover/__init__.py @@ -1,4 +1,4 @@ -""" Discover credit card .csv importer.""" +"""Discover credit card .csv importer.""" from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking @@ -9,21 +9,23 @@ class Importer(csvreader.Importer, banking.Importer): def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'Discover.*' - self.header_identifier = 'Trans. Date,Post Date,Description,Amount,Category' - self.date_format = '%m/%d/%Y' + self.filename_pattern_def = "Discover.*" + self.header_identifier = "Trans. Date,Post Date,Description,Amount,Category" + self.date_format = "%m/%d/%Y" + # fmt: off self.header_map = { - "Category": 'payee', - "Description": 'memo', - "Trans. Date": 'date', - "Post Date": 'postDate', - "Amount": 'amount', - } + "Category": "payee", + "Description": "memo", + "Trans. Date": "date", + "Post Date": "postDate", + "Amount": "amount", + } + # fmt: on def skip_transaction(self, ot): return False def prepare_processed_table(self, rdr): # Need to invert numbers supplied by Discover - rdr = rdr.convert('amount', lambda x: -1 * x) + rdr = rdr.convert("amount", lambda x: -1 * x) return rdr diff --git a/beancount_reds_importers/importers/discover/discover_ofx.py b/beancount_reds_importers/importers/discover/discover_ofx.py index 0593a2e..abee824 100644 --- a/beancount_reds_importers/importers/discover/discover_ofx.py +++ b/beancount_reds_importers/importers/discover/discover_ofx.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Discover' + IMPORTER_NAME = "Discover" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Discover' + self.filename_pattern_def = ".*Discover" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/etrade/__init__.py b/beancount_reds_importers/importers/etrade/__init__.py index b5eb96a..bc6d3d3 100644 --- a/beancount_reds_importers/importers/etrade/__init__.py +++ b/beancount_reds_importers/importers/etrade/__init__.py @@ -1,18 +1,18 @@ -""" ETrade Brokerage ofx importer.""" +"""ETrade Brokerage ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'ETrade Brokerage OFX' + IMPORTER_NAME = "ETrade Brokerage OFX" def custom_init(self): self.max_rounding_error = 0.11 - self.filename_pattern_def = '.*etrade' + self.filename_pattern_def = ".*etrade" self.get_ticker_info = self.get_ticker_info_from_id def skip_transaction(self, ot): - if 'JNL' in ot.memo: + if "JNL" in ot.memo: return True return False diff --git a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py index fefd32d..656fa32 100644 --- a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py +++ b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py @@ -1,54 +1,51 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest -from beancount_reds_importers.importers import etrade +from beancount_reds_importers.importers import etrade fund_data = [ - ('TSM', '874039100', 'Taiwan Semiconductor Mfg LTD'), - ('VISA', '92826C839', 'Visa Inc'), + ("TSM", "874039100", "Taiwan Semiconductor Mfg LTD"), + ("VISA", "92826C839", "Visa Inc"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} def build_config(): acct = "Assets:Investments:Etrade" - root = 'Investments' - taxability = 'Taxable' - leaf = 'Etrade' - currency = 'USD' + root = "Investments" + taxability = "Taxable" + leaf = "Etrade" + currency = "USD" config = { - 'account_number' : '555555555', - 'main_account' : acct + ':{ticker}', - 'cash_account' : f'{acct}:{{currency}}', - 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', - 'dividends' : f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', - 'interest' : f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', - 'cg' : f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', - 'capgainsd_lt' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', - 'capgainsd_st' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', - 'fees' : f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', - 'invexpense' : f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', - 'rounding_error' : 'Equity:Rounding-Errors:Imports', - 'fund_info' : fund_info, - 'currency' : currency, + "account_number": "555555555", + "main_account": acct + ":{ticker}", + "cash_account": f"{acct}:{{currency}}", + "transfer": "Assets:Zero-Sum-Accounts:Transfers:Bank-Account", + "dividends": f"Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}", + "interest": f"Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}", + "cg": f"Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}", + "capgainsd_lt": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}", + "capgainsd_st": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}", + "fees": f"Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}", + "invexpense": f"Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}", + "rounding_error": "Equity:Rounding-Errors:Imports", + "fund_info": fund_info, + "currency": currency, } return config -@regtest.with_importer( - etrade.Importer( - build_config() - ) -) +@regtest.with_importer(etrade.Importer(build_config())) @regtest.with_testdir(path.dirname(__file__)) class TestEtradeQFX(regtest.ImporterTestBase): pass diff --git a/beancount_reds_importers/importers/fidelity/__init__.py b/beancount_reds_importers/importers/fidelity/__init__.py index d0438b7..96d869a 100644 --- a/beancount_reds_importers/importers/fidelity/__init__.py +++ b/beancount_reds_importers/importers/fidelity/__init__.py @@ -1,32 +1,33 @@ """Fidelity Net Benefits and Fidelity Investments OFX importer.""" import ntpath + from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Fidelity Net Benefits / Fidelity Investments OFX' + IMPORTER_NAME = "Fidelity Net Benefits / Fidelity Investments OFX" def custom_init(self): self.max_rounding_error = 0.18 - self.filename_pattern_def = '.*fidelity' + self.filename_pattern_def = ".*fidelity" self.get_ticker_info = self.get_ticker_info_from_id - self.get_payee = lambda ot: ot.memo.split(";", 1)[0] if ';' in ot.memo else ot.memo + self.get_payee = lambda ot: ot.memo.split(";", 1)[0] if ";" in ot.memo else ot.memo def security_narration(self, ot): ticker, ticker_long_name = self.get_ticker_info(ot.security) return f"[{ticker}]" def file_name(self, file): - return 'fidelity-{}-{}'.format(self.config['account_number'], ntpath.basename(file.name)) + return "fidelity-{}-{}".format(self.config["account_number"], ntpath.basename(file.name)) def get_target_acct_custom(self, transaction, ticker=None): if transaction.memo.startswith("CONTRIBUTION"): - return self.config['transfer'] + return self.config["transfer"] if transaction.memo.startswith("FEES"): - return self.config['fees'] + return self.config["fees"] return None def get_available_cash(self, settlement_fund_balance=0): - return getattr(self.ofx_account.statement, 'available_cash', None) + return getattr(self.ofx_account.statement, "available_cash", None) diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py index c609e1c..d2b9076 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py @@ -1,50 +1,53 @@ """Fidelity CMA/checking csv importer for beancount.""" +import re + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking -import re class Importer(banking.Importer, csvreader.Importer): - IMPORTER_NAME = 'Fidelity Cash Management Account' + IMPORTER_NAME = "Fidelity Cash Management Account" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*History' - self.date_format = '%m/%d/%Y' - header_s0 = ".*Run Date,Action,Symbol,Security Description,Security Type,Quantity,Price \\(\\$\\)," + self.filename_pattern_def = ".*History" + self.date_format = "%m/%d/%Y" + header_s0 = ( + ".*Run Date,Action,Symbol,Security Description,Security Type,Quantity,Price \\(\\$\\)," + ) header_s1 = "Commission \\(\\$\\),Fees \\(\\$\\),Accrued Interest \\(\\$\\),Amount \\(\\$\\),Settlement Date" header_sum = header_s0 + header_s1 self.header_identifier = header_sum self.skip_head_rows = 5 self.skip_tail_rows = 16 + # fmt: off self.header_map = { - "Run Date": 'date', - "Action": 'description', - "Amount ($)": 'amount', - - "Settlement Date": 'settleDate', - "Accrued Interest ($)": 'accrued_interest', - "Fees ($)": 'fees', - "Security Type": 'security_type', - "Commission ($)": 'commission', - "Security Description": 'security_description', - "Symbol": 'security', - "Price ($)": 'unit_price', - } + "Run Date": "date", + "Action": "description", + "Amount ($)": "amount", + "Settlement Date": "settleDate", + "Accrued Interest ($)": "accrued_interest", + "Fees ($)": "fees", + "Security Type": "security_type", + "Commission ($)": "commission", + "Security Description": "security_description", + "Symbol": "security", + "Price ($)": "unit_price", + } + # fmt: on def deep_identify(self, file): return re.match(self.header_identifier, file.head(), flags=re.DOTALL) def prepare_raw_columns(self, rdr): - - for field in ['Action']: + for field in ["Action"]: rdr = rdr.convert(field, lambda x: x.lstrip()) - rdr = rdr.capture('Action', '(?:\\s)(?:\\w*)(.*)', ['memo'], include_original=True) - rdr = rdr.capture('Action', '(\\S+(?:\\s+\\S+)?)', ['payee'], include_original=True) + rdr = rdr.capture("Action", "(?:\\s)(?:\\w*)(.*)", ["memo"], include_original=True) + rdr = rdr.capture("Action", "(\\S+(?:\\s+\\S+)?)", ["payee"], include_original=True) - for field in ['memo', 'payee']: + for field in ["memo", "payee"]: rdr = rdr.convert(field, lambda x: x.lstrip()) return rdr diff --git a/beancount_reds_importers/importers/morganstanley/__init__.py b/beancount_reds_importers/importers/morganstanley/__init__.py index 7084e1d..5a859d8 100644 --- a/beancount_reds_importers/importers/morganstanley/__init__.py +++ b/beancount_reds_importers/importers/morganstanley/__init__.py @@ -1,13 +1,13 @@ -""" Morgan Stanley Investments ofx importer.""" +"""Morgan Stanley Investments ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Morgan Stanley Investments' + IMPORTER_NAME = "Morgan Stanley Investments" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*morganstanley' + self.filename_pattern_def = ".*morganstanley" self.get_ticker_info = self.get_ticker_info_from_id diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py index 5f52426..98fdc32 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py @@ -1,42 +1,47 @@ -""" Schwab csv importer.""" +"""Schwab csv importer.""" import datetime import re + from beancount.core.number import D + from beancount_reds_importers.libreader import csv_multitable_reader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, csv_multitable_reader.Importer): - IMPORTER_NAME = 'Schwab Brokerage Balances CSV' + IMPORTER_NAME = "Schwab Brokerage Balances CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Balances_' - self.header_identifier = 'Balances for account' + self.filename_pattern_def = ".*_Balances_" + self.header_identifier = "Balances for account" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on def prepare_table(self, rdr): return rdr def convert_columns(self, rdr): # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.]', "", x) # noqa: W605 - currencies = ['unit_price'] + return re.sub(r"[^0-9\.]", "", x) # noqa: W605 + + currencies = ["unit_price"] for i in currencies: rdr = rdr.convert(i, remove_non_numeric) rdr = rdr.convert(i, D) @@ -51,18 +56,18 @@ def get_max_transaction_date(self): def prepare_tables(self): # first row has date - d = self.raw_rdr[0][0].rsplit(' ', 1)[1] + d = self.raw_rdr[0][0].rsplit(" ", 1)[1] self.date = datetime.datetime.strptime(d, self.date_format) for section, table in self.alltables.items(): - if section in self.config['section_headers']: + if section in self.config["section_headers"]: table = table.rename(self.header_map) table = self.convert_columns(table) - table = table.cut('memo', 'security', 'units', 'unit_price') - table = table.selectne('memo', '--') # we don't need total rows - table = table.addfield('date', self.date) + table = table.cut("memo", "security", "units", "unit_price") + table = table.selectne("memo", "--") # we don't need total rows + table = table.addfield("date", self.date) self.alltables[section] = table def get_balance_positions(self): - for section in self.config['section_headers']: + for section in self.config["section_headers"]: yield from self.alltables[section].namedtuples() diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index 01583fc..ee01119 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -1,76 +1,81 @@ -""" Schwab Brokerage .csv importer.""" +"""Schwab Brokerage .csv importer.""" import re + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(csvreader.Importer, investments.Importer): - IMPORTER_NAME = 'Schwab Brokerage CSV' + IMPORTER_NAME = "Schwab Brokerage CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Transactions_' - self.header_identifier = '' - self.column_labels_line = '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' + self.filename_pattern_def = ".*_Transactions_" + self.header_identifier = "" + self.column_labels_line = ( + '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' + ) self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" self.get_payee = lambda ot: ot.Action + # fmt: off self.header_map = { - "Date": 'date', - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - "Amount": 'amount', - # "tradeDate": 'tradeDate', - # "total": 'total', - "Fees & Comm": 'fees', - } + "Date": "date", + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + "Amount": "amount", + # "tradeDate": "tradeDate", + # "total": "total", + "Fees & Comm": "fees", + } self.transaction_type_map = { - 'Bank Interest': 'income', - 'Bank Transfer': 'cash', - 'Buy': 'buystock', - 'Journaled Shares': 'buystock', # These are in-kind tranfers - 'Reinvestment Adj': 'buystock', - 'Div Adjustment': 'dividends', - 'Long Term Cap Gain Reinvest': 'capgainsd_lt', - 'Misc Credits': 'cash', - 'MoneyLink Deposit': 'cash', - 'MoneyLink Transfer': 'cash', - 'Pr Yr Div Reinvest': 'dividends', - 'Journal': 'cash', # These are transfers - 'Reinvest Dividend': 'dividends', - 'Qualified Dividend': 'dividends', - 'Cash Dividend': 'dividends', - 'Reinvest Shares': 'buystock', - 'Sell': 'sellstock', - 'Short Term Cap Gain Reinvest': 'capgainsd_st', - 'Wire Funds Received': 'cash', - 'Wire Received': 'cash', - 'Funds Received': 'cash', - 'Stock Split': 'cash', - 'Cash In Lieu': 'cash', - } + "Bank Interest": "income", + "Bank Transfer": "cash", + "Buy": "buystock", + "Journaled Shares": "buystock", # These are in-kind tranfers + "Reinvestment Adj": "buystock", + "Div Adjustment": "dividends", + "Long Term Cap Gain Reinvest": "capgainsd_lt", + "Misc Credits": "cash", + "MoneyLink Deposit": "cash", + "MoneyLink Transfer": "cash", + "Pr Yr Div Reinvest": "dividends", + "Journal": "cash", # These are transfers + "Reinvest Dividend": "dividends", + "Qualified Dividend": "dividends", + "Cash Dividend": "dividends", + "Reinvest Shares": "buystock", + "Sell": "sellstock", + "Short Term Cap Gain Reinvest": "capgainsd_st", + "Wire Funds Received": "cash", + "Wire Received": "cash", + "Funds Received": "cash", + "Stock Split": "cash", + "Cash In Lieu": "cash", + } + # fmt: on def deep_identify(self, file): - last_three = self.config.get('account_number', '')[-3:] - return re.match(self.header_identifier, file.head()) and f'XX{last_three}' in file.name + last_three = self.config.get("account_number", "")[-3:] + return re.match(self.header_identifier, file.head()) and f"XX{last_three}" in file.name def skip_transaction(self, ot): - return ot.type in ['', 'Journal'] + return ot.type in ["", "Journal"] def prepare_table(self, rdr): - if '' in rdr.fieldnames(): - rdr = rdr.cutout('') # clean up last column + if "" in rdr.fieldnames(): + rdr = rdr.cutout("") # clean up last column def cleanup_date(d): """'11/16/2018 as of 11/15/2018' --> '11/16/2018'""" - return d.split(' ', 1)[0] + return d.split(" ", 1)[0] - rdr = rdr.convert('Date', cleanup_date) - rdr = rdr.addfield('tradeDate', lambda x: x['Date']) - rdr = rdr.addfield('total', lambda x: x['Amount']) - rdr = rdr.addfield('type', lambda x: x['Action']) + rdr = rdr.convert("Date", cleanup_date) + rdr = rdr.addfield("tradeDate", lambda x: x["Date"]) + rdr = rdr.addfield("total", lambda x: x["Amount"]) + rdr = rdr.addfield("type", lambda x: x["Action"]) return rdr diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py index 7a3072c..a26db2c 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py @@ -1,43 +1,47 @@ -""" Schwab Checking .csv importer.""" +"""Schwab Checking .csv importer.""" from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'Schwab Checking account CSV' + IMPORTER_NAME = "Schwab Checking account CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Checking_Transactions_' - self.header_identifier = '' + self.filename_pattern_def = ".*_Checking_Transactions_" + self.header_identifier = "" self.column_labels_line = '"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' - self.date_format = '%m/%d/%Y' - self.skip_comments = '# ' + self.date_format = "%m/%d/%Y" + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "Type": "type", - "CheckNumber": "checknum", - "Description": "payee", - "Withdrawal": "withdrawal", - "Deposit": "deposit", - "RunningBalance": "balance" + "Date": "date", + "Type": "type", + "CheckNumber": "checknum", + "Description": "payee", + "Withdrawal": "withdrawal", + "Deposit": "deposit", + "RunningBalance": "balance", } self.transaction_type_map = { - "INTADJUST": 'income', - "TRANSFER": 'transfer', - "ACH": 'transfer' + "INTADJUST": "income", + "TRANSFER": "transfer", + "ACH": "transfer", } - self.skip_transaction_types = ['Journal'] + # fmt: on + self.skip_transaction_types = ["Journal"] def deep_identify(self, file): - last_three = self.config.get('account_number', '')[-3:] - return self.column_labels_line in file.head() and f'XX{last_three}' in file.name + last_three = self.config.get("account_number", "")[-3:] + return self.column_labels_line in file.head() and f"XX{last_three}" in file.name def prepare_table(self, rdr): - rdr = rdr.addfield('amount', - lambda x: "-" + x['Withdrawal'] if x['Withdrawal'] != '' else x['Deposit']) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.addfield( + "amount", + lambda x: "-" + x["Withdrawal"] if x["Withdrawal"] != "" else x["Deposit"], + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def get_balance_statement(self, file=None): diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py index 5f452de..be9e836 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py @@ -1,11 +1,21 @@ -""" Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" +"""Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" from beancount_reds_importers.importers.schwab import schwab_csv_checking +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(schwab_csv_checking.Importer): - IMPORTER_NAME = 'Schwab Line of Credit CSV' + IMPORTER_NAME = "Schwab Line of Credit CSV" def custom_init(self): super().custom_init() - self.filename_pattern_def = '.*_Transactions_' - self.column_labels_line = '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' + self.filename_pattern_def = ".*_Transactions_" + self.column_labels_line = ( + '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' + ) + + def get_balance_statement(self, file=None): + """Return the balance on the first and last dates""" + + for i in super().get_balance_statement(file): + yield banking.Balance(i.date, -1 * i.amount, i.currency) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py index 8f3bd1a..3e7de52 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py @@ -1,10 +1,12 @@ -""" Schwab CSV Positions importer. +"""Schwab CSV Positions importer. Note: Schwab "Positions" CSV is not the same as Schwab "Balances" CSV.""" import datetime import re + from beancount.core.number import D + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import investments @@ -14,30 +16,33 @@ class Importer(investments.Importer, csvreader.Importer): def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*-Positions-' + self.filename_pattern_def = ".*-Positions-" self.header_identifier = '["]+Positions for account' self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%Y/%m/%d' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%Y/%m/%d" + self.funds_db_txt = "funds_by_ticker" self.column_labels_line = '"Symbol","Description","Quantity","Price","Price Change %","Price Change $","Market Value","Day Change %","Day Change $","Cost Basis","Gain/Loss %","Gain/Loss $","Ratings","Reinvest Dividends?","Capital Gains?","% Of Account","Security Type"' # noqa: #501 + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on self.skip_transaction_types = [] def convert_columns(self, rdr): # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.]', "", x) # noqa: W605 - currencies = ['unit_price'] + return re.sub(r"[^0-9\.]", "", x) # noqa: W605 + + currencies = ["unit_price"] for i in currencies: rdr = rdr.convert(i, remove_non_numeric) rdr = rdr.convert(i, D) @@ -53,7 +58,7 @@ def get_max_transaction_date(self): def prepare_raw_file(self, rdr): # first row has date - d = rdr[0][0].rsplit(' ', 1)[1] + d = rdr[0][0].rsplit(" ", 1)[1] self.date = datetime.datetime.strptime(d, self.date_format) return rdr @@ -68,5 +73,5 @@ def prepare_table(self, rdr): def get_balance_positions(self): for pos in self.rdr.namedtuples(): - if pos.memo != '--': + if pos.memo != "--": yield pos diff --git a/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py index 3c50ad1..4ad4504 100644 --- a/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py @@ -1,18 +1,18 @@ -""" Schwab Brokerage .csv importer.""" +"""Schwab Brokerage .csv importer.""" from beancount_reds_importers.libreader import jsonreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(jsonreader.Importer, investments.Importer): - IMPORTER_NAME = 'Schwab Brokerage JSON' + IMPORTER_NAME = "Schwab Brokerage JSON" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Transactions_' + self.filename_pattern_def = ".*_Transactions_" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" def skip_transaction(self, ot): - return ot.type in ['', 'Journal', 'Journaled Shares'] + return ot.type in ["", "Journal", "Journaled Shares"] diff --git a/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py b/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py index fda1141..03df212 100644 --- a/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py +++ b/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Schwab Bank' + IMPORTER_NAME = "Schwab Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Checking_Transations' + self.filename_pattern_def = ".*Checking_Transations" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py index 3e28c73..af7aeb0 100644 --- a/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py @@ -1,13 +1,13 @@ -""" Schwab Brokerage ofx importer.""" +"""Schwab Brokerage ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Schwab Brokerage' + IMPORTER_NAME = "Schwab Brokerage" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*schwab' + self.filename_pattern_def = ".*schwab" self.get_ticker_info = self.get_ticker_info_from_id diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py index 546fca6..ea91049 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py @@ -1,55 +1,52 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest -from beancount_reds_importers.importers.schwab import schwab_csv_brokerage +from beancount_reds_importers.importers.schwab import schwab_csv_brokerage fund_data = [ - ('SWVXX', '123', 'SCHWAB VALUE ADVANTAGE MONEY INV'), - ('GIS', '456', 'GENERAL MILLS INC'), - ('BND', '789', 'Vanguard Total Bond Market Index Fund'), + ("SWVXX", "123", "SCHWAB VALUE ADVANTAGE MONEY INV"), + ("GIS", "456", "GENERAL MILLS INC"), + ("BND", "789", "Vanguard Total Bond Market Index Fund"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} def build_config(): acct = "Assets:Investments:Schwab" - root = 'Investments' - taxability = 'Taxable' - leaf = 'Schwab' - currency = 'USD' + root = "Investments" + taxability = "Taxable" + leaf = "Schwab" + currency = "USD" config = { - 'account_number' : '9876', - 'main_account' : acct + ':{ticker}', - 'cash_account' : f'{acct}:{{currency}}', - 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', - 'dividends' : f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', - 'interest' : f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', - 'cg' : f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', - 'capgainsd_lt' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', - 'capgainsd_st' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', - 'fees' : f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', - 'invexpense' : f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', - 'rounding_error' : 'Equity:Rounding-Errors:Imports', - 'fund_info' : fund_info, - 'currency' : currency, + "account_number": "9876", + "main_account": acct + ":{ticker}", + "cash_account": f"{acct}:{{currency}}", + "transfer": "Assets:Zero-Sum-Accounts:Transfers:Bank-Account", + "dividends": f"Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}", + "interest": f"Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}", + "cg": f"Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}", + "capgainsd_lt": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}", + "capgainsd_st": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}", + "fees": f"Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}", + "invexpense": f"Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}", + "rounding_error": "Equity:Rounding-Errors:Imports", + "fund_info": fund_info, + "currency": currency, } return config -@regtest.with_importer( - schwab_csv_brokerage.Importer( - build_config() - ) -) +@regtest.with_importer(schwab_csv_brokerage.Importer(build_config())) @regtest.with_testdir(path.dirname(__file__)) class TestSchwabBrokerage(regtest.ImporterTestBase): pass diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py index b44919e..04e3bb7 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py @@ -1,15 +1,18 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers.schwab import schwab_csv_checking + @regtest.with_importer( schwab_csv_checking.Importer( { - 'account_number' : '1234', - 'main_account' : 'Assets:Banks:Schwab', - 'currency' : 'USD', + "account_number": "1234", + "main_account": "Assets:Banks:Schwab", + "currency": "USD", } ) ) diff --git a/beancount_reds_importers/importers/stanchart/scbbank.py b/beancount_reds_importers/importers/stanchart/scbbank.py index 6a73fb7..06f6319 100644 --- a/beancount_reds_importers/importers/stanchart/scbbank.py +++ b/beancount_reds_importers/importers/stanchart/scbbank.py @@ -1,50 +1,59 @@ """SCB Banking .csv importer.""" -from beancount_reds_importers.libreader import csvreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import csvreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Banking Account CSV' + IMPORTER_NAME = "SCB Banking Account CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'AccountTransactions[0-9]*' - self.header_identifier = self.config.get('custom_header', 'Account transactions shown:') - self.column_labels_line = 'Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance' - self.balance_column_labels_line = 'Account Name,Account Number,Currency,Current Balance,Available Balance' - self.date_format = '%d/%m/%Y' + self.filename_pattern_def = "AccountTransactions[0-9]*" + self.header_identifier = self.config.get("custom_header", "Account transactions shown:") + self.column_labels_line = ( + "Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance" + ) + self.balance_column_labels_line = ( + "Account Name,Account Number,Currency,Current Balance,Available Balance" + ) + self.date_format = "%d/%m/%Y" self.skip_tail_rows = 0 - self.skip_comments = '# ' + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "Transaction": "payee", - "Currency": "currency", - "Withdrawal": "withdrawal", - "Deposit": "deposit", - "Running Balance": "balance_running", - "SGD Equivalent Balance": "balance", + "Date": "date", + "Transaction": "payee", + "Currency": "currency", + "Withdrawal": "withdrawal", + "Deposit": "deposit", + "Running Balance": "balance_running", + "SGD Equivalent Balance": "balance", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): - rdr = rdr.addfield('amount', - lambda x: "-" + x['Withdrawal'] if x['Withdrawal'] != '' else x['Deposit']) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.addfield( + "amount", + lambda x: "-" + x["Withdrawal"] if x["Withdrawal"] != "" else x["Deposit"], + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) return rdr @@ -54,18 +63,18 @@ def get_balance_statement(self, file=None): if date: rdr = self.read_raw(file) rdr = self.prepare_raw_file(rdr) - col_labels = self.balance_column_labels_line.split(',') + col_labels = self.balance_column_labels_line.split(",") rdr = self.extract_table_with_header(rdr, col_labels) - header_map = {k: k.replace(' ', '_') for k in col_labels} + header_map = {k: k.replace(" ", "_") for k in col_labels} rdr = rdr.rename(header_map) - while '' in rdr.header(): - rdr = rdr.cutout('') + while "" in rdr.header(): + rdr = rdr.cutout("") row = rdr.namedtuples()[0] amount = row.Current_Balance units, debitcredit = amount.split() - if debitcredit != 'CR': - units = '-' + units + if debitcredit != "CR": + units = "-" + units yield banking.Balance(date, D(units), row.Currency) diff --git a/beancount_reds_importers/importers/stanchart/scbcard.py b/beancount_reds_importers/importers/stanchart/scbcard.py index b4ab04d..7aba70b 100644 --- a/beancount_reds_importers/importers/stanchart/scbcard.py +++ b/beancount_reds_importers/importers/stanchart/scbcard.py @@ -1,77 +1,86 @@ """SCB Credit .csv importer.""" -from beancount_reds_importers.libreader import csvreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import csvreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Card CSV' + IMPORTER_NAME = "SCB Card CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'CardTransactions[0-9]*' - self.header_identifier = self.config.get('custom_header', 'PRIORITY BANKING VISA INFINITE CARD') - self.column_labels_line = 'Date,DESCRIPTION,Foreign Currency Amount,SGD Amount' - self.date_format = '%d/%m/%Y' + self.filename_pattern_def = "CardTransactions[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "PRIORITY BANKING VISA INFINITE CARD" + ) + self.column_labels_line = "Date,DESCRIPTION,Foreign Currency Amount,SGD Amount" + self.date_format = "%d/%m/%Y" self.skip_tail_rows = 6 - self.skip_comments = '# ' + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "DESCRIPTION": "payee", + "Date": "date", + "DESCRIPTION": "payee", } + # fmt: on self.transaction_type_map = {} def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return re.match(self.header_identifier, file.head()) and account_number in file.head() def skip_transaction(self, row): - return '[UNPOSTED]' in row.payee + return "[UNPOSTED]" in row.payee def prepare_table(self, rdr): - rdr = rdr.select(lambda r: 'UNPOSTED' not in r['DESCRIPTION']) + rdr = rdr.select(lambda r: "UNPOSTED" not in r["DESCRIPTION"]) # parse foreign_currency amount: "YEN 74,000" - if self.config.get('convert_currencies', False): + if self.config.get("convert_currencies", False): # Currency conversions won't work as expected since Beancount v2 # doesn't support adding @@ (total price conversions) via code. # See https://groups.google.com/g/beancount/c/nMvuoR4yOmM # This means the '@' generated by this code below needs to be replaced with an '@@' - rdr = rdr.capture('Foreign Currency Amount', '(.*) (.*)', - ['foreign_currency', 'foreign_amount'], - fill=' ', include_original=True) - rdr = rdr.cutout('Foreign Currency Amount') + rdr = rdr.capture( + "Foreign Currency Amount", + "(.*) (.*)", + ["foreign_currency", "foreign_amount"], + fill=" ", + include_original=True, + ) + rdr = rdr.cutout("Foreign Currency Amount") # parse SGD Amount: "SGD 141.02 CR" into a single amount column - rdr = rdr.capture('SGD Amount', '(.*) (.*) (.*)', ['currency', 'amount', 'crdr']) + rdr = rdr.capture("SGD Amount", "(.*) (.*) (.*)", ["currency", "amount", "crdr"]) # change DR into -ve. TODO: move this into csvreader or csvreader.utils - crdrdict = {'DR': '-', 'CR': ''} - rdr = rdr.convert('amount', lambda i, row: crdrdict[row.crdr] + i, pass_row=True) + crdrdict = {"DR": "-", "CR": ""} + rdr = rdr.convert("amount", lambda i, row: crdrdict[row.crdr] + i, pass_row=True) - rdr = rdr.addfield('memo', lambda x: '') # TODO: make this non-mandatory in csvreader + rdr = rdr.addfield("memo", lambda x: "") # TODO: make this non-mandatory in csvreader return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" date = self.get_balance_assertion_date() if date: - balance_row = self.get_row_by_label(file, 'Current Balance') + balance_row = self.get_row_by_label(file, "Current Balance") currency, amount = balance_row[1], balance_row[2] units, debitcredit = amount.split() - if debitcredit != 'CR': - units = '-' + units + if debitcredit != "CR": + units = "-" + units yield banking.Balance(date, D(units), currency) diff --git a/beancount_reds_importers/importers/target/__init__.py b/beancount_reds_importers/importers/target/__init__.py index 1a83926..69b6cc7 100644 --- a/beancount_reds_importers/importers/target/__init__.py +++ b/beancount_reds_importers/importers/target/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Target Credit Card' + IMPORTER_NAME = "Target Credit Card" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = 'Transactions' + self.filename_pattern_def = "Transactions" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/tdameritrade/__init__.py b/beancount_reds_importers/importers/tdameritrade/__init__.py index fd19326..24e4ace 100644 --- a/beancount_reds_importers/importers/tdameritrade/__init__.py +++ b/beancount_reds_importers/importers/tdameritrade/__init__.py @@ -1,17 +1,16 @@ - from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'TDAmeritrade' + IMPORTER_NAME = "TDAmeritrade" def custom_init(self): super(Importer, self).custom_init() self.max_rounding_error = 0.07 - self.filename_pattern_def = '.*tdameritrade' + self.filename_pattern_def = ".*tdameritrade" self.get_ticker_info = self.get_ticker_info_from_id def get_ticker_info(self, security): - ticker = self.config['fund_info']['cusip_map'][security] - return ticker, '' + ticker = self.config["fund_info"]["cusip_map"][security] + return ticker, "" diff --git a/beancount_reds_importers/importers/techcubank/__init__.py b/beancount_reds_importers/importers/techcubank/__init__.py index 725ca1b..a70a185 100644 --- a/beancount_reds_importers/importers/techcubank/__init__.py +++ b/beancount_reds_importers/importers/techcubank/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Tech Credit Union' + IMPORTER_NAME = "Tech Credit Union" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Accounts' + self.filename_pattern_def = ".*Accounts" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py index f1d115b..f989eb4 100644 --- a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py +++ b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py @@ -1,15 +1,19 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers.unitedoverseas import uobbank @regtest.with_importer( - uobbank.Importer({ - 'main_account': 'Assets:Banks:UOB:UNIPLUS', - 'account_number': '1234567890', - 'currency': 'SGD', - 'rounding_error': 'Equity:Rounding-Errors:Imports', - }) + uobbank.Importer( + { + "main_account": "Assets:Banks:UOB:UNIPLUS", + "account_number": "1234567890", + "currency": "SGD", + "rounding_error": "Equity:Rounding-Errors:Imports", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestUOB(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/importers/unitedoverseas/uobbank.py b/beancount_reds_importers/importers/unitedoverseas/uobbank.py index b59fa09..753faf6 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobbank.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobbank.py @@ -1,51 +1,62 @@ """United Overseas Bank, Bank account .csv importer.""" -from beancount_reds_importers.libreader import xlsreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import xlsreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(xlsreader.Importer, banking.Importer): IMPORTER_NAME = __doc__ def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'ACC_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:Uniplus Account') - self.column_labels_line = 'Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance' - self.date_format = '%d %b %Y' + self.filename_pattern_def = "ACC_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", + "United Overseas Bank Limited.*Account Type:Uniplus Account", + ) + self.column_labels_line = ( + "Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance" + ) + self.date_format = "%d %b %Y" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Transaction Description': 'payee', - 'Available Balance': 'balance' + "Transaction Date": "date", + "Transaction Description": "payee", + "Available Balance": "balance", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move these into utils, since this is probably a common operation def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Transaction Description', lambda x: x.replace('\n', ' ')) + rdr = rdr.convert("Transaction Description", lambda x: x.replace("\n", " ")) def Ds(x): return D(str(x)) - rdr = rdr.addfield('amount', - lambda x: -1 * Ds(x['Withdrawal']) if x['Withdrawal'] != 0 else Ds(x['Deposit'])) - rdr = rdr.addfield('memo', lambda x: '') + + rdr = rdr.addfield( + "amount", + lambda x: -1 * Ds(x["Withdrawal"]) if x["Withdrawal"] != 0 else Ds(x["Deposit"]), + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr @@ -55,6 +66,6 @@ def get_balance_statement(self, file=None): if date: row = self.rdr.namedtuples()[0] # Get currency from input file - currency = self.get_row_by_label(file, 'Account Number:')[2] + currency = self.get_row_by_label(file, "Account Number:")[2] yield banking.Balance(date, D(str(row.balance)), currency) diff --git a/beancount_reds_importers/importers/unitedoverseas/uobcard.py b/beancount_reds_importers/importers/unitedoverseas/uobcard.py index 2dfc6b7..5a50f2c 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobcard.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobcard.py @@ -1,68 +1,73 @@ """SCB Credit .csv importer.""" -from beancount_reds_importers.libreader import xlsreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import xlsreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(xlsreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Card CSV' + IMPORTER_NAME = "SCB Card CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '^CC_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:VISA SIGNATURE') - self.column_labels_line = 'Transaction Date,Posting Date,Description,Foreign Currency Type,Transaction Amount(Foreign),Local Currency Type,Transaction Amount(Local)' # noqa: E501 - self.date_format = '%d %b %Y' + self.filename_pattern_def = "^CC_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "United Overseas Bank Limited.*Account Type:VISA SIGNATURE" + ) + self.column_labels_line = "Transaction Date,Posting Date,Description,Foreign Currency Type,Transaction Amount(Foreign),Local Currency Type,Transaction Amount(Local)" # noqa: E501 + self.date_format = "%d %b %Y" # Remove _DISABLED below to include currency conversions. This won't work as expected since # Beancount v2 doesn't support adding @@ (total price conversions) via code. See # https://groups.google.com/g/beancount/c/nMvuoR4yOmM This means the '@' generated by this # code below needs to be replaced with an '@@' - foreign_currency = 'foreign_currency_DISABLED' - foreign_amount = 'foreign_amount_DISABLED' - if self.config.get('convert_currencies', False): - foreign_currency = 'foreign_currency' - foreign_amount = 'foreign_amount' + foreign_currency = "foreign_currency_DISABLED" + foreign_amount = "foreign_amount_DISABLED" + if self.config.get("convert_currencies", False): + foreign_currency = "foreign_currency" + foreign_amount = "foreign_amount" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Posting Date': 'date_posting', - 'Description': 'payee', - 'Foreign Currency Type': foreign_currency, - 'Transaction Amount(Foreign)': foreign_amount, - 'Local Currency Type': 'currency', - 'Transaction Amount(Local)': 'amount' + "Transaction Date": "date", + "Posting Date": "date_posting", + "Description": "payee", + "Foreign Currency Type": foreign_currency, + "Transaction Amount(Foreign)": foreign_amount, + "Local Currency Type": "currency", + "Transaction Amount(Local)": "amount", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Description', lambda x: x.replace('\n', ' ')) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.convert("Description", lambda x: x.replace("\n", " ")) + rdr = rdr.addfield("memo", lambda x: "") # delete empty rows - rdr = rdr.select(lambda x: x['Transaction Date'] != '') + rdr = rdr.select(lambda x: x["Transaction Date"] != "") return rdr def prepare_processed_table(self, rdr): - return rdr.convert('amount', lambda x: -1 * D(str(x))) + return rdr.convert("amount", lambda x: -1 * D(str(x))) def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr @@ -70,6 +75,6 @@ def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" date = self.get_balance_assertion_date() if date: - balance_row = self.get_row_by_label(file, 'Statement Balance:') + balance_row = self.get_row_by_label(file, "Statement Balance:") units, currency = balance_row[1], balance_row[2] yield banking.Balance(date, -1 * D(str(units)), currency) diff --git a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py index 7a689dc..ac6a761 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py @@ -1,48 +1,56 @@ """UOB SRS importer.""" import re + +from beancount.core.number import D + from beancount_reds_importers.libreader import xlsreader from beancount_reds_importers.libtransactionbuilder import banking -from beancount.core.number import D class Importer(xlsreader.Importer, banking.Importer): - IMPORTER_NAME = 'UOB SRS' + IMPORTER_NAME = "UOB SRS" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'SRS_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:SRS Account') - self.column_labels_line = 'Transaction Date,Transaction Description,Withdrawal,Deposit' - self.date_format = '%Y%m%d' + self.filename_pattern_def = "SRS_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "United Overseas Bank Limited.*Account Type:SRS Account" + ) + self.column_labels_line = "Transaction Date,Transaction Description,Withdrawal,Deposit" + self.date_format = "%Y%m%d" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Transaction Description': 'payee', + "Transaction Date": "date", + "Transaction Description": "payee", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return re.match(self.header_identifier, file.head()) and account_number in file.head() def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Transaction Description', lambda x: x.replace('\n', ' ')) + rdr = rdr.convert("Transaction Description", lambda x: x.replace("\n", " ")) def Ds(x): return D(str(x)) - rdr = rdr.addfield('amount', - lambda x: -1 * Ds(x['Withdrawal']) if x['Withdrawal'] != '' else Ds(x['Deposit'])) - rdr = rdr.addfield('memo', lambda x: '') + + rdr = rdr.addfield( + "amount", + lambda x: -1 * Ds(x["Withdrawal"]) if x["Withdrawal"] != "" else Ds(x["Deposit"]), + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr diff --git a/beancount_reds_importers/importers/vanguard/__init__.py b/beancount_reds_importers/importers/vanguard/__init__.py index dc2d4f2..76f48e9 100644 --- a/beancount_reds_importers/importers/vanguard/__init__.py +++ b/beancount_reds_importers/importers/vanguard/__init__.py @@ -1,12 +1,13 @@ -""" Vanguard Brokerage ofx importer.""" +"""Vanguard Brokerage ofx importer.""" import ntpath + from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Vanguard' + IMPORTER_NAME = "Vanguard" # Any memo in the source OFX that's in this set is not carried forward. # Vanguard sets memos that aren't very useful and would create noise in the @@ -17,7 +18,7 @@ class Importer(investments.Importer, ofxreader.Importer): def custom_init(self): self.max_rounding_error = 0.11 - self.filename_pattern_def = '.*OfxDownload' + self.filename_pattern_def = ".*OfxDownload" self.get_ticker_info = self.get_ticker_info_from_id self.get_payee = self.cleanup_memo @@ -31,21 +32,21 @@ def custom_init(self): self.price_cost_both_zero_handler = lambda *args: None def file_name(self, file): - return 'vanguard-all-{}'.format(ntpath.basename(file.name)) + return "vanguard-all-{}".format(ntpath.basename(file.name)) def get_target_acct_custom(self, transaction, ticker=None): - if 'LT CAP GAIN' in transaction.memo: - return self.config['capgainsd_lt'] - elif 'ST CAP GAIN' in transaction.memo: - return self.config['capgainsd_st'] + if "LT CAP GAIN" in transaction.memo: + return self.config["capgainsd_lt"] + elif "ST CAP GAIN" in transaction.memo: + return self.config["capgainsd_st"] return None def cleanup_memo(self, ot): # some vanguard files have memos repeated like this: # 'DIVIDEND REINVESTMENTDIVIDEND REINVESTMENT' retval = ot.memo - if ot.memo[:int(len(ot.memo)/2)] == ot.memo[int(len(ot.memo)/2):]: - retval = ot.memo[:int(len(ot.memo)/2)] + if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2) :]: + retval = ot.memo[: int(len(ot.memo) / 2)] return retval # For users to comment out in their local file if they so prefer diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py b/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py index 379661b..8473b32 100644 --- a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import vanguard diff --git a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py index ecb8b81..75ba296 100644 --- a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py +++ b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py @@ -1,4 +1,4 @@ -""" Vanguard screenscrape importer. Unsettled trades are not available in Vanguard's qfx and need to be +"""Vanguard screenscrape importer. Unsettled trades are not available in Vanguard's qfx and need to be screenscrapped into a tsv""" from beancount_reds_importers.libreader import tsvreader @@ -6,54 +6,74 @@ class Importer(investments.Importer, tsvreader.Importer): - IMPORTER_NAME = 'Vanguard screenscrape tsv' + IMPORTER_NAME = "Vanguard screenscrape tsv" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*vanguardss.*' - self.header_identifier = '' + self.filename_pattern_def = ".*vanguardss.*" + self.header_identifier = "" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "date": 'date', - "settledate": 'tradeDate', - "symbol": 'security', - "description": 'memo', - "action": 'type', - "quantity": 'units', - "price": 'unit_price', - "fees": 'fees', - "amount": 'amount', - "total": 'total', - } + "date": "date", + "settledate": "tradeDate", + "symbol": "security", + "description": "memo", + "action": "type", + "quantity": "units", + "price": "unit_price", + "fees": "fees", + "amount": "amount", + "total": "total", + } self.transaction_type_map = { - 'Buy': 'buystock', - 'Sell': 'sellstock', - } - self.skip_transaction_types = [''] + "Buy": "buystock", + "Sell": "sellstock", + } + # fmt: on + self.skip_transaction_types = [""] def prepare_table(self, rdr): def extract_numbers(x): - replacements = {'– ': '-', - '$': '', - ',': '', - 'Free': '0', - } + replacements = { + "– ": "-", + "$": "", + ",": "", + "Free": "0", + } for k, v in replacements.items(): x = x.replace(k, v) return x - header = ('date', 'settledate', 'symbol', 'description', 'quantity', 'price', 'fees', 'amount') + header = ( + "date", + "settledate", + "symbol", + "description", + "quantity", + "price", + "fees", + "amount", + ) rdr = rdr.pushheader(header) - rdr = rdr.addfield('action', lambda x: x['description'].rsplit(' ', 2)[1].strip()) + rdr = rdr.addfield("action", lambda x: x["description"].rsplit(" ", 2)[1].strip()) - for field in ["date", "settledate", "symbol", "description", "quantity", "price", "fees"]: + for field in [ + "date", + "settledate", + "symbol", + "description", + "quantity", + "price", + "fees", + ]: rdr = rdr.convert(field, lambda x: x.strip()) for field in ["quantity", "amount", "price", "fees"]: rdr = rdr.convert(field, extract_numbers) - rdr = rdr.addfield('total', lambda x: x['amount']) + rdr = rdr.addfield("total", lambda x: x["amount"]) return rdr diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index eb0e55d..2a1f08b 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -1,6 +1,7 @@ -""" Workday paycheck importer.""" +"""Workday paycheck importer.""" import datetime + from beancount_reds_importers.libreader import xlsx_multitable_reader from beancount_reds_importers.libtransactionbuilder import paycheck @@ -15,33 +16,35 @@ class Importer(paycheck.Importer, xlsx_multitable_reader.Importer): - IMPORTER_NAME = 'Workday Paycheck' + IMPORTER_NAME = "Workday Paycheck" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Complete' - self.header_identifier = '- Complete' + self.config.get('custom_header', '') - self.date_format = '%m/%d/%Y' + self.filename_pattern_def = ".*_Complete" + self.header_identifier = "- Complete" + self.config.get("custom_header", "") + self.date_format = "%m/%d/%Y" self.skip_head_rows = 1 # TODO: need to be smarter about this, and skip only when needed self.skip_tail_rows = 0 - self.funds_db_txt = 'funds_by_ticker' + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on def paycheck_date(self, input_file): self.read_file(input_file) - d = self.alltables['Payslip Information'].namedtuples()[0].check_date + d = self.alltables["Payslip Information"].namedtuples()[0].check_date self.date = datetime.datetime.strptime(d, self.date_format) return self.date.date() def prepare_tables(self): def valid_header_label(label): - return label.lower().replace(' ', '_') + return label.lower().replace(" ", "_") for section, table in self.alltables.items(): for header in table.header(): @@ -49,4 +52,5 @@ def valid_header_label(label): self.alltables[section] = table def build_metadata(self, file, metatype=None, data={}): - return {'filing_account': self.config['main_account']} + acct = self.config.get("filing_account", self.config.get("main_account", None)) + return {"filing_account": acct} diff --git a/beancount_reds_importers/libreader/csv_multitable_reader.py b/beancount_reds_importers/libreader/csv_multitable_reader.py index 8a8f4ef..6cf320b 100644 --- a/beancount_reds_importers/libreader/csv_multitable_reader.py +++ b/beancount_reds_importers/libreader/csv_multitable_reader.py @@ -59,22 +59,26 @@ def read_file(self, file): self.raw_rdr = rdr = self.read_raw(file) - rdr = rdr.skip(getattr(self, 'skip_head_rows', 0)) # chop unwanted file header rows - rdr = rdr.head(len(rdr) - getattr(self, 'skip_tail_rows', 0) - 1) # chop unwanted file footer rows + rdr = rdr.skip(getattr(self, "skip_head_rows", 0)) # chop unwanted file header rows + rdr = rdr.head( + len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 + ) # chop unwanted file footer rows # [0, 2, 10] <-- starts # [-1, 1, 9] <-- ends - table_starts = [i for (i, row) in enumerate(rdr) if self.is_section_title(row)] + [len(rdr)] - table_ends = [r-1 for r in table_starts][1:] + table_starts = [i for (i, row) in enumerate(rdr) if self.is_section_title(row)] + [ + len(rdr) + ] + table_ends = [r - 1 for r in table_starts][1:] table_indexes = zip(table_starts, table_ends) # build the dictionary of tables self.alltables = {} - for (s, e) in table_indexes: + for s, e in table_indexes: if s == e: continue - table = rdr.skip(s+1) # skip past start index and header row - table = table.head(e-s-1) # chop lines after table section data + table = rdr.skip(s + 1) # skip past start index and header row + table = table.head(e - s - 1) # chop lines after table section data self.alltables[rdr[s][0]] = table for section, table in self.alltables.items(): diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 84a3719..0bd11af 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -3,14 +3,16 @@ import datetime import re +import sys import traceback -from beancount.ingest import importer -from beancount.core.number import D + import petl as etl +from beancount.core.number import D +from beancount.ingest import importer + from beancount_reds_importers.libreader import reader -import sys -# This csv reader uses petl to read a .csv into a table for maniupulation. The output of this reader is a list +# This csv reader uses petl to read a .csv into a table for manipulation. The output of this reader is a list # of namedtuples corresponding roughly to ofx transactions. The following steps achieve this. When writing # your own importer, you only should need to: # - override prepare_table() @@ -59,10 +61,10 @@ class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['csv'] + FILE_EXTS = ["csv"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.reader_ready = self.deep_identify(file) if self.reader_ready: @@ -72,7 +74,10 @@ def initialize_reader(self, file): # print(self.header_identifier, file.head()) def deep_identify(self, file): - return re.match(self.header_identifier, file.head()) + return re.match( + self.header_identifier, + file.head(encoding=getattr(self, "file_encoding", None)), + ) def file_date(self, file): "Get the maximum date from the file." @@ -96,19 +101,26 @@ def prepare_processed_table(self, rdr): def convert_columns(self, rdr): # convert data in transaction types column - if 'type' in rdr.header(): - rdr = rdr.convert('type', self.transaction_type_map) + if "type" in rdr.header(): + rdr = rdr.convert("type", self.transaction_type_map) # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: if i in rdr.header(): rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.-]', "", str(x).strip()) # noqa: W605 - currencies = getattr(self, 'currency_fields', []) + ['unit_price', 'fees', 'total', 'amount', 'balance'] + return re.sub(r"[^0-9\.-]", "", str(x).strip()) # noqa: W605 + + currencies = getattr(self, "currency_fields", []) + [ + "unit_price", + "fees", + "total", + "amount", + "balance", + ] for i in currencies: if i in rdr.header(): rdr = rdr.convert(i, remove_non_numeric) @@ -117,7 +129,8 @@ def remove_non_numeric(x): # fixup dates def convert_date(d): return datetime.datetime.strptime(d, self.date_format) - dates = getattr(self, 'date_fields', []) + ['date', 'tradeDate', 'settleDate'] + + dates = getattr(self, "date_fields", []) + ["date", "tradeDate", "settleDate"] for i in dates: if i in rdr.header(): rdr = rdr.convert(i, convert_date) @@ -125,14 +138,14 @@ def convert_date(d): return rdr def read_raw(self, file): - return etl.fromcsv(file.name) + return etl.fromcsv(file.name, encoding=getattr(self, "file_encoding", None)) def skip_until_main_table(self, rdr, col_labels=None): """Skip csv lines until the header line is found.""" # TODO: convert this into an 'extract_table()' method that handles the tail as well if not col_labels: - if hasattr(self, 'column_labels_line'): - col_labels = self.column_labels_line.replace('"', '').split(',') + if hasattr(self, "column_labels_line"): + col_labels = self.column_labels_line.replace('"', "").split(",") else: return rdr skip = None @@ -151,8 +164,8 @@ def skip_until_main_table(self, rdr, col_labels=None): def extract_table_with_header(self, rdr, col_labels=None): rdr = self.skip_until_main_table(rdr, col_labels) nrows = len(rdr) - for (n, r) in enumerate(rdr): - if not r or all(i == '' for i in r): + for n, r in enumerate(rdr): + if not r or all(i == "" for i in r): # blank line, terminate nrows = n - 1 break @@ -170,18 +183,20 @@ def skip_until_row_contains(self, rdr, value): return rdr.rowslice(start, len(rdr)) def read_file(self, file): - if not getattr(self, 'file_read_done', False): + if not getattr(self, "file_read_done", False): # read file rdr = self.read_raw(file) rdr = self.prepare_raw_file(rdr) # extract main table - rdr = rdr.skip(getattr(self, 'skip_head_rows', 0)) # chop unwanted header rows - rdr = rdr.head(len(rdr) - getattr(self, 'skip_tail_rows', 0) - 1) # chop unwanted footer rows + rdr = rdr.skip(getattr(self, "skip_head_rows", 0)) # chop unwanted header rows + rdr = rdr.head( + len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 + ) # chop unwanted footer rows rdr = self.extract_table_with_header(rdr) - if hasattr(self, 'skip_comments'): + if hasattr(self, "skip_comments"): rdr = rdr.skipcomments(self.skip_comments) - rdr = rdr.rowslice(getattr(self, 'skip_data_rows', 0), None) + rdr = rdr.rowslice(getattr(self, "skip_data_rows", 0), None) rdr = self.prepare_table(rdr) # process table @@ -201,7 +216,7 @@ def get_transactions(self): # TOOD: custom, overridable def skip_transaction(self, row): - return getattr(row, 'type', 'NO_TYPE') in self.skip_transaction_types + return getattr(row, "type", "NO_TYPE") in self.skip_transaction_types def get_balance_assertion_date(self): """ @@ -220,8 +235,10 @@ def get_max_transaction_date(self): # TODO: clean this up. this probably suffices: # return max(ot.date for ot in self.get_transactions()).date() - date = max(ot.tradeDate if hasattr(ot, 'tradeDate') else ot.date - for ot in self.get_transactions()).date() + date = max( + ot.tradeDate if hasattr(ot, "tradeDate") else ot.date + for ot in self.get_transactions() + ).date() except Exception as err: print("ERROR: no end_date. SKIPPING input.") traceback.print_tb(err.__traceback__) diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 5ae1e82..38ffc8f 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -1,4 +1,3 @@ - """JSON importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers. @@ -12,23 +11,27 @@ Until that happens, perhaps this file should be renamed to schwabjsonreader.py. """ +import json + +# import re +import warnings + # import datetime # import ofxparse # from collections import namedtuple from beancount.ingest import importer -from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning -import json -# import re -import warnings + +from beancount_reds_importers.libreader import reader + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['json'] + FILE_EXTS = ["json"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.reader_ready = self.deep_identify(file) if self.reader_ready: @@ -61,8 +64,6 @@ def read_file(self, file): # total = transaction['Amount'] # ) - - # def get_transactions(self): # Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', # 'units', 'fees', 'total']) diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index b40970e..d4b2a69 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -2,20 +2,23 @@ beancount_reds_importers.""" import datetime -import ofxparse +import warnings from collections import namedtuple + +import ofxparse from beancount.ingest import importer -from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning -import warnings + +from beancount_reds_importers.libreader import reader + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['ofx', 'qfx'] + FILE_EXTS = ["ofx", "qfx"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.ofx_account = None self.reader_ready = False @@ -26,9 +29,10 @@ def initialize_reader(self, file): for acc in self.ofx.accounts: # account identifying info fieldname varies across institutions # self.acc_num_field can be overridden in self.custom_init() if needed - acc_num_field = getattr(self, 'account_number_field', 'account_id') - if self.match_account_number(getattr(acc, acc_num_field), - self.config['account_number']): + acc_num_field = getattr(self, "account_number_field", "account_id") + if self.match_account_number( + getattr(acc, acc_num_field), self.config["account_number"] + ): self.ofx_account = acc self.reader_ready = True if self.reader_ready: @@ -41,7 +45,7 @@ def match_account_number(self, file_account, config_account): def file_date(self, file): """Get the ending date of the statement.""" - if not getattr(self, 'ofx_account', None): + if not getattr(self, "ofx_account", None): self.initialize(file) try: return self.ofx_account.statement.end_date @@ -56,27 +60,27 @@ def get_transactions(self): yield from self.ofx_account.statement.transactions def get_balance_statement(self, file=None): - if not hasattr(self.ofx_account.statement, 'balance'): + if not hasattr(self.ofx_account.statement, "balance"): return [] date = self.get_balance_assertion_date() if date: - Balance = namedtuple('Balance', ['date', 'amount']) + Balance = namedtuple("Balance", ["date", "amount"]) yield Balance(date, self.ofx_account.statement.balance) def get_balance_positions(self): - if not hasattr(self.ofx_account.statement, 'positions'): + if not hasattr(self.ofx_account.statement, "positions"): return [] yield from self.ofx_account.statement.positions def get_available_cash(self, settlement_fund_balance=0): - available_cash = getattr(self.ofx_account.statement, 'available_cash', None) + available_cash = getattr(self.ofx_account.statement, "available_cash", None) if available_cash is not None: # Some institutions compute available_cash this way. For others, override this method # in the importer return available_cash - settlement_fund_balance return None - def get_ofx_end_date(self, field='end_date'): + def get_ofx_end_date(self, field="end_date"): end_date = getattr(self.ofx_account.statement, field, None) if end_date: @@ -86,7 +90,7 @@ def get_ofx_end_date(self, field='end_date'): return None def get_smart_date(self): - """ We want the latest date we can assert balance on. Let's consider all the dates we have: + """We want the latest date we can assert balance on. Let's consider all the dates we have: b--------e-------(s-2)----(s)----(d) - b: date of first transaction in this ofx file (end_date) @@ -105,28 +109,41 @@ def get_smart_date(self): have. """ - ofx_max_transation_date = self.get_ofx_end_date('end_date') - ofx_balance_date1 = self.get_ofx_end_date('available_balance_date') - ofx_balance_date2 = self.get_ofx_end_date('balance_date') + ofx_max_transation_date = self.get_ofx_end_date("end_date") + ofx_balance_date1 = self.get_ofx_end_date("available_balance_date") + ofx_balance_date2 = self.get_ofx_end_date("balance_date") max_transaction_date = self.get_max_transaction_date() if ofx_balance_date1: - ofx_balance_date1 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) + ofx_balance_date1 -= datetime.timedelta( + days=self.config.get("balance_assertion_date_fudge", 2) + ) if ofx_balance_date2: - ofx_balance_date2 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) - - dates = [ofx_max_transation_date, max_transaction_date, ofx_balance_date1, ofx_balance_date2] - if all(v is None for v in dates[:2]): # because ofx_balance_date appears even for closed accounts + ofx_balance_date2 -= datetime.timedelta( + days=self.config.get("balance_assertion_date_fudge", 2) + ) + + dates = [ + ofx_max_transation_date, + max_transaction_date, + ofx_balance_date1, + ofx_balance_date2, + ] + if all( + v is None for v in dates[:2] + ): # because ofx_balance_date appears even for closed accounts return None - def vd(x): return x if x else datetime.date.min + def vd(x): + return x if x else datetime.date.min + return_date = max(*[vd(x) for x in dates]) # print("Smart date computation. Dates were: ", dates) return return_date def get_balance_assertion_date(self): - """ Choices for the date of the generated balance assertion can be specified in + """Choices for the date of the generated balance assertion can be specified in self.config['balance_assertion_date_type'], which can be: - 'smart': smart date (default) - 'ofx_date': date specified in ofx file @@ -139,11 +156,13 @@ def get_balance_assertion_date(self): on the beginning of the assertion date. """ - date_type_map = {'smart': self.get_smart_date, - 'ofx_date': self.get_ofx_end_date, - 'last_transaction': self.get_max_transaction_date, - 'today': datetime.date.today} - date_type = self.config.get('balance_assertion_date_type', 'smart') + date_type_map = { + "smart": self.get_smart_date, + "ofx_date": self.get_ofx_end_date, + "last_transaction": self.get_max_transaction_date, + "today": datetime.date.today, + } + date_type = self.config.get("balance_assertion_date_type", "smart") return_date = date_type_map[date_type]() if not return_date: @@ -160,9 +179,10 @@ def get_max_transaction_date(self): """ try: - - date = max(ot.tradeDate if hasattr(ot, 'tradeDate') else ot.date - for ot in self.get_transactions()).date() + date = max( + ot.tradeDate if hasattr(ot, "tradeDate") else ot.date + for ot in self.get_transactions() + ).date() except TypeError: return None except ValueError: diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index 9efc9b6..f93aa2b 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -1,13 +1,13 @@ """Reader module base class for beancount_reds_importers. ofx, csv, etc. readers inherit this.""" import ntpath -from os import path import re +from os import path -class Reader(): - FILE_EXTS = [''] - IMPORTER_NAME = 'NOT SET' +class Reader: + FILE_EXTS = [""] + IMPORTER_NAME = "NOT SET" def identify(self, file): # quick check to filter out files that are not the right format @@ -18,17 +18,17 @@ def identify(self, file): # print("No match on extension") return False self.custom_init() - self.filename_pattern = self.config.get('filename_pattern', self.filename_pattern_def) + self.filename_pattern = self.config.get("filename_pattern", self.filename_pattern_def) if not re.match(self.filename_pattern, path.basename(file.name)): # print("No match on filename_pattern", self.filename_pattern, path.basename(file.name)) return False - self.currency = self.config.get('currency', 'CURRENCY_NOT_CONFIGURED') + self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED") self.initialize_reader(file) # print("reader_ready:", self.reader_ready) return self.reader_ready def file_name(self, file): - return '{}'.format(ntpath.basename(file.name)) + return "{}".format(ntpath.basename(file.name)) def file_account(self, file): # Ugly hack to handle an interaction with smart_importer. See: @@ -36,17 +36,18 @@ def file_account(self, file): # https://github.com/beancount/smart_importer/issues/122 # https://github.com/beancount/smart_importer/issues/30 import inspect + curframe = inspect.currentframe() calframe = inspect.getouterframes(curframe, 2) - if any('predictor' in i.filename for i in calframe): - if 'smart_importer_hack' in self.config: - return self.config['smart_importer_hack'] + if any("predictor" in i.filename for i in calframe): + if "smart_importer_hack" in self.config: + return self.config["smart_importer_hack"] # Otherwise handle a typical bean-file call self.initialize(file) - if 'filing_account' in self.config: - return self.config['filing_account'] - return self.config['main_account'] + if "filing_account" in self.config: + return self.config["filing_account"] + return self.config["main_account"] def get_balance_statement(self, file=None): return [] diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py index 848e283..fb4b451 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py @@ -1,14 +1,18 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - "balance_assertion_date_type": "last_transaction", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + "balance_assertion_date_type": "last_transaction", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py index 0a1a210..b13ac82 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py @@ -1,14 +1,18 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - "balance_assertion_date_type": "ofx_date", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + "balance_assertion_date_type": "ofx_date", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py index ab7cecd..a194f50 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py @@ -1,14 +1,18 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally # default balance_assertion_date_type is "smart" @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tsvreader.py b/beancount_reds_importers/libreader/tsvreader.py index 4ab4560..4f8fad8 100644 --- a/beancount_reds_importers/libreader/tsvreader.py +++ b/beancount_reds_importers/libreader/tsvreader.py @@ -1,14 +1,14 @@ """tsv (tab separated values) importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" - -from beancount.ingest import importer import petl as etl +from beancount.ingest import importer + from beancount_reds_importers.libreader import csvreader class Importer(csvreader.Importer, importer.ImporterProtocol): - FILE_EXTS = ['tsv'] + FILE_EXTS = ["tsv"] def read_raw(self, file): return etl.fromtsv(file.name) diff --git a/beancount_reds_importers/libreader/xlsreader.py b/beancount_reds_importers/libreader/xlsreader.py index e2077a3..765357e 100644 --- a/beancount_reds_importers/libreader/xlsreader.py +++ b/beancount_reds_importers/libreader/xlsreader.py @@ -1,26 +1,28 @@ """xlsx importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" -import petl as etl import re -from beancount_reds_importers.libreader import csvreader from os import devnull +import petl as etl + +from beancount_reds_importers.libreader import csvreader + class Importer(csvreader.Importer): - FILE_EXTS = ['xls'] + FILE_EXTS = ["xls"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.file_read_done = False self.reader_ready = False # TODO: this reads the entire file. Chop off after perhaps 2k or n lines rdr = self.read_raw(file) - header = '' + header = "" for r in rdr: - line = ''.join(str(x) for x in r) + line = "".join(str(x) for x in r) header += line # TODO @@ -33,4 +35,4 @@ def initialize_reader(self, file): def read_raw(self, file): # set logfile to ignore WARNING *** file size (92598) not 512 + multiple of sector size (512) - return etl.fromxls(file.name, logfile=open(devnull, 'w')) + return etl.fromxls(file.name, logfile=open(devnull, "w")) diff --git a/beancount_reds_importers/libreader/xlsx_multitable_reader.py b/beancount_reds_importers/libreader/xlsx_multitable_reader.py index 8274988..d9d12c0 100644 --- a/beancount_reds_importers/libreader/xlsx_multitable_reader.py +++ b/beancount_reds_importers/libreader/xlsx_multitable_reader.py @@ -1,11 +1,13 @@ """xlsx importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" -import petl as etl -from io import StringIO import csv -import openpyxl import warnings +from io import StringIO + +import openpyxl +import petl as etl + from beancount_reds_importers.libreader import csv_multitable_reader # This xlsx reader uses petl to read a .csv with multiple tables into a dictionary of petl tables. The section @@ -13,10 +15,10 @@ class Importer(csv_multitable_reader.Importer): - FILE_EXTS = ['xlsx'] + FILE_EXTS = ["xlsx"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.file_read_done = False self.reader_ready = True @@ -43,4 +45,4 @@ def read_raw(self, file): def is_section_title(self, row): if len(row) == 1: return True - return all(i == '' or i is None for i in row[1:]) + return all(i == "" or i is None for i in row[1:]) diff --git a/beancount_reds_importers/libreader/xlsxreader.py b/beancount_reds_importers/libreader/xlsxreader.py index 06199d0..7235766 100644 --- a/beancount_reds_importers/libreader/xlsxreader.py +++ b/beancount_reds_importers/libreader/xlsxreader.py @@ -2,11 +2,12 @@ beancount_reds_importers.""" import petl as etl + from beancount_reds_importers.libreader import xlsreader class Importer(xlsreader.Importer): - FILE_EXTS = ['xlsx'] + FILE_EXTS = ["xlsx"] def read_raw(self, file): rdr = etl.fromxlsx(file.name) diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index 21b0fa7..c97791a 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -2,13 +2,13 @@ import itertools from collections import namedtuple -from beancount.core import data -from beancount.core import amount + +from beancount.core import amount, data from beancount.ingest import importer -from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder +from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder -Balance = namedtuple('Balance', ['date', 'amount', 'currency']) +Balance = namedtuple("Balance", ["date", "amount", "currency"]) class Importer(importer.ImporterProtocol, transactionbuilder.TransactionBuilder): @@ -43,19 +43,13 @@ def build_account_map(self): # } pass - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} - def match_account_number(self, file_account, config_account): return file_account.endswith(config_account) def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*bank_specific_filename.*' + self.filename_pattern_def = ".*bank_specific_filename.*" self.custom_init_run = True # def get_target_acct(self, transaction): @@ -68,11 +62,11 @@ def fields_contain_data(ot, fields): def get_main_account(self, ot): """Can be overridden by importer""" - return self.config['main_account'] + return self.config["main_account"] def get_target_account(self, ot): """Can be overridden by importer""" - return self.config.get('target_account') + return self.config.get("target_account") # -------------------------------------------------------------------------------- @@ -82,10 +76,15 @@ def extract_balance(self, file, counter): for bal in self.get_balance_statement(file=file): if bal: metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance')) - balance_entry = data.Balance(metadata, bal.date, self.config['main_account'], - amount.Amount(bal.amount, self.get_currency(bal)), - None, None) + metadata.update(self.build_metadata(file, metatype="balance")) + balance_entry = data.Balance( + metadata, + bal.date, + self.config["main_account"], + amount.Amount(bal.amount, self.get_currency(bal)), + None, + None, + ) entries.append(balance_entry) return entries @@ -110,9 +109,9 @@ def extract(self, file, existing_entries=None): continue metadata = data.new_metadata(file.name, next(counter)) # metadata['type'] = ot.type # Optional metadata, useful for debugging #TODO - metadata.update(self.build_metadata(file, - metatype='transaction', - data={'transaction': ot})) + metadata.update( + self.build_metadata(file, metatype="transaction", data={"transaction": ot}) + ) # description fields: With OFX, ot.payee tends to be the "main" description field, # while ot.memo is optional @@ -125,22 +124,28 @@ def extract(self, file, existing_entries=None): # Banking transactions might include foreign currency transactions. TODO: figure out # how ofx handles this and use the same interface for csv and other files entry = data.Transaction( - meta=metadata, - date=ot.date.date(), - flag=self.FLAG, - # payee and narration are switched. See the preceding note - payee=self.get_narration(ot), - narration=self.get_payee(ot), - tags=self.get_tags(ot), - links=data.EMPTY_SET, - postings=[]) + meta=metadata, + date=ot.date.date(), + flag=self.FLAG, + # payee and narration are switched. See the preceding note + payee=self.get_narration(ot), + narration=self.get_payee(ot), + tags=self.get_tags(ot), + links=data.EMPTY_SET, + postings=[], + ) main_account = self.get_main_account(ot) - if self.fields_contain_data(ot, ['foreign_amount', 'foreign_currency']): - common.create_simple_posting_with_price(entry, main_account, - ot.amount, self.get_currency(ot), - ot.foreign_amount, ot.foreign_currency) + if self.fields_contain_data(ot, ["foreign_amount", "foreign_currency"]): + common.create_simple_posting_with_price( + entry, + main_account, + ot.amount, + self.get_currency(ot), + ot.foreign_amount, + ot.foreign_currency, + ) else: data.create_simple_posting(entry, main_account, ot.amount, self.get_currency(ot)) @@ -149,6 +154,7 @@ def extract(self, file, existing_entries=None): if target_acct: data.create_simple_posting(entry, target_acct, None, None) + self.add_custom_postings(entry, ot) new_entries.append(entry) new_entries += self.extract_balance(file, counter) diff --git a/beancount_reds_importers/libtransactionbuilder/common.py b/beancount_reds_importers/libtransactionbuilder/common.py index 7d1fa79..c348bb2 100644 --- a/beancount_reds_importers/libtransactionbuilder/common.py +++ b/beancount_reds_importers/libtransactionbuilder/common.py @@ -2,38 +2,61 @@ from beancount.core import data from beancount.core.amount import Amount +from beancount.core.number import D, Decimal from beancount.core.position import Cost -from beancount.core.number import Decimal -from beancount.core.number import D class PriceCostBothZeroException(Exception): """Raised when the input value is too small""" + pass -def create_simple_posting_with_price(entry, account, - number, currency, - price_number, price_currency): - return create_simple_posting_with_cost_or_price(entry, account, - number, currency, - price_number=price_number, price_currency=price_currency) +def create_simple_posting_with_price( + entry, account, number, currency, price_number, price_currency +): + return create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + price_number=price_number, + price_currency=price_currency, + ) -def create_simple_posting_with_cost(entry, account, - number, currency, - cost_number, cost_currency, price_cost_both_zero_handler=None): - return create_simple_posting_with_cost_or_price(entry, account, - number, currency, - cost_number=cost_number, cost_currency=cost_currency, - price_cost_both_zero_handler=price_cost_both_zero_handler) +def create_simple_posting_with_cost( + entry, + account, + number, + currency, + cost_number, + cost_currency, + price_cost_both_zero_handler=None, +): + return create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + cost_number=cost_number, + cost_currency=cost_currency, + price_cost_both_zero_handler=price_cost_both_zero_handler, + ) -def create_simple_posting_with_cost_or_price(entry, account, - number, currency, - price_number=None, price_currency=None, - cost_number=None, cost_currency=None, costspec=None, - price_cost_both_zero_handler=None): +def create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + price_number=None, + price_currency=None, + cost_number=None, + cost_currency=None, + costspec=None, + price_cost_both_zero_handler=None, +): """Create a simple posting on the entry, with a cost (for purchases) or price (for sell transactions). Args: @@ -59,7 +82,11 @@ def create_simple_posting_with_cost_or_price(entry, account, if price_cost_both_zero_handler: price_cost_both_zero_handler() else: - print("WARNING: Either price ({}) or cost ({}) must be specified ({})".format(price_number, cost_number, entry)) + print( + "WARNING: Either price ({}) or cost ({}) must be specified ({})".format( + price_number, cost_number, entry + ) + ) raise PriceCostBothZeroException # import pdb; pdb.set_trace() diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index f0b9646..12ecd63 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -3,10 +3,11 @@ import itertools import sys -from beancount.core import data -from beancount.core import amount -from beancount.ingest import importer + +from beancount.core import amount, data from beancount.core.position import CostSpec +from beancount.ingest import importer + from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder @@ -83,75 +84,82 @@ def initialize(self, file): self.initialize_reader(file) if self.reader_ready: - config_subst_vars = {'currency': self.currency, - # Leave the other values as is - 'ticker': '{ticker}', - 'source401k': '{source401k}', - } + config_subst_vars = { + "currency": self.currency, + # Leave the other values as is + "ticker": "{ticker}", + "source401k": "{source401k}", + } self.set_config_variables(config_subst_vars) - self.money_market_funds = self.config['fund_info']['money_market'] - self.fund_data = self.config['fund_info']['fund_data'] # [(ticker, id, long_name), ...] + self.money_market_funds = self.config["fund_info"]["money_market"] + self.fund_data = self.config["fund_info"][ + "fund_data" + ] # [(ticker, id, long_name), ...] self.funds_by_id = {i: (ticker, desc) for ticker, i, desc in self.fund_data} self.funds_by_ticker = {ticker: (ticker, desc) for ticker, _, desc in self.fund_data} # Most ofx/csv files refer to funds by id (cusip/isin etc.) Some use tickers instead - self.funds_db = getattr(self, getattr(self, 'funds_db_txt', 'funds_by_id')) + self.funds_db = getattr(self, getattr(self, "funds_db_txt", "funds_by_id")) self.build_account_map() self.initialized = True def build_account_map(self): + # fmt: off # map transaction types to target posting accounts self.target_account_map = { - "buymf": self.config['cash_account'], - "sellmf": self.config['cash_account'], - "buystock": self.config['cash_account'], - "sellstock": self.config['cash_account'], - "buyother": self.config['cash_account'], - "sellother": self.config['cash_account'], - "buydebt": self.config['cash_account'], - "reinvest": self.config['dividends'], - "dividends": self.config['dividends'], - "capgainsd_lt": self.config['capgainsd_lt'], - "capgainsd_st": self.config['capgainsd_st'], - "income": self.config['interest'], - "fee": self.config['fees'], - "invexpense": self.config.get('invexpense', "ACCOUNT_NOT_CONFIGURED:INVEXPENSE"), + "buymf": self.config["cash_account"], + "sellmf": self.config["cash_account"], + "buystock": self.config["cash_account"], + "sellstock": self.config["cash_account"], + "buyother": self.config["cash_account"], + "sellother": self.config["cash_account"], + "buydebt": self.config["cash_account"], + "reinvest": self.config["dividends"], + "dividends": self.config["dividends"], + "capgainsd_lt": self.config["capgainsd_lt"], + "capgainsd_st": self.config["capgainsd_st"], + "income": self.config["interest"], + "fee": self.config["fees"], + "invexpense": self.config.get("invexpense", "ACCOUNT_NOT_CONFIGURED:INVEXPENSE"), } - - if 'transfer' in self.config: - self.target_account_map.update({ - "other": self.config['transfer'], - "credit": self.config['transfer'], - "debit": self.config['transfer'], - "transfer": self.config['transfer'], - "cash": self.config['transfer'], - "dep": self.config['transfer'], - }) - - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} + # fmt: on + + if "transfer" in self.config: + # fmt: off + self.target_account_map.update( + { + "other": self.config["transfer"], + "credit": self.config["transfer"], + "debit": self.config["transfer"], + "transfer": self.config["transfer"], + "cash": self.config["transfer"], + "dep": self.config["transfer"], + } + ) + # fmt: on def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*bank_specific_filename.*' + self.filename_pattern_def = ".*bank_specific_filename.*" self.custom_init_run = True def get_ticker_info(self, security_id): - return security_id, 'UNKNOWN' + return security_id, "UNKNOWN" def get_ticker_info_from_id(self, security_id): try: # isin might look like "US293409829" while the ofx use only a substring like "29340982" ticker = None try: # first try a full match, fall back to substring - ticker, ticker_long_name = [v for k, v in self.funds_db.items() if security_id == k][0] + ticker, ticker_long_name = [ + v for k, v in self.funds_db.items() if security_id == k + ][0] except IndexError: - ticker, ticker_long_name = [v for k, v in self.funds_db.items() if security_id in k][0] + ticker, ticker_long_name = [ + v for k, v in self.funds_db.items() if security_id in k + ][0] except IndexError: print(f"Error: fund info not found for {security_id}", file=sys.stderr) securities = self.get_security_list() @@ -189,8 +197,8 @@ def get_target_acct(self, transaction, ticker): target = self.get_target_acct_custom(transaction, ticker) if target: return target - if transaction.type == 'income' and getattr(transaction, 'income_type', None) == 'DIV': - return self.target_account_map.get('dividends', None) + if transaction.type == "income" and getattr(transaction, "income_type", None) == "DIV": + return self.target_account_map.get("dividends", None) return self.target_account_map.get(transaction.type, None) def security_narration(self, ot): @@ -200,32 +208,33 @@ def security_narration(self, ot): def get_security_list(self): tickers = set() for ot in self.get_transactions(): - if hasattr(ot, 'security'): + if hasattr(ot, "security"): tickers.add(ot.security) return tickers def subst_acct_vars(self, raw_acct, ot, ticker): - """Resolve variables within an account like {ticker}. - """ + """Resolve variables within an account like {ticker}.""" ot = ot if ot else {} # inv401ksource is an ofx field that is 'PRETAX', 'AFTERTAX', etc. - kwargs = {'ticker': ticker, 'source401k': getattr(ot, 'inv401ksource', '').title()} + kwargs = { + "ticker": ticker, + "source401k": getattr(ot, "inv401ksource", "").title(), + } acct = raw_acct.format(**kwargs) return self.remove_empty_subaccounts(acct) # if 'inv401ksource' was unavailable def get_acct(self, acct, ot, ticker): - """Get an account from self.config, resolve variables, and return - """ + """Get an account from self.config, resolve variables, and return""" template = self.config.get(acct) if not template: - raise KeyError(f'{acct} not set in importer configuration. Config: {self.config}') + raise KeyError(f"{acct} not set in importer configuration. Config: {self.config}") return self.subst_acct_vars(template, ot, ticker) # extract() and supporting methods # -------------------------------------------------------------------------------- def generate_trade_entry(self, ot, file, counter): - """ Involves a commodity. One of: ['buymf', 'sellmf', 'buystock', 'sellstock', 'buyother', + """Involves a commodity. One of: ['buymf', 'sellmf', 'buystock', 'sellstock', 'buyother', 'sellother', 'reinvest']""" config = self.config @@ -234,9 +243,11 @@ def generate_trade_entry(self, ot, file, counter): # Build metadata metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='transaction_trade', data={'transaction': ot})) - if getattr(ot, 'settleDate', None) is not None and ot.settleDate != ot.tradeDate: - metadata['settlement_date'] = str(ot.settleDate.date()) + metadata.update( + self.build_metadata(file, metatype="transaction_trade", data={"transaction": ot}) + ) + if getattr(ot, "settleDate", None) is not None and ot.settleDate != ot.tradeDate: + metadata["settlement_date"] = str(ot.settleDate.date()) narration = self.security_narration(ot) raw_target_acct = self.get_target_acct(ot, ticker) @@ -244,45 +255,71 @@ def generate_trade_entry(self, ot, file, counter): total = ot.total # special cases - if 'sell' in ot.type: + if "sell" in ot.type: units = -1 * abs(ot.units) if not is_money_market: - metadata['todo'] = 'TODO: this entry is incomplete until lots are selected (bean-doctor context )' # noqa: E501 - if ot.type in ['reinvest']: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI + metadata["todo"] = ( + "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 + ) + if ot.type in [ + "reinvest" + ]: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI ticker_val = ticker else: ticker_val = self.currency target_acct = self.subst_acct_vars(raw_target_acct, ot, ticker_val) # Build transaction entry - entry = data.Transaction(metadata, ot.tradeDate.date(), self.FLAG, - self.get_payee(ot), narration, - self.get_tags(ot), data.EMPTY_SET, []) + entry = data.Transaction( + metadata, + ot.tradeDate.date(), + self.FLAG, + self.get_payee(ot), + narration, + self.get_tags(ot), + data.EMPTY_SET, + [], + ) # Main posting(s): - main_acct = self.get_acct('main_account', ot, ticker) + main_acct = self.get_acct("main_account", ot, ticker) if is_money_market: # Use price conversions instead of holding these at cost - common.create_simple_posting_with_price(entry, main_acct, - units, ticker, ot.unit_price, self.currency) - elif 'sell' in ot.type: - common.create_simple_posting_with_cost_or_price(entry, main_acct, - units, ticker, price_number=ot.unit_price, - price_currency=self.currency, - costspec=CostSpec(None, None, None, None, None, None)) - cg_acct = self.get_acct('cg', ot, ticker) + common.create_simple_posting_with_price( + entry, main_acct, units, ticker, ot.unit_price, self.currency + ) + elif "sell" in ot.type: + common.create_simple_posting_with_cost_or_price( + entry, + main_acct, + units, + ticker, + price_number=ot.unit_price, + price_currency=self.currency, + costspec=CostSpec(None, None, None, None, None, None), + ) + cg_acct = self.get_acct("cg", ot, ticker) data.create_simple_posting(entry, cg_acct, None, None) else: # buy stock/fund - unit_price = getattr(ot, 'unit_price', 0) + unit_price = getattr(ot, "unit_price", 0) # annoyingly, vanguard reinvests have ot.unit_price set to zero. so manually compute it - if (hasattr(ot, 'security') and ot.security) and ot.units and not ot.unit_price: + if (hasattr(ot, "security") and ot.security) and ot.units and not ot.unit_price: unit_price = round(abs(ot.total) / ot.units, 4) - common.create_simple_posting_with_cost(entry, main_acct, units, ticker, unit_price, - self.currency, self.price_cost_both_zero_handler) + common.create_simple_posting_with_cost( + entry, + main_acct, + units, + ticker, + unit_price, + self.currency, + self.price_cost_both_zero_handler, + ) # "Other" account posting reverser = 1 - if units > 0 and total > 0: # (ugly) hack for some brokerages with incorrect signs (TODO: remove) + if ( + units > 0 and total > 0 + ): # (ugly) hack for some brokerages with incorrect signs (TODO: remove) reverser = -1 data.create_simple_posting(entry, target_acct, reverser * total, self.currency) @@ -290,7 +327,8 @@ def generate_trade_entry(self, ot, file, counter): rounding_error = (reverser * total) + (ot.unit_price * units) if 0.0005 <= abs(rounding_error) <= self.max_rounding_error: data.create_simple_posting( - entry, config['rounding_error'], -1 * rounding_error, self.currency) + entry, config["rounding_error"], -1 * rounding_error, self.currency + ) # if abs(rounding_error) > self.max_rounding_error: # print("Transactions legs do not sum up! Difference: {}. Entry: {}, ot: {}".format( # rounding_error, entry, ot)) @@ -298,21 +336,32 @@ def generate_trade_entry(self, ot, file, counter): return entry def generate_transfer_entry(self, ot, file, counter): - """ Cash transactions, or in-kind transfers. One of: - [credit, debit, dep, transfer, income, dividends, capgainsd_lt, capgainsd_st, other]""" + """Cash transactions, or in-kind transfers. One of: + [credit, debit, dep, transfer, income, dividends, capgainsd_lt, capgainsd_st, other]""" config = self.config metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='transaction_transfer', data={'transaction': ot})) + metadata.update( + self.build_metadata(file, metatype="transaction_transfer", data={"transaction": ot}) + ) ticker = None - date = getattr(ot, 'tradeDate', None) + date = getattr(ot, "tradeDate", None) if not date: date = ot.date date = date.date() try: - if ot.type in ['transfer']: + if ot.type in ["transfer"]: units = ot.units - elif ot.type in ['other', 'credit', 'debit', 'dep', 'cash', 'payment', 'check', 'xfer']: + elif ot.type in [ + "other", + "credit", + "debit", + "dep", + "cash", + "payment", + "check", + "xfer", + ]: units = ot.amount else: units = ot.total @@ -321,28 +370,46 @@ def generate_transfer_entry(self, ot, file, counter): # import pdb; pdb.set_trace() main_acct = None - if ot.type in ['income', 'dividends', 'capgainsd_lt', - 'capgainsd_st', 'transfer'] and (hasattr(ot, 'security') and ot.security): + if ot.type in [ + "income", + "dividends", + "capgainsd_lt", + "capgainsd_st", + "transfer", + ] and (hasattr(ot, "security") and ot.security): ticker, ticker_long_name = self.get_ticker_info(ot.security) narration = self.security_narration(ot) - main_acct = self.get_acct('main_account', ot, ticker) + main_acct = self.get_acct("main_account", ot, ticker) else: # cash transaction narration = ot.type ticker = self.currency - main_acct = config['cash_account'] + main_acct = config["cash_account"] # Build transaction entry - entry = data.Transaction(metadata, date, self.FLAG, - self.get_payee(ot), narration, - self.get_tags(ot), data.EMPTY_SET, []) + entry = data.Transaction( + metadata, + date, + self.FLAG, + self.get_payee(ot), + narration, + self.get_tags(ot), + data.EMPTY_SET, + [], + ) target_acct = self.get_target_acct(ot, ticker) if target_acct: target_acct = self.subst_acct_vars(target_acct, ot, ticker) # Build postings - if ot.type in ['income', 'dividends', 'capgainsd_st', 'capgainsd_lt', 'fee']: # cash - amount = ot.total if hasattr(ot, 'total') else ot.amount - data.create_simple_posting(entry, config['cash_account'], amount, self.currency) + if ot.type in [ + "income", + "dividends", + "capgainsd_st", + "capgainsd_lt", + "fee", + ]: # cash + amount = ot.total if hasattr(ot, "total") else ot.amount + data.create_simple_posting(entry, config["cash_account"], amount, self.currency) data.create_simple_posting(entry, target_acct, -1 * amount, self.currency) else: data.create_simple_posting(entry, main_acct, units, ticker) @@ -371,14 +438,38 @@ def extract_transactions(self, file, counter): for ot in self.get_transactions(): if self.skip_transaction(ot): continue - if ot.type in ['buymf', 'sellmf', 'buystock', 'buydebt', 'sellstock', 'buyother', 'sellother', 'reinvest']: + if ot.type in [ + "buymf", + "sellmf", + "buystock", + "buydebt", + "sellstock", + "buyother", + "sellother", + "reinvest", + ]: entry = self.generate_trade_entry(ot, file, counter) - elif ot.type in ['other', 'credit', 'debit', 'transfer', 'xfer', 'dep', 'income', 'fee', - 'dividends', 'capgainsd_st', 'capgainsd_lt', 'cash', 'payment', 'check', 'invexpense']: + elif ot.type in [ + "other", + "credit", + "debit", + "transfer", + "xfer", + "dep", + "income", + "fee", + "dividends", + "capgainsd_st", + "capgainsd_lt", + "cash", + "payment", + "check", + "invexpense", + ]: entry = self.generate_transfer_entry(ot, file, counter) else: print("ERROR: unknown entry type:", ot.type) - raise Exception('Unknown entry type') + raise Exception("Unknown entry type") self.add_fee_postings(entry, ot) self.add_custom_postings(entry, ot) new_entries.append(entry) @@ -392,37 +483,53 @@ def extract_balances_and_prices(self, file, counter): for pos in self.get_balance_positions(): ticker, ticker_long_name = self.get_ticker_info(pos.security) metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance', data={'pos': pos})) + metadata.update(self.build_metadata(file, metatype="balance", data={"pos": pos})) # if there are no transactions, use the date in the source file for the balance. This gives us the # bonus of an updated, recent balance assertion bal_date = date if date else pos.date.date() - main_acct = self.get_acct('main_account', None, ticker) - balance_entry = data.Balance(metadata, bal_date, main_acct, - amount.Amount(pos.units, ticker), - None, None) + main_acct = self.get_acct("main_account", None, ticker) + balance_entry = data.Balance( + metadata, + bal_date, + main_acct, + amount.Amount(pos.units, ticker), + None, + None, + ) new_entries.append(balance_entry) if ticker in self.money_market_funds: settlement_fund_balance = pos.units # extract price info if available - if hasattr(pos, 'unit_price') and hasattr(pos, 'date'): + if hasattr(pos, "unit_price") and hasattr(pos, "date"): metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='price', data={'pos': pos})) - price_entry = data.Price(metadata, pos.date.date(), ticker, - amount.Amount(pos.unit_price, self.currency)) + metadata.update(self.build_metadata(file, metatype="price", data={"pos": pos})) + price_entry = data.Price( + metadata, + pos.date.date(), + ticker, + amount.Amount(pos.unit_price, self.currency), + ) new_entries.append(price_entry) # ----------------- available cash available_cash = self.get_available_cash(settlement_fund_balance) if available_cash is not None: metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance_cash')) + metadata.update(self.build_metadata(file, metatype="balance_cash")) try: - bal_date = date if date else self.file_date(file).date() # unavailable file_date raises AttributeError - balance_entry = data.Balance(metadata, bal_date, self.config['cash_account'], - amount.Amount(available_cash, self.currency), - None, None) + bal_date = ( + date if date else self.file_date(file).date() + ) # unavailable file_date raises AttributeError + balance_entry = data.Balance( + metadata, + bal_date, + self.config["cash_account"], + amount.Amount(available_cash, self.currency), + None, + None, + ) new_entries.append(balance_entry) except AttributeError: pass @@ -431,14 +538,11 @@ def extract_balances_and_prices(self, file, counter): def add_fee_postings(self, entry, ot): config = self.config - if hasattr(ot, 'fees') or hasattr(ot, 'commission'): - if getattr(ot, 'fees', 0) != 0: - data.create_simple_posting(entry, config['fees'], ot.fees, self.currency) - if getattr(ot, 'commission', 0) != 0: - data.create_simple_posting(entry, config['fees'], ot.commission, self.currency) - - def add_custom_postings(self, entry, ot): - pass + if hasattr(ot, "fees") or hasattr(ot, "commission"): + if getattr(ot, "fees", 0) != 0: + data.create_simple_posting(entry, config["fees"], ot.fees, self.currency) + if getattr(ot, "commission", 0) != 0: + data.create_simple_posting(entry, config["fees"], ot.commission, self.currency) def extract_custom_entries(self, file, counter): """For custom importers to override""" diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index d6709fe..a6b3b25 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -1,9 +1,11 @@ """Generic banking ofx importer for beancount.""" +from collections import defaultdict + from beancount.core import data from beancount.core.number import D -from beancount_reds_importers.libtransactionbuilder import banking +from beancount_reds_importers.libtransactionbuilder import banking # paychecks are typically transaction with many (10-40) postings including several each of income, taxes, # pre-tax and post-tax deductions, transfers, reimbursements, etc. This importer enables importing a single @@ -50,10 +52,13 @@ # }, # } + def flip_if_needed(amount, account): - if amount >= 0 and any(account.startswith(prefix) for prefix in ['Income:', 'Equity:', 'Liabilities:']): + if amount >= 0 and any( + account.startswith(prefix) for prefix in ["Income:", "Equity:", "Liabilities:"] + ): amount *= -1 - if amount < 0 and any(account.startswith(prefix) for prefix in ['Expenses:', 'Assets:']): + if amount < 0 and any(account.startswith(prefix) for prefix in ["Expenses:", "Assets:"]): amount *= -1 return amount @@ -63,23 +68,34 @@ def file_date(self, input_file): return self.paycheck_date(input_file) def build_postings(self, entry): - template = self.config['paycheck_template'] - currency = self.config['currency'] + template = self.config["paycheck_template"] + currency = self.config["currency"] total = 0 + template_missing = defaultdict(set) for section, table in self.alltables.items(): if section not in template: + template_missing[section] = set() continue for row in table.namedtuples(): # TODO: 'bank' is workday specific; move it there - row_description = getattr(row, 'description', getattr(row, 'bank', None)) - row_pattern = next(filter(lambda ts: row_description.startswith(ts), template[section]), None) - if row_pattern: + row_description = getattr(row, "description", getattr(row, "bank", None)) + row_pattern = next( + filter(lambda ts: row_description.startswith(ts), template[section]), + None, + ) + if not row_pattern: + template_missing[section].add(row_description) + else: accounts = template[section][row_pattern] accounts = [accounts] if not isinstance(accounts, list) else accounts for account in accounts: # TODO: 'amount_in_pay_group_currency' is workday specific; move it there - amount = getattr(row, 'amount', getattr(row, 'amount_in_pay_group_currency', None)) + amount = getattr( + row, + "amount", + getattr(row, "amount_in_pay_group_currency", None), + ) # import pdb; pdb.set_trace() if not amount: @@ -89,31 +105,41 @@ def build_postings(self, entry): total += amount if amount: data.create_simple_posting(entry, account, amount, currency) + + if self.config.get("show_unconfigured", False): + for section in template_missing: + print(section) + if template_missing[section]: + print(" " + "\n ".join(i for i in template_missing[section])) + print() + if total != 0: data.create_simple_posting(entry, "TOTAL:NONZERO", total, currency) - if self.config.get('sort_postings', True): + if self.config.get("sort_postings", True): postings = sorted(entry.postings) else: postings = entry.postings newentry = entry._replace(postings=postings) return newentry - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} - def extract(self, file, existing_entries=None): self.initialize(file) config = self.config self.read_file(file) metadata = data.new_metadata(file.name, 0) - metadata.update(self.build_metadata(file, metatype='transaction')) - entry = data.Transaction(metadata, self.paycheck_date(file), self.FLAG, - None, config['desc'], self.get_tags(), data.EMPTY_SET, []) + metadata.update(self.build_metadata(file, metatype="transaction")) + entry = data.Transaction( + metadata, + self.paycheck_date(file), + self.FLAG, + None, + config["desc"], + self.get_tags(), + data.EMPTY_SET, + [], + ) entry = self.build_postings(entry) return [entry] diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index 56ea30b..d9f9992 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -4,7 +4,7 @@ from beancount.core import data -class TransactionBuilder(): +class TransactionBuilder: def skip_transaction(self, ot): """For custom importers to override""" return False @@ -16,7 +16,7 @@ def get_tags(self, ot=None): @staticmethod def remove_empty_subaccounts(acct): """Translates 'Assets:Foo::Bar' to 'Assets:Foo:Bar'.""" - return ':'.join(x for x in acct.split(':') if x) + return ":".join(x for x in acct.split(":") if x) def set_config_variables(self, substs): """ @@ -31,11 +31,30 @@ def set_config_variables(self, substs): 'source401k': '{source401k}', } """ - self.config = {k: v.format(**substs) if isinstance(v, str) else v for k, v in self.config.items()} + self.config = { + k: v.format(**substs) if isinstance(v, str) else v for k, v in self.config.items() + } # Prevent the replacement fields from appearing in the output of # the file_account method - if 'filing_account' not in self.config: - kwargs = {k: '' for k in substs} - filing_account = self.config['main_account'].format(**kwargs) - self.config['filing_account'] = self.remove_empty_subaccounts(filing_account) + if "filing_account" not in self.config: + kwargs = {k: "" for k in substs} + filing_account = self.config["main_account"].format(**kwargs) + self.config["filing_account"] = self.remove_empty_subaccounts(filing_account) + + def add_custom_postings(self, entry, ot): + """This method is for importers to override. Add arbitrary posting to each entry.""" + pass + + def build_metadata(self, file, metatype=None, data={}): + """This method is for importers to override. The overridden method can + look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) + and the data dictionary to return additional metadata""" + + # This 'filing_account' is read by a patch to bean-extract so it can output transactions to + # a file that corresponds with filing_account, when the one-file-per-account feature is + # used. + if self.config.get("emit_filing_account_metadata"): + acct = self.config.get("filing_account", self.config.get("main_account", None)) + return {"filing_account": acct} + return {} diff --git a/beancount_reds_importers/util/bean_download.py b/beancount_reds_importers/util/bean_download.py index 9bf37ec..2cfe3af 100755 --- a/beancount_reds_importers/util/bean_download.py +++ b/beancount_reds_importers/util/bean_download.py @@ -2,13 +2,15 @@ """Download account statements automatically when possible, or display a reminder of how to download them. Multi-threaded.""" -from click_aliases import ClickAliasedGroup import asyncio -import click import configparser import os + +import click import tabulate import tqdm +from click_aliases import ClickAliasedGroup + import beancount_reds_importers.util.needs_update as needs_update @@ -30,26 +32,31 @@ def readConfigFile(configfile): def get_sites(sites, t, config): - return [s for s in sites if config[s]['type'] == t] - - -@cli.command(aliases=['list']) -@click.option('-c', '--config-file', envvar='BEAN_DOWNLOAD_CONFIG', required=True, - help='Config file. The environment variable BEAN_DOWNLOAD_CONFIG can also be used to specify this', - type=click.Path(exists=True)) -@click.option('-s', '--sort', is_flag=True, help='Sort output') + return [s for s in sites if config[s]["type"] == t] + + +@cli.command(aliases=["list"]) +@click.option( + "-c", + "--config-file", + envvar="BEAN_DOWNLOAD_CONFIG", + required=True, + help="Config file. The environment variable BEAN_DOWNLOAD_CONFIG can also be used to specify this", + type=click.Path(exists=True), +) +@click.option("-s", "--sort", is_flag=True, help="Sort output") def list_institutions(config_file, sort): """List institutions (sites) currently configured.""" config = readConfigFile(config_file) all_sites = config.sections() - types = set([config[s]['type'] for s in all_sites]) + types = set([config[s]["type"] for s in all_sites]) for t in sorted(types): sites = get_sites(all_sites, t, config) if sort: sites = sorted(sites) name = f"{t} ({len(sites)})".ljust(14) - print(f"{name}:", end='') - print(*sites, sep=', ') + print(f"{name}:", end="") + print(*sites, sep=", ") print() @@ -57,31 +64,48 @@ def get_sites_and_sections(config_file): if config_file and os.path.exists(config_file): config = readConfigFile(config_file) all_sites = config.sections() - types = set([config[s]['type'] for s in all_sites]) + types = set([config[s]["type"] for s in all_sites]) return all_sites, types def complete_sites(ctx, param, incomplete): - config_file = ctx.params['config_file'] + config_file = ctx.params["config_file"] all_sites, _ = get_sites_and_sections(config_file) return [s for s in all_sites if s.startswith(incomplete)] def complete_site_types(ctx, param, incomplete): - config_file = ctx.params['config_file'] + config_file = ctx.params["config_file"] _, types = get_sites_and_sections(config_file) return [s for s in types if s.startswith(incomplete)] @cli.command() -@click.option('-c', '--config-file', envvar='BEAN_DOWNLOAD_CONFIG', required=True, help='Config file') -@click.option('-i', '--sites', '--institutions', help="Institutions to download (comma separated); unspecified means all", - default='', shell_complete=complete_sites) -@click.option('-t', '--site-types', '--institution-types', - help="Download all institutions of specified types (comma separated)", - default='', shell_complete=complete_site_types) -@click.option('--dry-run', is_flag=True, help="Do not actually download", default=False) -@click.option('--verbose', is_flag=True, help="Verbose", default=False) +@click.option( + "-c", + "--config-file", + envvar="BEAN_DOWNLOAD_CONFIG", + required=True, + help="Config file", +) +@click.option( + "-i", + "--sites", + "--institutions", + help="Institutions to download (comma separated); unspecified means all", + default="", + shell_complete=complete_sites, +) +@click.option( + "-t", + "--site-types", + "--institution-types", + help="Download all institutions of specified types (comma separated)", + default="", + shell_complete=complete_site_types, +) +@click.option("--dry-run", is_flag=True, help="Do not actually download", default=False) +@click.option("--verbose", is_flag=True, help="Verbose", default=False) def download(config_file, sites, site_types, dry_run, verbose): # noqa: C901 """Download statements for the specified institutions (sites).""" @@ -91,11 +115,11 @@ def pverbose(*args, **kwargs): config = readConfigFile(config_file) if sites: - sites = sites.split(',') + sites = sites.split(",") else: sites = config.sections() if site_types: - site_types = site_types.split(',') + site_types = site_types.split(",") sites_lists = [get_sites(sites, site_type, config) for site_type in site_types] sites = [j for i in sites_lists for j in i] @@ -106,8 +130,8 @@ def pverbose(*args, **kwargs): print(f"Processing {numsites} institutions.") async def download_site(i, site): - tid = f'[{i+1}/{numsites} {site}]' - pverbose(f'{tid}: Begin') + tid = f"[{i+1}/{numsites} {site}]" + pverbose(f"{tid}: Begin") try: options = config[site] except KeyError: @@ -116,11 +140,11 @@ async def download_site(i, site): return # We support cmd and display, and type to filter - if 'display' in options: + if "display" in options: displays.append([site, f"{options['display']}"]) success.append(site) - if 'cmd' in options: - cmd = os.path.expandvars(options['cmd']) + if "cmd" in options: + cmd = os.path.expandvars(options["cmd"]) pverbose(f"{tid}: Executing: {cmd}") if dry_run: await asyncio.sleep(2) @@ -129,9 +153,8 @@ async def download_site(i, site): else: # https://docs.python.org/3.8/library/asyncio-subprocess.html#asyncio.create_subprocess_exec proc = await asyncio.create_subprocess_shell( - cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) + cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) stdout, stderr = await proc.communicate() if proc.returncode: @@ -150,26 +173,30 @@ async def perform_downloads(sites): if displays: print() displays = [[i + 1, *row] for i, row in enumerate(displays)] - click.secho(tabulate.tabulate(displays, - headers=["#", "Institution", "Instructions"], tablefmt="plain"), fg='blue') + click.secho( + tabulate.tabulate( + displays, headers=["#", "Institution", "Instructions"], tablefmt="plain" + ), + fg="blue", + ) print() s = len(sites) if success: print(f"{len(success)}/{s} sites succeeded: {', '.join(success)}") if errors: - click.secho(f"{len(errors)}/{s} sites failed: {', '.join(errors)}", fg='red') + click.secho(f"{len(errors)}/{s} sites failed: {', '.join(errors)}", fg="red") -@cli.command(aliases=['init']) +@cli.command(aliases=["init"]) def config_template(): """Output a template for download.cfg that you can then use to build your own.""" path = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(*[path, 'template.cfg'])) as f: + with open(os.path.join(*[path, "template.cfg"])) as f: for line in f: - print(line, end='') + print(line, end="") -if __name__ == '__main__': +if __name__ == "__main__": cli() diff --git a/beancount_reds_importers/util/needs_update.py b/beancount_reds_importers/util/needs_update.py index e93c1fc..ed61da7 100755 --- a/beancount_reds_importers/util/needs_update.py +++ b/beancount_reds_importers/util/needs_update.py @@ -1,43 +1,45 @@ #!/usr/bin/env python3 """Determine the list of accounts needing updates based on the last balance entry.""" -import click +import ast import re +from datetime import datetime + +import click import tabulate from beancount import loader from beancount.core import getters from beancount.core.data import Balance, Close, Custom -from datetime import datetime -import ast - -tbl_options = {'tablefmt': 'simple'} +tbl_options = {"tablefmt": "simple"} def get_config(entries, args): """Get beancount config for the given plugin that can then be used on the command line""" global excluded_re, included_re - _extension_entries = [e for e in entries - if isinstance(e, Custom) and e.type == 'reds-importers'] - config_meta = {entry.values[0].value: - (entry.values[1].value if (len(entry.values) == 2) else None) - for entry in _extension_entries} - - config = {k: ast.literal_eval(v) for k, v in config_meta.items() if 'needs-updates' in k} - config = config.get('needs-updates', {}) - if args['all_accounts']: - config['included_account_pats'] = [] - config['excluded_account_pats'] = ['$-^'] - included_account_pats = config.get('included_account_pats', ['^Assets:', '^Liabilities:']) - excluded_account_pats = config.get('excluded_account_pats', ['$-^']) # exclude nothing by default - excluded_re = re.compile('|'.join(excluded_account_pats)) - included_re = re.compile('|'.join(included_account_pats)) + _extension_entries = [ + e for e in entries if isinstance(e, Custom) and e.type == "reds-importers" + ] + config_meta = { + entry.values[0].value: (entry.values[1].value if (len(entry.values) == 2) else None) + for entry in _extension_entries + } + + config = {k: ast.literal_eval(v) for k, v in config_meta.items() if "needs-updates" in k} + config = config.get("needs-updates", {}) + if args["all_accounts"]: + config["included_account_pats"] = [] + config["excluded_account_pats"] = ["$-^"] + included_account_pats = config.get("included_account_pats", ["^Assets:", "^Liabilities:"]) + excluded_account_pats = config.get( + "excluded_account_pats", ["$-^"] + ) # exclude nothing by default + excluded_re = re.compile("|".join(excluded_account_pats)) + included_re = re.compile("|".join(included_account_pats)) def is_interesting_account(account, closes): - return account not in closes and \ - included_re.match(account) and \ - not excluded_re.match(account) + return account not in closes and included_re.match(account) and not excluded_re.match(account) def handle_commodity_leaf_accounts(last_balance): @@ -49,9 +51,9 @@ def handle_commodity_leaf_accounts(last_balance): considered to be the latest date of a balance assertion on any child. """ d = {} - pat_ticker = re.compile(r'^[A-Z0-9]+$') + pat_ticker = re.compile(r"^[A-Z0-9]+$") for acc in last_balance: - parent, leaf = acc.rsplit(':', 1) + parent, leaf = acc.rsplit(":", 1) if pat_ticker.match(leaf): if parent in d: if d[parent].date < last_balance[acc].date: @@ -72,33 +74,44 @@ def accounts_with_no_balance_entries(entries, closes, last_balance): # Handle commodity leaf accounts accs_no_bal = [] - pat_ticker = re.compile(r'^[A-Z0-9]+$') + pat_ticker = re.compile(r"^[A-Z0-9]+$") def acc_or_parent(acc): - parent, leaf = acc.rsplit(':', 1) + parent, leaf = acc.rsplit(":", 1) if pat_ticker.match(leaf): return parent return acc + accs_no_bal = [acc_or_parent(i) for i in accs_no_bal_raw] # Remove accounts where one or more children do have a balance entry. Needed because of # commodity leaf accounts - accs_no_bal = [(i,) for i in set(accs_no_bal) if not any(j.startswith(i) for j in last_balance)] + accs_no_bal = [ + (i,) for i in set(accs_no_bal) if not any(j.startswith(i) for j in last_balance) + ] return accs_no_bal def pretty_print_table(not_updated_accounts, sort_by_date): field = 0 if sort_by_date else 1 output = sorted([(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field]) - headers = ['Last Updated', 'Account'] + headers = ["Last Updated", "Account"] print(click.style(tabulate.tabulate(output, headers=headers, **tbl_options))) -@click.command("needs-update", context_settings={'show_default': True}) -@click.argument('beancount-file', type=click.Path(exists=True), envvar='BEANCOUNT_FILE') -@click.option('--recency', help='How many days ago should the last balance assertion be to be considered old', default=15) -@click.option('--sort-by-date', help='Sort output by date (instead of account name)', is_flag=True) -@click.option('--all-accounts', help='Show all account (ignore include/exclude in config)', is_flag=True) +@click.command("needs-update", context_settings={"show_default": True}) +@click.argument("beancount-file", type=click.Path(exists=True), envvar="BEANCOUNT_FILE") +@click.option( + "--recency", + help="How many days ago should the last balance assertion be to be considered old", + default=15, +) +@click.option("--sort-by-date", help="Sort output by date (instead of account name)", is_flag=True) +@click.option( + "--all-accounts", + help="Show all account (ignore include/exclude in config)", + is_flag=True, +) def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts): """ Show a list of accounts needing updates, and the date of the last update (which is defined as @@ -141,21 +154,30 @@ def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts entries, _, _ = loader.load_file(beancount_file) get_config(entries, locals()) closes = [a.account for a in entries if isinstance(a, Close)] - balance_entries = [a for a in entries if isinstance(a, Balance) and - is_interesting_account(a.account, closes)] + balance_entries = [ + a for a in entries if isinstance(a, Balance) and is_interesting_account(a.account, closes) + ] last_balance = {v.account: v for v in balance_entries} d = handle_commodity_leaf_accounts(last_balance) # find accounts with balance assertions older than N days - need_updates = {acc: bal for acc, bal in d.items() if ((datetime.now().date() - d[acc].date).days > recency)} + need_updates = { + acc: bal + for acc, bal in d.items() + if ((datetime.now().date() - d[acc].date).days > recency) + } pretty_print_table(need_updates, sort_by_date) # If there are accounts with zero balance entries, print them accs_no_bal = accounts_with_no_balance_entries(entries, closes, last_balance) if accs_no_bal: - headers = ['Accounts without balance entries:'] - print(click.style('\n' + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options))) + headers = ["Accounts without balance entries:"] + print( + click.style( + "\n" + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options) + ) + ) -if __name__ == '__main__': +if __name__ == "__main__": accounts_needing_updates() diff --git a/beancount_reds_importers/util/ofx_summarize.py b/beancount_reds_importers/util/ofx_summarize.py index e5ab2f6..e83a9a7 100755 --- a/beancount_reds_importers/util/ofx_summarize.py +++ b/beancount_reds_importers/util/ofx_summarize.py @@ -1,17 +1,19 @@ #!/usr/bin/env python3 """Quick and dirty way to summarize a .ofx file and peek inside it.""" -import click import os import sys +import warnings from collections import defaultdict -from ofxparse import OfxParser + +import click from bs4.builder import XMLParsedAsHTMLWarning -import warnings +from ofxparse import OfxParser + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) -def analyze(filename, ttype='dividends', pdb_explore=False): +def analyze(filename, ttype="dividends", pdb_explore=False): ts = defaultdict(list) ofx = OfxParser.parse(open(filename)) for acc in ofx.accounts: @@ -19,14 +21,19 @@ def analyze(filename, ttype='dividends', pdb_explore=False): ts[t.type].append(t) if pdb_explore: import pdb + pdb.set_trace() @click.command() -@click.argument('filename', type=click.Path(exists=True)) -@click.option('-n', '--num-transactions', default=5, help='Number of transactions to show') -@click.option('-e', '--pdb-explore', is_flag=True, help='Open a pdb shell to explore') -@click.option('--stats-only', is_flag=True, help='Print total number of transactions contained in the file, and quit') +@click.argument("filename", type=click.Path(exists=True)) +@click.option("-n", "--num-transactions", default=5, help="Number of transactions to show") +@click.option("-e", "--pdb-explore", is_flag=True, help="Open a pdb shell to explore") +@click.option( + "--stats-only", + is_flag=True, + help="Print total number of transactions contained in the file, and quit", +) def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C901 """Quick and dirty way to summarize a .ofx file and peek inside it.""" if os.stat(filename).st_size == 0: @@ -45,33 +52,49 @@ def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C90 sys.exit(0) print("Total number of accounts:", len(ofx.accounts)) for acc in ofx.accounts: - print('----------------') + print("----------------") try: - print("Account info: ", acc.account_type, acc.account_id, acc.institution.organization) + print( + "Account info: ", + acc.account_type, + acc.account_id, + acc.institution.organization, + ) except AttributeError: print("Account info: ", acc.account_type, acc.account_id) pass try: - print("Statement info: {} -- {}. Bal: {}".format(acc.statement.start_date, - acc.statement.end_date, acc.statement.balance)) + print( + "Statement info: {} -- {}. Bal: {}".format( + acc.statement.start_date, + acc.statement.end_date, + acc.statement.balance, + ) + ) except AttributeError: try: positions = [(p.units, p.security) for p in acc.statement.positions] - print("Statement info: {} -- {}. Bal: {}".format(acc.statement.start_date, - acc.statement.end_date, positions)) + print( + "Statement info: {} -- {}. Bal: {}".format( + acc.statement.start_date, acc.statement.end_date, positions + ) + ) except AttributeError: print("Statement info: UNABLE to get start_date and end_date") print("Types: ", set([t.type for t in acc.statement.transactions])) print() - txns = sorted(acc.statement.transactions, reverse=True, - key=lambda t: t.date if hasattr(t, 'date') else t.tradeDate) + txns = sorted( + acc.statement.transactions, + reverse=True, + key=lambda t: t.date if hasattr(t, "date") else t.tradeDate, + ) for t in txns[:num_transactions]: - date = t.date if hasattr(t, 'date') else t.tradeDate - description = t.payee + ' ' + t.memo if hasattr(t, 'payee') else t.memo - amount = t.amount if hasattr(t, 'amount') else t.total + date = t.date if hasattr(t, "date") else t.tradeDate + description = t.payee + " " + t.memo if hasattr(t, "payee") else t.memo + amount = t.amount if hasattr(t, "amount") else t.total print(date, t.type, description, amount) if pdb_explore: print("Hints:") @@ -80,10 +103,11 @@ def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C90 if len(ofx.accounts) > 1: print("- type 'c' to explore the next account in this file") import pdb + pdb.set_trace() print() print() -if __name__ == '__main__': +if __name__ == "__main__": summarize() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..574e924 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.ruff] +line-length = 99 + +[tool.ruff.format] +docstring-code-format = true +indent-style = "space" +line-ending = "lf" +quote-style = "double" + +[tool.isort] +profile = "black" diff --git a/setup.py b/setup.py index a23c92f..78474f3 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,28 @@ from os import path + from setuptools import find_packages, setup -with open(path.join(path.dirname(__file__), 'README.md')) as readme: +with open(path.join(path.dirname(__file__), "README.md")) as readme: LONG_DESCRIPTION = readme.read() setup( - name='beancount_reds_importers', + name="beancount_reds_importers", use_scm_version=True, - setup_requires=['setuptools_scm'], - description='Importers for various institutions for Beancount', + setup_requires=["setuptools_scm"], + description="Importers for various institutions for Beancount", long_description=LONG_DESCRIPTION, - long_description_content_type='text/markdown', - url='https://github.com/redstreet/beancount_reds_importers', - author='Red Street', - author_email='redstreet@users.noreply.github.com', - keywords='importer ingestor beancount accounting', - license='GPL-3.0', + long_description_content_type="text/markdown", + url="https://github.com/redstreet/beancount_reds_importers", + author="Red Street", + author_email="redstreet@users.noreply.github.com", + keywords="importer ingestor beancount accounting", + license="GPL-3.0", packages=find_packages(), include_package_data=True, extras_require={ - 'dev': [ - 'ruff', + "dev": [ + "ruff", + "isort", ] }, install_requires=[ @@ -31,26 +33,26 @@ 'ofxparse >= 0.21', 'openpyxl >= 3.0.9', 'packaging >= 20.3', - 'pdfplumber>=0.11.0', + 'pdfplumber >= 0.11.0', 'petl >= 1.7.4', 'tabulate >= 0.8.9', 'tqdm >= 4.64.0', ], entry_points={ - 'console_scripts': [ - 'ofx-summarize = beancount_reds_importers.util.ofx_summarize:summarize', - 'bean-download = beancount_reds_importers.util.bean_download:cli', + "console_scripts": [ + "ofx-summarize = beancount_reds_importers.util.ofx_summarize:summarize", + "bean-download = beancount_reds_importers.util.bean_download:cli", ] }, zip_safe=False, classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Financial and Insurance Industry', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Natural Language :: English', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', - 'Topic :: Office/Business :: Financial :: Accounting', - 'Topic :: Office/Business :: Financial :: Investment', + "Development Status :: 4 - Beta", + "Intended Audience :: Financial and Insurance Industry", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Topic :: Office/Business :: Financial :: Accounting", + "Topic :: Office/Business :: Financial :: Investment", ], )