From 67b09f417b0f43d31481b2b3179531bec6291e1e Mon Sep 17 00:00:00 2001 From: dev Date: Sat, 5 Oct 2024 01:00:14 +0200 Subject: [PATCH 1/3] fix: constrain Beancount dependency version The project doesn't work with beancount v3. --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 53470d3..6a151c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Automatically generated by https://github.com/damnever/pigar. -beancount>=2.3.5 +beancount >=2.3.5,<3.0.0 beautifulsoup4>=4.12.3 click>=8.1.7 click-aliases>=1.0.4 diff --git a/setup.py b/setup.py index a60d832..0df449a 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ }, install_requires=[ "Click >= 7.0", - "beancount >= 2.3.5", + "beancount >=2.3.5,<3.0.0", "click_aliases >= 1.0.1", "dateparser >= 1.2.0", "ofxparse >= 0.21", From 8194ac405a3e933915af789ba43f193b5accfe46 Mon Sep 17 00:00:00 2001 From: dev Date: Sat, 5 Oct 2024 01:47:04 +0200 Subject: [PATCH 2/3] feat! : add generic json reader BREAKING CHANGE: old jsonreader.py is rename into schwabjsonreader.py --- .../libreader/jsonreader.py | 106 ++++++------------ .../libreader/schwabjsonreader.py | 92 +++++++++++++++ 2 files changed, 129 insertions(+), 69 deletions(-) create mode 100644 beancount_reds_importers/libreader/schwabjsonreader.py diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 38ffc8f..0d046d3 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -1,92 +1,60 @@ -"""JSON importer module for beancount to be used along with investment/banking/other importer modules in -beancount_reds_importers. +#!/usr/bin/env python3 ------------------------------- -This is WIP and incomplete. ------------------------------- +"""JSON reader for beancount-reds-importers. -JSON schemas vary widely. This one is based on Charles Schwab's json format. In the future, the -goal is to make this reader automatically "understand" the schema of any json given to it. +JSON files have widely varying specifications, and thus, this is a very generic reader, and most of +the logic will have to be the institution specific readers. -Until that happens, perhaps this file should be renamed to schwabjsonreader.py. """ import json -# import re -import warnings - -# import datetime -# import ofxparse -# from collections import namedtuple from beancount.ingest import importer -from bs4.builder import XMLParsedAsHTMLWarning - from beancount_reds_importers.libreader import reader -warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) - - class Importer(reader.Reader, importer.ImporterProtocol): FILE_EXTS = ["json"] def initialize_reader(self, file): if getattr(self, "file", None) != file: self.file = file + self.reader_ready = False + with open(file.name, 'r') as f: + self.json_data = json.load(f) self.reader_ready = self.deep_identify(file) - if self.reader_ready: - self.file_read_done = False + if self.reader_ready: + self.set_currency() def deep_identify(self, file): - # identify based on filename - return True + """For overriding by institution specific importer which can check if an account name + matches, and other such things.""" + # default value to False, else jsonreader.initialize_reader fail to execute because missing attribut "config" + return False def file_date(self, file): - "Get the maximum date from the file." - self.initialize(file) # self.date_format gets set via this - self.read_file(file) - return max(ot.date for ot in self.get_transactions()).date() + """Get the ending date of the statement.""" + if not getattr(self, "json_data", None): + self.initialize(file) + # TODO: + return None def read_file(self, file): - with open(file.name) as fh: - self.rdr = json.load(fh) - - # transactions = [] - # for transaction in self.rdr['BrokerageTransactions']: - # raw_ot = Transaction( - # date = transaction['Date'], - # type = transaction['Action'], - # security = transaction['Symbol'], - # memo = transaction['Description'], - # unit_price = transaction['Price'], - # units = transaction['Quantity'], - # fees = transaction['Fees & Comm'], - # total = transaction['Amount'] - # ) - - # def get_transactions(self): - # Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', - # 'units', 'fees', 'total']) - # for transaction in self.rdr['BrokerageTransactions']: - # raw_ot = Transaction( - # date = transaction['Date'], - # type = transaction['Action'], - # security = transaction['Symbol'], - # memo = transaction['Description'], - # unit_price = transaction['Price'], - # units = transaction['Quantity'], - # fees = transaction['Fees & Comm'], - # total = transaction['Amount'] - # ) - # ot = self.fixup(ot) - # import pdb; pdb.set_trace() - # yield ot - - def fixup(self, ot): - ot.date = self.convert_date(ot.date) - - # def convert_date(d): - # return datetime.datetime.strptime(d, self.date_format) - - def get_balance_assertion_date(self): - return None + with open(file.name, 'r') as f: + self.json_data = json.load(f) + + def get_json_elements(self, json_path, json_interpreter=lambda x: x): + """Extract a list of elements in the JSON file at the given JSON path. Typically, + transactions are stored in a JSON path, and this extracts them.""" + elements = self.json_data + for key in json_path.split('.'): + if key in elements: + elements = elements[key] + else: + return [] + for elem in elements: + yield json_interpreter(elem) + + def get_transactions(self): + """/Transactions/Transaction is a dummy default path for transactions that needs to be + overriden in the institution specific importer.""" + yield from self.get_json_elements("Transactions.Transaction") diff --git a/beancount_reds_importers/libreader/schwabjsonreader.py b/beancount_reds_importers/libreader/schwabjsonreader.py new file mode 100644 index 0000000..38ffc8f --- /dev/null +++ b/beancount_reds_importers/libreader/schwabjsonreader.py @@ -0,0 +1,92 @@ +"""JSON importer module for beancount to be used along with investment/banking/other importer modules in +beancount_reds_importers. + +------------------------------ +This is WIP and incomplete. +------------------------------ + +JSON schemas vary widely. This one is based on Charles Schwab's json format. In the future, the +goal is to make this reader automatically "understand" the schema of any json given to it. + +Until that happens, perhaps this file should be renamed to schwabjsonreader.py. +""" + +import json + +# import re +import warnings + +# import datetime +# import ofxparse +# from collections import namedtuple +from beancount.ingest import importer +from bs4.builder import XMLParsedAsHTMLWarning + +from beancount_reds_importers.libreader import reader + +warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) + + +class Importer(reader.Reader, importer.ImporterProtocol): + FILE_EXTS = ["json"] + + def initialize_reader(self, file): + if getattr(self, "file", None) != file: + self.file = file + self.reader_ready = self.deep_identify(file) + if self.reader_ready: + self.file_read_done = False + + def deep_identify(self, file): + # identify based on filename + return True + + def file_date(self, file): + "Get the maximum date from the file." + self.initialize(file) # self.date_format gets set via this + self.read_file(file) + return max(ot.date for ot in self.get_transactions()).date() + + def read_file(self, file): + with open(file.name) as fh: + self.rdr = json.load(fh) + + # transactions = [] + # for transaction in self.rdr['BrokerageTransactions']: + # raw_ot = Transaction( + # date = transaction['Date'], + # type = transaction['Action'], + # security = transaction['Symbol'], + # memo = transaction['Description'], + # unit_price = transaction['Price'], + # units = transaction['Quantity'], + # fees = transaction['Fees & Comm'], + # total = transaction['Amount'] + # ) + + # def get_transactions(self): + # Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', + # 'units', 'fees', 'total']) + # for transaction in self.rdr['BrokerageTransactions']: + # raw_ot = Transaction( + # date = transaction['Date'], + # type = transaction['Action'], + # security = transaction['Symbol'], + # memo = transaction['Description'], + # unit_price = transaction['Price'], + # units = transaction['Quantity'], + # fees = transaction['Fees & Comm'], + # total = transaction['Amount'] + # ) + # ot = self.fixup(ot) + # import pdb; pdb.set_trace() + # yield ot + + def fixup(self, ot): + ot.date = self.convert_date(ot.date) + + # def convert_date(d): + # return datetime.datetime.strptime(d, self.date_format) + + def get_balance_assertion_date(self): + return None From 85080cd461aa021dba464d209003b1f55d71e26f Mon Sep 17 00:00:00 2001 From: dev Date: Sun, 6 Oct 2024 23:27:55 +0200 Subject: [PATCH 3/3] refactor: formatting whole project --- .../fidelity/fidelity_brokerage_csv.py | 39 +++--- .../importers/ibkr/__init__.py | 113 +++++++++++------- .../importers/ibkr/flexquery_download.py | 32 ++--- .../importers/vanguard/__init__.py | 2 +- .../libreader/jsonreader.py | 8 +- .../libreader/xmlreader.py | 2 +- .../libtransactionbuilder/investments.py | 4 +- 7 files changed, 116 insertions(+), 84 deletions(-) diff --git a/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py index cfefedc..79b8d9f 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py @@ -14,34 +14,35 @@ def custom_init(self): self.filename_pattern_def = ".*History" self.date_format = "%m/%d/%Y" self.header_identifier = "" - self.column_labels_line = ( - "Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date" - ) + self.column_labels_line = "Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date" self.header_map = { - "Run Date": "date", - "Action": "memo", - "Symbol": "security", - "Amount ($)": "amount", - "Settlement Date": "settleDate", - "Quantity": "units", + "Run Date": "date", + "Action": "memo", + "Symbol": "security", + "Amount ($)": "amount", + "Settlement Date": "settleDate", + "Quantity": "units", "Accrued Interest ($)": "accrued_interest", - "Fees ($)": "fees", - "Commission ($)": "commission", - "Cash Balance ($)": "balance", - "Price ($)": "unit_price", + "Fees ($)": "fees", + "Commission ($)": "commission", + "Cash Balance ($)": "balance", + "Price ($)": "unit_price", } self.transaction_type_map = { - "DIVIDEND RECEIVED": "dividends", - "TRANSFERRED FROM": "cash", - "YOU BOUGHT": "buystock", - "YOU SOLD": "sellstock", + "DIVIDEND RECEIVED": "dividends", + "TRANSFERRED FROM": "cash", + "YOU BOUGHT": "buystock", + "YOU SOLD": "sellstock", } self.skip_transaction_types = [] # fmt: on def deep_identify(self, file): last_four = self.config.get("account_number", "")[-4:] - return re.match(self.header_identifier, file.head(), flags=re.DOTALL) and f"{last_four}" in file.name + return ( + re.match(self.header_identifier, file.head(), flags=re.DOTALL) + and f"{last_four}" in file.name + ) def prepare_table(self, rdr): for field in ["Action", "Symbol", "Description"]: @@ -49,7 +50,7 @@ def prepare_table(self, rdr): rdr = rdr.addfield("total", lambda x: x["Amount ($)"]) rdr = rdr.addfield("tradeDate", lambda x: x["Run Date"]) - rdr = rdr.cutout('Type') + rdr = rdr.cutout("Type") rdr = rdr.capture("Action", "(\\S+(?:\\s+\\S+)?)", ["type"], include_original=True) # for field in ["memo"]: diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index 4ab1978..999cb48 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -93,9 +93,11 @@ """ import datetime + +from beancount.core.number import D + from beancount_reds_importers.libreader import xmlreader from beancount_reds_importers.libtransactionbuilder import investments -from beancount.core.number import D class DictToObject: @@ -106,8 +108,8 @@ def __init__(self, dictionary): # xml on left, ofx on right ofx_type_map = { - 'BUY': 'buystock', - 'SELL': 'selltock', + "BUY": "buystock", + "SELL": "selltock", } @@ -119,14 +121,21 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = "ibkr" self.custom_init_run = True - self.date_format = '%Y-%m-%d' + self.date_format = "%Y-%m-%d" self.get_ticker_info = self.get_ticker_info_from_id def deep_identify(self, file): try: - if self.config.get('account_number', None): + if self.config.get("account_number", None): # account number specific matching - return self.config['account_number'] == list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['accountId'] + return ( + self.config["account_number"] + == list( + self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation" + ) + )[0]["accountId"] + ) else: # base check: simply ensure this looks like a valid IBKR Flex Query file return list(self.get_xpath_elements("/FlexQueryResponse"))[0] is not None @@ -134,77 +143,97 @@ def deep_identify(self, file): return False def set_currency(self): - self.currency = list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['currency'] + self.currency = list( + self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation" + ) + )[0]["currency"] # fixup dates def convert_date(self, d): - d = d.split(' ')[0] + d = d.split(" ")[0] return datetime.datetime.strptime(d, self.date_format) def xml_transfer_interpreter(self, xml_data): # map, with ofx fields on the left and xml fields on the right ofx_dict = { - 'security': xml_data['isin'], - 'tradeDate': self.convert_date(xml_data['dateTime']), - 'units': D(xml_data['quantity']), - 'memo': 'Transfer in kind', - 'type': 'transfer', + "security": xml_data["isin"], + "tradeDate": self.convert_date(xml_data["dateTime"]), + "units": D(xml_data["quantity"]), + "memo": "Transfer in kind", + "type": "transfer", } return DictToObject(ofx_dict) def xml_trade_interpreter(self, xml_data): # map, with ofx fields on the left and xml fields on the right ofx_dict = { - 'security': xml_data['isin'], - 'tradeDate': self.convert_date(xml_data['dateTime']), - 'memo': xml_data['transactionType'], - 'type': ofx_type_map[xml_data['buySell']], - 'units': D(xml_data['quantity']), - 'unit_price': D(xml_data['tradePrice']), - 'commission': -1 * D(xml_data['ibCommission']), - 'total': D(xml_data['netCash']), + "security": xml_data["isin"], + "tradeDate": self.convert_date(xml_data["dateTime"]), + "memo": xml_data["transactionType"], + "type": ofx_type_map[xml_data["buySell"]], + "units": D(xml_data["quantity"]), + "unit_price": D(xml_data["tradePrice"]), + "commission": -1 * D(xml_data["ibCommission"]), + "total": D(xml_data["netCash"]), } return DictToObject(ofx_dict) def xml_cash_interpreter(self, xml_data): # map, with ofx fields on the left and xml fields on the right ofx_dict = { - 'tradeDate': self.convert_date(xml_data['dateTime']), - 'amount': D(xml_data['amount']), - 'security': xml_data.get('isin', None), - 'type': 'cash', - 'memo': xml_data['type'], + "tradeDate": self.convert_date(xml_data["dateTime"]), + "amount": D(xml_data["amount"]), + "security": xml_data.get("isin", None), + "type": "cash", + "memo": xml_data["type"], } - if xml_data['type'] == 'Dividends': - ofx_dict['type'] = 'dividends' - ofx_dict['total'] = ofx_dict['amount'] + if xml_data["type"] == "Dividends": + ofx_dict["type"] = "dividends" + ofx_dict["total"] = ofx_dict["amount"] return DictToObject(ofx_dict) def get_transactions(self): - yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade', - xml_interpreter=self.xml_trade_interpreter) - yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction', - xml_interpreter=self.xml_cash_interpreter) - yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer', - xml_interpreter=self.xml_transfer_interpreter) + yield from self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade", + xml_interpreter=self.xml_trade_interpreter, + ) + yield from self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction", + xml_interpreter=self.xml_cash_interpreter, + ) + yield from self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer", + xml_interpreter=self.xml_transfer_interpreter, + ) def get_balance_assertion_date(self): - ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0] - return self.convert_date(ac['toDate']).date() + ac = list( + self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency" + ) + )[0] + return self.convert_date(ac["toDate"]).date() def get_available_cash(self, settlement_fund_balance=0): """Assumes there's only one cash currency. TODO: get investments transaction builder to accept date from get_available_cash """ - ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0] - return D(ac['slbNetCash']) + ac = list( + self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency" + ) + )[0] + return D(ac["slbNetCash"]) def get_balance_positions(self): - for pos in self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition'): + for pos in self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition" + ): balance = { - 'security': pos['isin'], - 'units': D(pos['position']), + "security": pos["isin"], + "units": D(pos["position"]), } yield DictToObject(balance) diff --git a/beancount_reds_importers/importers/ibkr/flexquery_download.py b/beancount_reds_importers/importers/ibkr/flexquery_download.py index 3512f2c..b4be179 100755 --- a/beancount_reds_importers/importers/ibkr/flexquery_download.py +++ b/beancount_reds_importers/importers/ibkr/flexquery_download.py @@ -1,33 +1,32 @@ #!/usr/bin/env python3 """IBKR Flex Query Downloader""" -import requests import click +import requests + @click.command() -@click.argument('token', required=True) -@click.argument('query_id', required=True) +@click.argument("token", required=True) +@click.argument("query_id", required=True) def flexquery_download(token, query_id): - url = "https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest" - + url = ( + "https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest" + ) + # Request Flex Query - request_payload = { - "v": "3", - "t": token, - "q": query_id - } - + request_payload = {"v": "3", "t": token, "q": query_id} + response = requests.post(url, data=request_payload) - + if response.status_code == 200: request_id = response.text.split("")[1].split("")[0] # print(f"Request ID: {request_id}") - + # Construct URL to get the query result result_url = f"https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.GetStatement?q={request_id}&t={token}&v=3" - + result_response = requests.get(result_url) - + if result_response.status_code == 200: print(result_response.text) else: @@ -37,5 +36,6 @@ def flexquery_download(token, query_id): print(f"Failed to request the query. Status Code: {response.status_code}") return None -if __name__ == '__main__': + +if __name__ == "__main__": flexquery_download() diff --git a/beancount_reds_importers/importers/vanguard/__init__.py b/beancount_reds_importers/importers/vanguard/__init__.py index 47faab5..76f48e9 100644 --- a/beancount_reds_importers/importers/vanguard/__init__.py +++ b/beancount_reds_importers/importers/vanguard/__init__.py @@ -45,7 +45,7 @@ def cleanup_memo(self, ot): # some vanguard files have memos repeated like this: # 'DIVIDEND REINVESTMENTDIVIDEND REINVESTMENT' retval = ot.memo - if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2):]: + if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2) :]: retval = ot.memo[: int(len(ot.memo) / 2)] return retval diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 0d046d3..d469439 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -10,8 +10,10 @@ import json from beancount.ingest import importer + from beancount_reds_importers.libreader import reader + class Importer(reader.Reader, importer.ImporterProtocol): FILE_EXTS = ["json"] @@ -19,7 +21,7 @@ def initialize_reader(self, file): if getattr(self, "file", None) != file: self.file = file self.reader_ready = False - with open(file.name, 'r') as f: + with open(file.name, "r") as f: self.json_data = json.load(f) self.reader_ready = self.deep_identify(file) if self.reader_ready: @@ -39,14 +41,14 @@ def file_date(self, file): return None def read_file(self, file): - with open(file.name, 'r') as f: + with open(file.name, "r") as f: self.json_data = json.load(f) def get_json_elements(self, json_path, json_interpreter=lambda x: x): """Extract a list of elements in the JSON file at the given JSON path. Typically, transactions are stored in a JSON path, and this extracts them.""" elements = self.json_data - for key in json_path.split('.'): + for key in json_path.split("."): if key in elements: elements = elements[key] else: diff --git a/beancount_reds_importers/libreader/xmlreader.py b/beancount_reds_importers/libreader/xmlreader.py index f7d1446..237cebc 100644 --- a/beancount_reds_importers/libreader/xmlreader.py +++ b/beancount_reds_importers/libreader/xmlreader.py @@ -5,9 +5,9 @@ """ +from beancount.ingest import importer from lxml import etree -from beancount.ingest import importer from beancount_reds_importers.libreader import reader diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 38d1986..f54eec2 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -163,8 +163,8 @@ def get_ticker_info_from_id(self, security_id): except IndexError: print(f"Error: fund info not found for {security_id}", file=sys.stderr) securities = self.get_security_list() - if '' in securities: - securities.remove('') + if "" in securities: + securities.remove("") securities_missing = list(securities) for s in securities: for k in self.funds_db: