feat: xml reader + ibkr flex query importer

beancount_reds_importers/importers/ibkr/__init__.py

@@ -0,0 +1,162 @@
+"""IBKR Flex Query importer for beancount.
+
+Activity Flex Query Details
+Query ID XXX
+Query Name XXX
+
+Sections
+========
+
+Account Information
+-------------------
+1.ClientAccountID
+2.CurrencyPrimary
+
+Cash Transactions
+-----------------
+
+Options: Dividends, Payment in Lieu of Dividends, Withholding Tax, 871(m) Withholding, Advisor Fees, Other Fees, Deposits & Withdrawals, Carbon Credits, Bill Pay, Broker Interest Paid, Broker Interest Received, Broker Fees, Bond Interest Paid, Bond Interest Received, Price Adjustments, Commission Adjustments, Detail
+
+1.Date/Time
+2.Amount
+3.Type
+4.CurrencyPrimary
+5.Symbol
+6.CommodityType
+7.ISIN
+
+Net Stock Position Summary
+--------------------------
+1.Symbol
+2.CUSIP
+
+Open Dividend Accruals
+----------------------
+1.Symbol
+2.GrossAmount
+3.NetAmount
+4.PayDate
+5.Quantity
+6.ISIN
+
+Trades
+------
+Options: Execution
+1.SecurityID
+2.DateTime
+3.TransactionType
+4.Quantity
+5.TradePrice
+6.TradeMoney
+7.Proceeds
+8.IBCommission
+9.IBCommissionCurrency
+10.NetCash
+11.CostBasis
+12.FifoPnlRealized
+13.Buy/Sell
+14.CurrencyPrimary
+15.ISIN
+
+
+Delivery Configuration
+----------------------
+Accounts
+Format XML
+Period Last N Calendar Days
+Number of Days 120
+
+
+General Configuration
+---------------------
+Profit and Loss Default
+Include Canceled Trades? No
+Include Currency Rates? No
+Include Audit Trail Fields? No
+Display Account Alias in Place of Account ID? No
+Breakout by Day? No
+Date Format yyyy-MM-dd
+Time Format HH:mm:ss TimeZone
+Date/Time Separator ' ' (single-space)
+
+"""
+
+import datetime
+from beancount_reds_importers.libreader import xmlreader
+from beancount_reds_importers.libtransactionbuilder import investments
+from beancount.core.number import D
+
+class DictToObject:
+    def __init__(self, dictionary):
+        for key, value in dictionary.items():
+            setattr(self, key, value)
+
+# xml on left, ofx on right
+ofx_type_map = {
+    'BUY': 'buystock',
+    'SELL': 'selltock',
+}
+
+
+class Importer(investments.Importer, xmlreader.Importer):
+    IMPORTER_NAME = "IBKR Flex Query"
+
+    def custom_init(self):
+        if not self.custom_init_run:
+            self.max_rounding_error = 0.04
+            self.filename_pattern_def = "Transaction_report"
+            self.custom_init_run = True
+            self.date_format = '%Y-%m-%d'
+            self.get_ticker_info = self.get_ticker_info_from_id
+
+    def set_currency(self):
+        self.currency = list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['currency']
+
+    # fixup dates
+    def convert_date(self, d):
+        d = d.split(' ')[0]
+        return datetime.datetime.strptime(d, self.date_format)
+
+    def trade_to_ofx_dict(self, xml_data):
+        # Mapping the input dictionary to the OFX dictionary format
+        ofx_dict = {
+            'security':   xml_data['isin'],
+            'tradeDate':  self.convert_date(xml_data['dateTime']),
+            'memo':       xml_data['transactionType'],
+            'type':       ofx_type_map[xml_data['buySell']],
+            'units':      D(xml_data['quantity']),
+            'unit_price': D(xml_data['tradePrice']),
+            'commission': D(xml_data['ibCommission']),
+            'total':      D(xml_data['proceeds']),
+        }
+        return ofx_dict
+
+    def cash_to_ofx_dict(self, xml_data):
+        # Mapping the input dictionary to the OFX dictionary format
+        ofx_dict = {
+            'tradeDate':  self.convert_date(xml_data['dateTime']),
+            'amount':     D(xml_data['amount']),
+            'security':   getattr(xml_data, 'isin', None),
+            'type':       'cash',
+            'memo':       xml_data['type'],
+        }
+
+        if xml_data['type'] == 'Dividends':
+            ofx_dict['type'] = 'dividends'
+            ofx_dict['total'] = ofx_dict['amount']
+
+        return ofx_dict
+
+    def xml_trade_interpreter(self, element):
+        ot = self.trade_to_ofx_dict(element)
+        return DictToObject(ot)
+
+    def xml_cash_interpreter(self, element):
+        ot = self.cash_to_ofx_dict(element)
+        return DictToObject(ot)
+
+    def get_transactions(self):
+        yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade',
+                                           xml_interpreter=self.xml_trade_interpreter)
+        yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction',
+                                           xml_interpreter=self.xml_cash_interpreter)

beancount_reds_importers/libreader/reader.py

@@ -24,9 +24,13 @@ def identify(self, file):
             return False
         self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED")
         self.initialize_reader(file)
-        # print("reader_ready:", self.reader_ready)
+        # print("reader_ready:", self.reader_ready, self.IMPORTER_NAME)
         return self.reader_ready
 
+    def set_currency(self):
+        """For overriding"""
+        self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED")
+
     def file_name(self, file):
         return "{}".format(ntpath.basename(file.name))
 
@@ -55,5 +59,8 @@ def get_balance_statement(self, file=None):
     def get_balance_positions(self):
         return []
 
+    def get_balance_assertion_date(self):
+        return None
+
     def get_available_cash(self, settlement_fund_balance=0):
         return None

beancount_reds_importers/libreader/xmlreader.py

@@ -0,0 +1,58 @@
+"""XML reader for beancount-reds-importers.
+
+XML files have widely varying specifications, and thus, this is a very generic reader, and most of
+the logic will have to be the institution specific readers.
+
+"""
+
+import datetime
+import warnings
+from collections import namedtuple
+from lxml import etree
+
+from beancount.ingest import importer
+from beancount_reds_importers.libreader import reader
+
+
+
+class Importer(reader.Reader, importer.ImporterProtocol):
+    FILE_EXTS = ["xml"]
+
+    def initialize_reader(self, file):
+        if getattr(self, "file", None) != file:
+            self.file = file
+            self.reader_ready = False
+            try:
+                self.xmltree = etree.parse(file.name)
+            except:
+                return
+            self.reader_ready = self.deep_identify()
+        if self.reader_ready:
+            self.set_currency()
+
+    def deep_identify(self):
+        """For overriding by institution specific importer which can check if an account name
+        matches, and oother such things."""
+        return True
+
+    def file_date(self, file):
+        """Get the ending date of the statement."""
+        if not getattr(self, "xmltree", None):
+            self.initialize(file)
+        # TODO:
+        return None
+
+    def read_file(self, file):
+        self.xmltree = etree.parse(file.name)
+
+    def get_xpath_elements(self, xpath_expr, xml_interpreter=lambda x: x):
+        """Extract a list of elements in the XML file at the given XPath expression. Typically,
+        transactions are stored in an xml path, and this extracts them."""
+        elements = self.xmltree.xpath(xpath_expr)
+        for elem in elements:
+            yield xml_interpreter(elem.attrib)
+
+    def get_transactions(self):
+        """/Transactions/Transaction is a dummy default path for transactions that needs to be
+        overriden in the institution specific importer."""
+        yield from self.get_xpath_elements("/Transactions/Transaction")