fix: resolved several gotchas with balance assertion dates with ofx f…

…iles

beancount_reds_importers/libreader/csvreader.py

@@ -229,7 +229,7 @@ def get_max_transaction_date(self):
         except Exception as err:
             print("ERROR: no end_date. SKIPPING input.")
             traceback.print_tb(err.__traceback__)
-            return False
+            return None
 
         return date
 

beancount_reds_importers/libreader/ofxreader.py

@@ -76,26 +76,56 @@ def get_available_cash(self, settlement_fund_balance=0):
             return available_cash - settlement_fund_balance
         return None
 
-    def get_ofx_end_date(self):
-        end_date = self.ofx_account.statement.end_date
-        # convert end_date from utc to local timezone
-        end_date = end_date.replace(tzinfo=datetime.timezone.utc).astimezone().date()
-        return end_date
+    def get_ofx_end_date(self, field='end_date'):
+        end_date = getattr(self.ofx_account.statement, field, None)
+
+        # Convert end_date from utc to local timezone if needed and return
+        # This is needed only if there is an actual timestamp other than time(0, 0)
+        if end_date:
+            if end_date.time() != datetime.time(0, 0):
+                end_date = end_date.replace(tzinfo=datetime.timezone.utc).astimezone()
+            return end_date.date()
+
+        return None
 
     def get_smart_date(self):
-        """ We find the statement's end date from the OFX file. However, banks and credit cards
+        """ We want the latest date we can assert balance on. Let's consider all the dates we have:
+        b--------e-------(s-2)----(s)----(d)
+
+        - b: date of first transaction in this ofx file (end_date)
+        - e: date of last  transaction in this ofx file (max_transaction_date)
+        - s: statement's end date as claimed by the ofx import fileinput (acc.statement.available_balance_date)
+        - d: date of download
+
+        Ideally, we would assert balance end of the day on (s) above. However, banks and credit cards
         typically have pending transactions that are not included in downloads. When we download
         the next statement, new transactions may appear prior to the balance assertion date that we
         generate for this statement. To attempt to avoid this, we set the balance assertion date to
-        either two days before the statement's end date or the last transaction's date, whichever
-        is later.
+        (s-2), where 2 is the fudge factor (the time where pending transactions not seen in this
+        statement might occur in the next statement).
+
+        Not all ofx files have all these dates. Hence we compute this date with the best info we
+        have.
         """
-        end_date = self.get_ofx_end_date()
-        end_date -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2))
 
+        ofx_max_transation_date = self.get_ofx_end_date('end_date')
+        ofx_balance_date1 = self.get_ofx_end_date('available_balance_date')
+        ofx_balance_date2 = self.get_ofx_end_date('balance_date')
         max_transaction_date = self.get_max_transaction_date()
-        max_transaction_date = max_transaction_date if max_transaction_date else datetime.date.min
-        return_date = max(end_date, max_transaction_date)
+
+        if ofx_balance_date1:
+            ofx_balance_date1 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2))
+        if ofx_balance_date2:
+            ofx_balance_date2 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2))
+
+        dates = [ofx_max_transation_date, max_transaction_date, ofx_balance_date1, ofx_balance_date2]
+        if all(v is None for v in dates[:2]):  # because ofx_balance_date appears even for closed accounts
+            return None
+
+        def vd(x): return x if x else datetime.date.min
+        return_date = max(*[vd(x) for x in dates])
+        print("Smart date computation. Dates were: ", dates)
+
         return return_date
 
     def get_balance_assertion_date(self):
@@ -119,6 +149,8 @@ def get_balance_assertion_date(self):
         date_type = self.config.get('balance_assertion_date_type', 'smart')
         return_date = date_type_map[date_type]()
 
+        if not return_date:
+            return None
         return return_date + datetime.timedelta(days=1)  # Next day, as defined by Beancount
 
     def get_max_transaction_date(self):
@@ -135,7 +167,7 @@ def get_max_transaction_date(self):
             date = max(ot.tradeDate if hasattr(ot, 'tradeDate') else ot.date
                        for ot in self.get_transactions()).date()
         except TypeError:
-            return False
+            return None
         except ValueError:
-            return False
+            return None
         return date

beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx

@@ -46,7 +46,7 @@ CHECKING</ACCTTYPE>
 <DTSTART>
 20190503000000</DTSTART>
 <DTEND>
-20190901000000</DTEND>
+20190109000000</DTEND>
 
 <STMTTRN>
 <TRNTYPE>
@@ -81,7 +81,7 @@ Check 23400</NAME>
 <BALAMT>
 150.65</BALAMT>
 <DTASOF>
-20150101000000.000[+0:UTC]</DTASOF>
+20190120000000.000[+0:UTC]</DTASOF>
 </LEDGERBAL>
 </STMTRS>
 </STMTTRNRS>

beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.extract

@@ -5,4 +5,4 @@
 2019-01-09 * "Check 23400"
   Assets:Banks:Checking  -1000 USD
 
-2019-08-30 balance Assets:Banks:Checking                           150.65 USD
+2019-01-19 balance Assets:Banks:Checking                           150.65 USD

beancount_reds_importers/libreader/tests/balance_assertion_date/smart/transactions.qfx.file_date

@@ -1 +1 @@
-2019-09-01T00:00:00
+2019-01-09T00:00:00