reconciler: Fix totals. Never consider payee if check_id in books.

conservancy_beancount/reconcile/prototype_amex_reconciler.py

@@ -215,14 +215,20 @@ def records_match(r1: Dict, r2: Dict) -> Tuple[bool, str]:
     else:
         amount_score, amount_message = 0.0, 'amount mismatch'
 
+    # We never consider payee if there's a check_id in the books.
+    check_message = ''
+    payee_message = ''
+    if r2['check_id']:
+        payee_score = 0.0
         if r1['check_id'] and r2['check_id'] and r1['check_id'] == r2['check_id']:
             check_score = 1.0
+        else:
+            check_message = 'Check id not matched'
+            check_score = 0.0
     else:
         check_score = 0.0
-
         payee_score = payee_match(r1['payee'], r2['payee'])
-
+        if payee_score > 0.8:
-    if check_score == 1.0 or payee_score > 0.8:
             payee_message = ''
         elif payee_score > 0.4:
             payee_message = 'partial payee match'
@@ -230,7 +236,7 @@ def records_match(r1: Dict, r2: Dict) -> Tuple[bool, str]:
             payee_message = 'payee mismatch'
 
     overall_score = (date_score + amount_score + check_score + payee_score) / 4
-    overall_message = [m for m in [date_message, amount_message, payee_message] if m]
+    overall_message = [m for m in [date_message, amount_message, check_message, payee_message] if m]
     return overall_score, overall_message
 
 
@@ -273,12 +279,13 @@ def match_statement_and_books(statement_trans: list, books_trans: list):
     return matches
 
 
-def format_matches(matches, csv_statement: str):
+def format_matches(matches, csv_statement: str, show_reconciled_matches):
     match_output = []
     for r1, r2, note in matches:
         note = ', '.join(note)
         note = ': ' + note if note else note
         if r1 and r2:
+            if show_reconciled_matches:
                 match_output.append([r1[0]['date'], f'{format_record(r1)}  →  {format_record(r2)}  ✓ Matched{note}'])
         elif r1:
             match_output.append([r1[0]['date'], f'{format_record(r1)}  →  {" ":^59}  ✗ NOT IN BOOKS ({os.path.basename(csv_statement)}:{r1[0]["line"]})'])
@@ -287,10 +294,6 @@ def format_matches(matches, csv_statement: str):
     return match_output
 
 
-# TODO: Could potentially return a score so that we can find the best match from
-# a pool of candidates. How would be then remove that candidate from the global
-# pool?
-
 def date_proximity(d1, d2):
     diff = abs((d1 - d2).days)
     if diff > 60:
@@ -377,6 +380,18 @@ def parse_args(argv):
     parser.add_argument('--non-interactive', action='store_true', help="Don't prompt to write to the books")
     return parser.parse_args(args=argv[1:])
 
+def totals(matches):
+    total_matched = decimal.Decimal(0)
+    total_missing_from_books = decimal.Decimal(0)
+    total_missing_from_statement = decimal.Decimal(0)
+    for statement_entries, books_entries, _ in matches:
+        if statement_entries and books_entries:
+            total_matched += sum(c['amount'] for c in statement_entries)
+        elif statement_entries:
+            total_missing_from_books += sum(c['amount'] for c in statement_entries)
+        else:
+            total_missing_from_statement += sum(c['amount'] for c in books_entries)
+    return total_matched, total_missing_from_books, total_missing_from_statement
 
 def main(args):
     # TODO: Should put in a sanity check to make sure the statement you're feeding
@@ -422,13 +437,11 @@ def main(args):
     books_trans = sort_records([standardize_beancount_record(row) for row in result_rows])
 
     matches = match_statement_and_books(statement_trans, books_trans)
-    match_output = format_matches(matches, args.csv_statement)
+    match_output = format_matches(matches, args.csv_statement, args.show_reconciled_matches)
 
     # assert books_balance == books_balance_reconciled + total_matched + total_missing_from_statement
 
-    total_matched = 0
+    total_matched, total_missing_from_books, total_missing_from_statement = totals(matches)
-    total_missing_from_statement = 0
-    total_missing_from_books = 0
 
     out = io.StringIO()
     print('-' * 155)
@@ -437,19 +450,18 @@ def main(args):
     for _, output in sorted(match_output):
         print(output)
     print('-' * 155)
-    print(f'Statement period: {begin_date} to {end_date}')
+    print(f'Statement period {begin_date} to {end_date}')
     print(f'Statement/cleared balance:    {args.statement_balance:12,.2f}    (as provided by you)')
-    print(f'Books balance (all):          {books_balance:12,.2f}    (all transactions, includes unreconciled)')
+    print(f'Books balance:                {books_balance:12,.2f}    (all transactions, includes unreconciled)')
-    print(f'Books balance (reconciled):   {books_balance_reconciled:12,.2f}    (transactions with "bank-statement" tag only)')
+    # print(f'Books balance (reconciled):   {books_balance_reconciled:12,.2f}    (transactions with "bank-statement" tag only)')
-    print(f'Matched above:                {total_matched:12,.2f}    ("bank-statement" tag yet to be applied)')
+    # print(f'Matched above:                {total_matched:12,.2f}    ("bank-statement" tag yet to be applied)')
-    print(f'Unmatched on books:           {total_missing_from_statement:12,.2f}')
+    print(f'Total not on statement:       {total_missing_from_statement:12,.2f}')
-    print(f'Unmatched statement:          {total_missing_from_books:12,.2f}')
+    print(f'Total not on books:           {total_missing_from_books:12,.2f}')
     print('-' * 155)
 
     # Write statement metadata back to books
     metadata_to_apply = []
     for match in matches:
-        # TODO: Shouldn't write if no match.
         metadata_to_apply.extend(metadata_for_match(match, args.bank_statement, args.csv_statement))
     if metadata_to_apply and not args.non_interactive:
         print('Mark matched transactions as reconciled in the books? (y/N) ', end='')

tests/test_reconcile.py

@@ -11,7 +11,8 @@ from conservancy_beancount.reconcile.prototype_amex_reconciler import (
     remove_duplicate_words,
     payee_match,
     metadata_for_match,
-    write_metadata_to_books
+    write_metadata_to_books,
+    totals,
 )
 
 S1 = {
@@ -199,6 +200,10 @@ def test_metadata_for_match(monkeypatch):
         ('2022/imports.beancount', 777, '    bank-statement-csv: statement.csv:222'),
     ]
 
+def test_no_metadata_if_no_matches():
+    assert metadata_for_match(([S1], [], ['no match']), 'statement.pdf', 'statement.csv') == []
+    assert metadata_for_match(([], [B1], ['no match']), 'statement.pdf', 'statement.csv') == []
+    assert metadata_for_match(([S1], [B2], ['no match']), 'statement.pdf', 'statement.csv') == []
 
 def test_write_to_books():
     books = textwrap.dedent("""\
@@ -218,3 +223,10 @@ def test_write_to_books():
             bank-statement: statement.pdf
           Expenses:Hosting                        15.50 USD""")
     os.remove(f.name)
+
+def test_totals():
+    assert totals([
+        ([S1], [B1], []),
+        ([S2], [], []),
+        ([], [B3_next_day], []),
+    ]) == (decimal.Decimal('10'), decimal.Decimal('20'), decimal.Decimal('30'))