diff --git a/conservancy_beancount/reconcile/helper.py b/conservancy_beancount/reconcile/helper.py index e24abf9..ea82193 100644 --- a/conservancy_beancount/reconcile/helper.py +++ b/conservancy_beancount/reconcile/helper.py @@ -15,6 +15,7 @@ Not implemented: """ import argparse +import csv from dateutil.relativedelta import relativedelta import datetime import decimal @@ -80,6 +81,38 @@ def tabulate(rows: List, headers: List=None) -> str: return output.getvalue().strip() +def reconciliation_report(account, end_date, bank_account_balance, uncleared, prev_end_date, our_account_balance, prev_uncleared): + *_, account_name = account.rpartition(':') + # end_date_iso = end_date.isoformat() + # prev_end_date_iso = prev_end_date.isoformat() + output = io.StringIO() + w = csv.writer(output) + w.writerow([f'title:{end_date}: {account_name}']) + w.writerow(['BANK RECONCILIATION', 'ENDING', end_date]) + w.writerow([f' {account}']) + w.writerow([]) + w.writerow(['DATE', 'CHECK NUM', 'PAYEE', 'AMOUNT']) + w.writerow([]) + w.writerow([end_date, '', 'BANK ACCOUNT BALANCE', bank_account_balance]) + w.writerow([]) + for trans in uncleared: + w.writerow(trans) + w.writerow([]) + w.writerow([end_date, '', 'OUR ACCOUNT BALANCE', our_account_balance]) + if prev_uncleared: + w.writerow([]) + w.writerow(['Items Still', 'Outstanding On', prev_end_date, 'Appeared By', end_date]) + w.writerow([]) + for trans in prev_uncleared: + w.writerow(trans) + return output.getvalue() + + +def reconciliation_report_path(account, end_date): + *_, account_name = account.rpartition(':') + return f'Financial/Controls/Reports-for-Treasurer/{end_date}_{account_name}_bank-reconciliation.csv' + + # Parse all the arguments parser = argparse.ArgumentParser(description='Reconciliation helper') parser.add_argument('--beancount-file', required=True) @@ -167,6 +200,9 @@ QUERIES = { # Run Beancount queries. print(f"START RECONCILIATION FOR {account} ENDING {lastDateInPeriod} (previous end date {preDate})") entries, _, options = loader.load_file(beancount_file) +uncleared_rows = [] # Hack to capture results of query 03. +cleared_balance = decimal.Decimal('0') +all_trans_balance = decimal.Decimal('0') for desc, query in QUERIES.items(): rtypes, rrows = run_query(entries, options, query, numberify=True) if not rrows: @@ -179,7 +215,19 @@ for desc, query in QUERIES.items(): elif len(rrows) == 1 and isinstance(rrows[0][0], decimal.Decimal): result = rrows[0][0] print(f'{desc:<55} {result:11,.2f}') + if desc.startswith('02'): + all_trans_balance = result + if desc.startswith('05'): + cleared_balance = result else: headers = [c[0].capitalize() for c in rtypes] + if desc.startswith('03'): + uncleared_rows = rrows print(desc) print(textwrap.indent(tabulate(rrows, headers=headers), ' ')) + +uncleared = [(r[0], r[2], r[4] or r[3], r[1]) for r in uncleared_rows] +report_path = os.path.join(os.getenv('CONSERVANCY_REPOSITORY', ''), reconciliation_report_path(account, lastDateInPeriod)) +with open(report_path, 'w') as f: + f.write(reconciliation_report(account, lastDateInPeriod, cleared_balance, uncleared, '1900-01-01', all_trans_balance, [])) +print(f'Wrote reconciliation report: {report_path}.') diff --git a/conservancy_beancount/reconcile/statement_reconciler.py b/conservancy_beancount/reconcile/statement_reconciler.py index df665ed..b907118 100644 --- a/conservancy_beancount/reconcile/statement_reconciler.py +++ b/conservancy_beancount/reconcile/statement_reconciler.py @@ -38,9 +38,6 @@ Q. How are reconciliation reports created currently? How do you read them? Problem is potentially similar to diff-ing, but in the books, transaction order isn't super significant. -TODO/ISSUES: - - AMEX statement doesn't provide bank balance or running total - """ import argparse import collections @@ -96,6 +93,7 @@ JUNK_WORDS = [ JUNK_WORDS_RES = [re.compile(word, re.IGNORECASE) for word in JUNK_WORDS] ZERO_RE = re.compile('^0+') + def remove_duplicate_words(text: str) -> str: unique_words = [] known_words = set() @@ -131,13 +129,13 @@ def read_transactions_from_csv(f: TextIO, standardize_statement_record: Callable return sort_records([standardize_statement_record(row, i) for i, row in enumerate(reader, 2)]) -# NOTE: Statement doesn't seem to give us a running balance or a final total. # CSV reconciliation report. -# Merge helper script. +# Merge helper script? def standardize_amex_record(row: Dict, line: int) -> Dict: """Turn an AMEX CSV row into a standard dict format representing a transaction.""" + # NOTE: Statement doesn't seem to give us a running balance or a final total. return { 'date': datetime.datetime.strptime(row['Date'], '%m/%d/%Y').date(), 'amount': -1 * decimal.Decimal(row['Amount']), @@ -197,7 +195,6 @@ def format_record(record: dict) -> str: def format_multirecord(r1s: list[dict], r2s: list[dict], note: str) -> list[list]: - total = sum(x['amount'] for x in r2s) assert len(r1s) == 1 assert len(r2s) > 1 match_output = [] @@ -219,7 +216,7 @@ def first_word_exact_match(a: str, b: str) -> float: if first_a.casefold() == first_b.casefold(): return min(1.0, 0.2 * len(first_a)) else: - return 0.0; + return 0.0 def payee_match(a: str, b: str) -> float: @@ -399,7 +396,7 @@ def parse_repo_relative_path(path: str) -> str: raise argparse.ArgumentTypeError(f'File {path} does not exist.') repo = os.getenv('CONSERVANCY_REPOSITORY') if not repo: - raise argparse.ArgumentTypeError(f'$CONSERVANCY_REPOSITORY is not set.') + raise argparse.ArgumentTypeError('$CONSERVANCY_REPOSITORY is not set.') if not path.startswith(repo): raise argparse.ArgumentTypeError(f'File {path} does not share a common prefix with $CONSERVANCY_REPOSITORY {repo}.') return path @@ -433,13 +430,13 @@ def totals(matches: List[Tuple[List, List, List]]) -> Tuple[decimal.Decimal, dec return total_matched, total_missing_from_books, total_missing_from_statement -def subset_match(statement_trans: List[dict], books_trans: List[dict]) -> Tuple[List[Tuple[List, List, List]], List[Dict], List[Dict]]: +def subset_match(statement_trans: List[dict], books_trans: List[dict]) -> Tuple[List[Tuple[List, List, List]], List[Dict], List[Dict]]: matches = [] remaining_books_trans = [] remaining_statement_trans = [] groups = itertools.groupby(books_trans, key=lambda x: (x['date'], x['payee'])) - for k, group in groups: + for _, group in groups: best_match_score = 0.0 best_match_index = None best_match_note = [] @@ -466,7 +463,7 @@ def subset_match(statement_trans: List[dict], books_trans: List[dict]) -> Tuple else: remaining_books_trans.append(r2) for r1 in statement_trans: - remaining_statement_trans.append(r1) + remaining_statement_trans.append(r1) return matches, remaining_statement_trans, remaining_books_trans @@ -512,18 +509,13 @@ def main(args: argparse.Namespace) -> None: books_balance_query = f"""SELECT sum(COST(position)) AS aa WHERE account = "{args.account}" AND date <= {end_date.isoformat()}""" - result_types, result_rows = run_query(entries, options, books_balance_query, numberify=True) + _, result_rows = run_query(entries, options, books_balance_query, numberify=True) books_balance = result_rows[0][0] if result_rows else 0 - books_balance_reconciled_query = f"""SELECT sum(COST(position)) AS aa WHERE account = "{args.account}" - AND date <= {end_date.isoformat()} AND META('bank-statement') != NULL""" - result_types, result_rows = run_query(entries, options, books_balance_reconciled_query, numberify=True) - books_balance_reconciled = result_rows[0][0] if result_rows else 0 - # String concatenation looks bad, but there's no SQL injection possible here # because BQL can't write back to the Beancount files. I hope! query = f'SELECT filename, META("lineno") AS line, META("bank-statement") AS bank_statement, date, number(cost(position)), payee, ENTRY_META("entity") as entity, ANY_META("check-id") as check_id, narration where account = "{args.account}" and date >= {begin_date} and date <= {end_date}' - result_types, result_rows = run_query(entries, options, query) + _, result_rows = run_query(entries, options, query) books_trans = sort_records([standardize_beancount_record(row) for row in result_rows]) @@ -535,9 +527,8 @@ def main(args: argparse.Namespace) -> None: match_output = format_matches(matches, args.csv_statement, args.show_reconciled_matches) - total_matched, total_missing_from_books, total_missing_from_statement = totals(matches) + _, total_missing_from_books, total_missing_from_statement = totals(matches) - out = io.StringIO() print('-' * 155) print(f'{"Statement transaction":<52} {"Books transaction":<58} Notes') print('-' * 155) @@ -547,8 +538,6 @@ def main(args: argparse.Namespace) -> None: print(f'Statement period {begin_date} to {end_date}') print(f'Statement/cleared balance: {args.statement_balance:12,.2f} (as provided by you)') print(f'Books balance: {books_balance:12,.2f} (all transactions, includes unreconciled)') - # print(f'Books balance (reconciled): {books_balance_reconciled:12,.2f} (transactions with "bank-statement" tag only)') - # print(f'Matched above: {total_matched:12,.2f} ("bank-statement" tag yet to be applied)') print(f'Total not on statement: {total_missing_from_statement:12,.2f}') print(f'Total not on books: {total_missing_from_books:12,.2f}') print('-' * 155) @@ -566,7 +555,3 @@ def main(args: argparse.Namespace) -> None: if __name__ == '__main__': args = parse_args(sys.argv) main(args) - -# Local Variables: -# python-shell-interpreter: "/home/ben/\.virtualenvs/conservancy-beancount-py39/bin/python" -# End: