statement_reconciler: Add initial Chase bank CSV statement matching

We currently don't have many examples to work with, so haven't done any
significant testing of the matching accuracy between statement and books.

conservancy_beancount/plugin/meta_payroll_type.py

@@ -10,7 +10,6 @@ import datetime
 from . import core
 from .. import ranges
 from .. import data
-from .. import errors as errormod
 
 from ..beancount_types import (
     Transaction,
@@ -18,6 +17,7 @@ from ..beancount_types import (
 
 METADATA_KEY = 'payroll-type'
 
+
 class _PayrollTypeHook(core._NormalizePostingMetadataHook):
     ACCOUNT: str
     VALUES_ENUM = core.MetadataEnum(METADATA_KEY, [])
@@ -74,6 +74,7 @@ class SalaryHook(_PayrollTypeHook):
         'CA:PTO:Earned',
         'CA:PTO:Taken',
         'US:403b:Employee',
+        'US:403b:Employee:Roth',
         'US:403b:Match',
         'US:General',
         'US:MA:Disability:PFL',
@@ -98,6 +99,7 @@ class TaxHook(_PayrollTypeHook):
         'CA:PP',
         'CA:WCB',
         'US:IL:Unemployment',
+        'US:TN:Unemployment',
         'US:MA:Health',
         'US:MA:Unemployment',
         'US:MA:WorkTrain',

conservancy_beancount/plugin/meta_tax_implication.py

@@ -54,6 +54,7 @@ class MetaTaxImplication(core._NormalizePostingMetadataHook):
         'USA-Corporation',
         'USA-Grantee',
         'W2',
+        'Asset-Sale'
     ]
     _ALIASES = dict(
         alias for value in _STDNAMES for alias in _make_aliases(value)

conservancy_beancount/reconcile/helper.py

@@ -12,7 +12,6 @@ In the spirit of bc-reconcile-helper.plx (the original Perl code)
 Not implemented:
  - --report-group-regex
  - git branch selection from bean-query-goofy-daemon.plx
-
 """
 import argparse
 import csv

conservancy_beancount/reconcile/statement_reconciler.py

@@ -17,7 +17,7 @@ representing health insurance for multiple employees.
 
 Run it like this:
 
-$ statement_reconciler \
+$ statement-reconciler \
   --beancount-file 2021.beancount \
   --account Liabilities:CreditCard:AMEX \
   --csv-statement ~/svn/2021-09-10_AMEX_activity.csv \
@@ -270,6 +270,41 @@ def read_fr_csv(f: TextIO) -> list:
     )
 
 
+def validate_chase_csv(sample: str) -> None:
+    required_cols = {'Date', 'Description', 'Account', 'Transaction Type', 'Amount'}
+    reader = csv.DictReader(io.StringIO(sample))
+    if reader.fieldnames and not required_cols.issubset(reader.fieldnames):
+        sys.exit(
+            f"This Chase CSV doesn't seem to have the columns we're expecting, including: {', '.join(required_cols)}. Please use an unmodified statement direct from the institution."
+        )
+
+
+def standardize_chase_record(row: Dict, line: int) -> Dict:
+    """Turn an Chase CSV row into a standard dict format representing a transaction."""
+    return {
+        'date': datetime.datetime.strptime(row['Date'], '%m/%d/%y').date(),
+        'amount': -1 * parse_amount(row['Amount']),
+        # Descriptions have quite a lot of information, but the format is a little
+        # idiosyncratic. We'll need to see more examples before coming up with any ways
+        # to handle it in code. Others have used regular expressions to match the
+        # various transaction types:
+        # https://github.com/mtlynch/beancount-chase-bank/blob/master/beancount_chase/checking.py
+        # See also: https://awesome-beancount.com/
+        'payee': (row['Description'] or '').replace('ORIG CO NAME:')[:20],
+        'check_id': '',
+        'line': line,
+    }
+
+
+def read_chase_csv(f: TextIO) -> list:
+    reader = csv.DictReader(f)
+    # The reader.line_num is the source line number, not the spreadsheet row
+    # number due to multi-line records.
+    return sort_records(
+        [standardize_chase_record(row, i) for i, row in enumerate(reader, 2)]
+    )
+
+
 def standardize_beancount_record(row) -> Dict:  # type: ignore[no-untyped-def]
     """Turn a Beancount query result row into a standard dict representing a transaction."""
     return {
@@ -666,12 +701,13 @@ def parse_repo_relative_path(path: str) -> str:
     """
     if not os.path.exists(path):
         raise argparse.ArgumentTypeError(f'File {path} does not exist.')
-    repo = os.getenv('CONSERVANCY_REPOSITORY')
+    real_path = os.path.realpath(path)
+    repo = os.path.realpath(os.getenv('CONSERVANCY_REPOSITORY'))
     if not repo:
         raise argparse.ArgumentTypeError('$CONSERVANCY_REPOSITORY is not set.')
-    if not path.startswith(repo):
+    if not real_path.startswith(repo):
         raise argparse.ArgumentTypeError(
-            f'File {path} does not share a common prefix with $CONSERVANCY_REPOSITORY {repo}.'
+            f'File {real_path} does not share a common prefix with $CONSERVANCY_REPOSITORY {repo}.'
         )
     return path
 
@@ -783,9 +819,14 @@ def main(
     if 'AMEX' in args.account:
         validate_csv = validate_amex_csv
         read_csv = read_amex_csv
-    else:
+    elif 'FR' in args.account:
         validate_csv = validate_fr_csv
         read_csv = read_fr_csv
+    elif 'Chase' in args.account:
+        validate_csv = validate_chase_csv
+        read_csv = read_chase_csv
+    else:
+        sys.exit("This account provided doesn't match one of AMEX, FR or Chase.")
 
     with open(args.csv_statement) as f:
         sample = f.read(200)
@@ -834,7 +875,7 @@ def main(
             AND date >= {begin_date}
             AND date <= {end_date}"""
     _, result_rows = run_query(entries, options, query)
-    books_trans = sort_records([standardize_beancount_record(row) for row in result_rows])
+    books_trans = sort_records(standardize_beancount_record(row) for row in result_rows)
 
     # Apply two passes of matching, one for standard matches and one
     # for subset matches.

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = conservancy_beancount
-version = 1.19.8
+version = 1.20.0
 author = Software Freedom Conservancy
 author_email = info@sfconservancy.org
 description = Plugin, library, and reports for reading Conservancy’s books
@@ -32,7 +32,8 @@ warn_unused_configs = True
 include_package_data = True
 install_requires =
   babel>=2.6
-  beancount>=2.2
+  beancount>=2.2,<3.0.0
+  colorama
   GitPython>=2.0
   odfpy>=1.4.0,!=1.4.1
   pdfminer.six>=20200101
@@ -105,4 +106,4 @@ filterwarnings =
   ignore::DeprecationWarning:^socks$
 
 [tox:tox]
-envlist = py36,py37
+envlist = py310