books: All tools use books.Loader.dispatch() and LoadResult.

This is significant code deduplication, as seen in the diffstat.

conservancy_beancount/reconcile/paypal.py

@@ -371,23 +371,17 @@ def main(arglist: Optional[Sequence[str]]=None,
             for post in rec.statement_posts
         ) + datetime.timedelta(days=1)
 
-    returncode = os.EX_OK
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
     date_fuzz = datetime.timedelta(days=args.date_fuzz)
-        entries, load_errors, options = books_loader.load_fy_range(
+    books_load = books.Loader.dispatch(
-            args.start_date - date_fuzz, args.stop_date + date_fuzz,
+        config.books_loader(),
+        args.start_date - date_fuzz,
+        args.stop_date + date_fuzz,
     )
-        if load_errors:
+    books_load.print_errors(stderr)
-            returncode = cliutil.ExitCode.BeancountErrors
+    returncode = books_load.returncode()
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
 
     date_range = DateRange(args.start_date, args.stop_date)
-    for bean_post in data.Posting.from_entries(entries):
+    for bean_post in books_load.iter_postings():
         if bean_post.account != args.account:
             continue
         paypal_post = PayPalPosting.from_books(bean_post)

conservancy_beancount/reconcile/statement.py

@@ -406,19 +406,12 @@ def main(arglist: Optional[Sequence[str]]=None,
     rec_range = DateRange(args.start_date, args.stop_date)
     post_range = DateRange(args.stop_date, args.stop_date + days_diff)
 
-    returncode = os.EX_OK
+    books_load = books.Loader.dispatch(
-    books_loader = config.books_loader()
+        config.books_loader(), pre_range.start, post_range.stop,
-    if books_loader is None:
+    )
-        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
+    books_load.load_account_metadata()
-        returncode = cliutil.ExitCode.NoConfiguration
+    returncode = books_load.returncode()
-    else:
-        entries, load_errors, options = books_loader.load_fy_range(pre_range.start, post_range.stop)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
 
-    data.Account.load_from_books(entries, options)
     real_accounts: Set[data.Account] = set()
     for account_spec in args.accounts:
         new_accounts = frozenset(
@@ -432,14 +425,10 @@ def main(arglist: Optional[Sequence[str]]=None,
             logger.critical("account %r did not match any open accounts", account_spec)
             return 2
 
-    for error in load_errors:
+    books_load.print_errors(stderr)
-        bc_printer.print_error(error, file=stderr)
     rt_wrapper = config.rt_wrapper()
     if rt_wrapper is None:
         logger.warning("could not initialize RT client; spreadsheet links will be broken")
-    postings = data.Posting.from_entries(entries)
-    for search_term in args.search_terms:
-        postings = search_term.filter_postings(postings)
 
     report = StatementReconciliation(
         rt_wrapper,
@@ -450,7 +439,7 @@ def main(arglist: Optional[Sequence[str]]=None,
         args.statement_metadata_key,
         args.id_metadata_key,
     )
-    report.write(postings)
+    report.write(books_load.iter_postings((), args.search_terms))
     if args.output_file is None:
         out_dir_path = config.repository_path() or Path()
         args.output_file = out_dir_path / 'ReconciliationReport_{}_{}.ods'.format(

conservancy_beancount/reports/accrual.py

@@ -624,15 +624,6 @@ class ReportType(enum.Enum):
     OUTGOINGS = OUTGOING
 
 
-def filter_search(postings: Iterable[data.Posting],
-                  search_terms: Iterable[cliutil.SearchTerm],
-) -> Iterable[data.Posting]:
-    accounts = tuple(AccrualAccount.account_names())
-    postings = (post for post in postings if post.account.is_under(*accounts))
-    for query in search_terms:
-        postings = query.filter_postings(postings)
-    return postings
-
 def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
     parser = argparse.ArgumentParser(prog=PROGNAME)
     cliutil.add_version_argument(parser)
@@ -705,37 +696,22 @@ def main(arglist: Optional[Sequence[str]]=None,
         config = configmod.Config()
         config.load_file()
 
-    returncode = 0
+    books_load = books.Loader.dispatch(
-    books_loader = config.books_loader()
+        config.books_loader(),
-    if books_loader is None:
+        None if args.report_type is ReportType.AGING else args.since,
-        entries, load_errors, _ = books.Loader.load_none(config.config_file_path())
+    )
-        returncode = cliutil.ExitCode.NoConfiguration
+    books_load.print_errors(stderr)
-    else:
+    returncode = books_load.returncode()
-        load_since = None if args.report_type == ReportType.AGING else args.since
+
-        entries, load_errors, _ = books_loader.load_all(load_since)
+    filters.remove_opening_balance_txn(books_load.entries)
-        if load_errors:
+    stop_date = args.stop_date or datetime.date(datetime.MAXYEAR, 12, 31)
-            returncode = cliutil.ExitCode.BeancountErrors
+    accrual_accounts = tuple(AccrualAccount.account_names())
-        elif not entries:
+    postings = list(
-            returncode = cliutil.ExitCode.NoDataLoaded
+        post
-    filters.remove_opening_balance_txn(entries)
+        for post in books_load.iter_postings(args.rewrite_rules, args.search_terms)
-    for error in load_errors:
+        if post.meta.date < stop_date
-        bc_printer.print_error(error, file=stderr)
+        and post.account.is_under(*accrual_accounts)
-
-    stop_date = args.stop_date or datetime.date(datetime.MAXYEAR, 12, 31)
-    postings_src: Iterator[data.Posting] = (
-        posting
-        for posting in data.Posting.from_entries(entries)
-        if posting.meta.date < stop_date
     )
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings_src = ruleset.rewrite(postings_src)
-    postings = list(filter_search(postings_src, args.search_terms))
     if not postings:
         logger.warning("no matching entries found to report")
         returncode = returncode or cliutil.ExitCode.NoDataFiltered

conservancy_beancount/reports/balance_sheet.py

@@ -523,34 +523,17 @@ def main(arglist: Optional[Sequence[str]]=None,
     if args.start_date is None:
         args.start_date = cliutil.diff_year(args.stop_date, -1)
 
-    returncode = 0
+    books_load = books.Loader.dispatch(
-    books_loader = config.books_loader()
+        config.books_loader(),
-    if books_loader is None:
+        config.fiscal_year_begin().for_date(args.start_date) - 1,
-        entries, load_errors, options_map = books.Loader.load_none(config.config_file_path())
+        args.stop_date,
-        returncode = cliutil.ExitCode.NoConfiguration
+    )
-    else:
+    books_load.print_errors(stderr)
-        start_fy = config.fiscal_year_begin().for_date(args.start_date) - 1
+    books_load.load_account_metadata()
-        entries, load_errors, options_map = books_loader.load_fy_range(start_fy, args.stop_date)
+    returncode = books_load.returncode()
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    data.Account.load_from_books(entries, options_map)
-    postings = data.Posting.from_entries(entries)
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings = ruleset.rewrite(postings)
 
     balances = core.Balances(
-        postings,
+        books_load.iter_postings(args.rewrite_rules),
         args.start_date,
         args.stop_date,
         'expense-type',

conservancy_beancount/reports/budget.py

@@ -214,37 +214,15 @@ def main(arglist: Optional[Sequence[str]]=None,
         config = configmod.Config()
         config.load_file()
 
-    returncode = 0
+    books_load = books.Loader.dispatch(
-    books_loader = config.books_loader()
+        config.books_loader(), args.start_date, args.stop_date,
-    if books_loader is None:
-        entries, load_errors, options_map = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        entries, load_errors, options_map = books_loader.load_fy_range(
-            args.start_date, args.stop_date,
     )
-        if load_errors:
+    books_load.print_errors(stderr)
-            returncode = cliutil.ExitCode.BeancountErrors
+    books_load.load_account_metadata()
-        elif not entries:
+    returncode = books_load.returncode()
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    data.Account.load_from_books(entries, options_map)
-    postings = data.Posting.from_entries(entries)
-    for search_term in args.search_terms:
-        postings = search_term.filter_postings(postings)
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings = ruleset.rewrite(postings)
 
     balances = Balances(
-        postings,
+        books_load.iter_postings(args.rewrite_rules, args.search_terms),
         args.start_date,
         args.stop_date,
         'expense-type',

conservancy_beancount/reports/fund.py

@@ -383,36 +383,17 @@ def main(arglist: Optional[Sequence[str]]=None,
     if args.start_date is None:
         args.start_date = cliutil.diff_year(args.stop_date, -1)
 
-    returncode = 0
+    books_load = books.Loader.dispatch(
-    books_loader = config.books_loader()
+        config.books_loader(), args.start_date, args.stop_date,
-    if books_loader is None:
+    )
-        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
+    books_load.print_errors(stderr)
-        returncode = cliutil.ExitCode.NoConfiguration
+    returncode = books_load.returncode()
-    else:
+    books_load.load_account_metadata()
-        entries, load_errors, options = books_loader.load_fy_range(args.start_date, args.stop_date)
+    postings = (
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    data.Account.load_from_books(entries, options)
-    postings = iter(
         post
-        for post in data.Posting.from_entries(entries)
+        for post in books_load.iter_postings(args.rewrite_rules, args.search_terms)
         if post.meta.date < args.stop_date
     )
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings = ruleset.rewrite(postings)
-    for search_term in args.search_terms:
-        postings = search_term.filter_postings(postings)
     balances = core.Balances(postings, args.start_date, args.stop_date, 'project')
     funds = sorted(balances.meta_values(), key=lambda s: locale.strxfrm(s.casefold()))
     if not funds:

conservancy_beancount/reports/ledger.py

@@ -827,32 +827,12 @@ def main(arglist: Optional[Sequence[str]]=None,
     elif args.stop_date is None:
         args.stop_date = cliutil.diff_year(args.start_date, 1)
 
-    returncode = 0
+    books_load = books.Loader.dispatch(
-    books_loader = config.books_loader()
+        config.books_loader(), args.start_date, args.stop_date,
-    if books_loader is None:
+    )
-        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
+    returncode = books_load.returncode()
-        returncode = cliutil.ExitCode.NoConfiguration
+    books_load.print_errors(stderr)
-    else:
+    books_load.load_account_metadata()
-        entries, load_errors, options = books_loader.load_fy_range(args.start_date, args.stop_date)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    data.Account.load_from_books(entries, options)
-    postings = data.Posting.from_entries(entries)
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings = ruleset.rewrite(postings)
-    for search_term in args.search_terms:
-        postings = search_term.filter_postings(postings)
 
     rt_wrapper = config.rt_wrapper()
     if rt_wrapper is None:
@@ -889,7 +869,7 @@ def main(arglist: Optional[Sequence[str]]=None,
         logger.error("%s: %r", *error.args)
         return 2
     report.set_common_properties(config.books_repo())
-    report.write(postings)
+    report.write(books_load.iter_postings(args.rewrite_rules, args.search_terms))
     if not any(report.account_groups.values()):
         logger.warning("no matching postings found to report")
         returncode = returncode or cliutil.ExitCode.NoDataFiltered

conservancy_beancount/tools/opening_balances.py

@@ -162,22 +162,12 @@ def main(arglist: Optional[Sequence[str]]=None,
     if isinstance(args.as_of_date, int):
         args.as_of_date = fy.first_date(args.as_of_date)
 
-    returncode = 0
+    books_load = books.Loader.dispatch(config.books_loader(), 0, args.as_of_date)
-    books_loader = config.books_loader()
+    returncode = books_load.returncode()
-    if books_loader is None:
+    books_load.print_errors(stderr)
-        entries, load_errors, _ = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        entries, load_errors, _ = books_loader.load_fy_range(0, args.as_of_date)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
 
     inventories: Mapping[AccountWithFund, Inventory] = collections.defaultdict(Inventory)
-    for post in Posting.from_entries(entries):
+    for post in books_load.iter_postings():
         if post.meta.date >= args.as_of_date:
             continue
         account = post.account

tests/test_reports_accrual.py

@@ -221,27 +221,6 @@ def check_aging_ods(ods_file, date, recv_rows=AGING_AR, pay_rows=AGING_AP):
     check_aging_sheet(sheets[0], recv_rows, date)
     check_aging_sheet(sheets[1], pay_rows, date)
 
-@pytest.mark.parametrize('search_terms,expect_count,check_func', [
-    ([], ACCRUALS_COUNT, lambda post: post.account.is_under(
-        'Assets:Receivable:', 'Liabilities:Payable:',
-    )),
-    ([('rt-id', '^rt:505$')], 2, lambda post: post.meta['entity'] == 'DonorA'),
-    ([('invoice', r'^rt:\D+515/')], 1, lambda post: post.meta['entity'] == 'MatchingProgram'),
-    ([('entity', '^Lawyer$')], 3, lambda post: post.meta['rt-id'] == 'rt:510'),
-    ([('entity', '^Lawyer$'), ('contract', '^rt:510/')], 2,
-     lambda post: post.meta['invoice'].startswith('rt:510/')),
-    ([('rt-id', '^rt:510$'), ('approval', '.')], 0, lambda post: False),
-])
-def test_filter_search(accrual_postings, search_terms, expect_count, check_func):
-    search_terms = [cliutil.SearchTerm._make(query) for query in search_terms]
-    actual = list(accrual.filter_search(accrual_postings, search_terms))
-    if expect_count < ACCRUALS_COUNT:
-        assert ACCRUALS_COUNT > len(actual) >= expect_count
-    else:
-        assert len(actual) == ACCRUALS_COUNT
-    for post in actual:
-        assert check_func(post)
-
 @pytest.mark.parametrize('acct_name,invoice,day', testutil.combine_values(
     INVOICE_ACCOUNTS,
     ['FIXME', '', None, *testutil.NON_STRING_METADATA_VALUES],

tests/testutil.py

@@ -238,7 +238,7 @@ class TestBooksLoader(books.Loader):
         self.source = source
 
     def load_all(self, from_year=None):
-        return bc_loader.load_file(self.source)
+        return books.LoadResult._make(bc_loader.load_file(self.source))
 
     def load_fy_range(self, from_fy, to_fy=None):
         return self.load_all()