books: All tools use books.Loader.dispatch() and LoadResult.

This is significant code deduplication, as seen in the diffstat.
2021-02-19 18:52:44 -05:00 · 2021-02-19 18:52:44 -05:00 · ee2bd6c096
commit ee2bd6c096
parent 3721226d17
10 changed files with 67 additions and 217 deletions
--- a/conservancy_beancount/reconcile/paypal.py
+++ b/conservancy_beancount/reconcile/paypal.py
@ -371,23 +371,17 @@ def main(arglist: Optional[Sequence[str]]=None,
            for post in rec.statement_posts
        ) + datetime.timedelta(days=1)

-    returncode = os.EX_OK
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
    date_fuzz = datetime.timedelta(days=args.date_fuzz)
-        entries, load_errors, options = books_loader.load_fy_range(
-            args.start_date - date_fuzz, args.stop_date + date_fuzz,
+    books_load = books.Loader.dispatch(
+        config.books_loader(),
+        args.start_date - date_fuzz,
+        args.stop_date + date_fuzz,
    )
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
+    books_load.print_errors(stderr)
+    returncode = books_load.returncode()

    date_range = DateRange(args.start_date, args.stop_date)
-    for bean_post in data.Posting.from_entries(entries):
+    for bean_post in books_load.iter_postings():
        if bean_post.account != args.account:
            continue
        paypal_post = PayPalPosting.from_books(bean_post)
--- a/conservancy_beancount/reconcile/statement.py
+++ b/conservancy_beancount/reconcile/statement.py
@ -406,19 +406,12 @@ def main(arglist: Optional[Sequence[str]]=None,
    rec_range = DateRange(args.start_date, args.stop_date)
    post_range = DateRange(args.stop_date, args.stop_date + days_diff)

-    returncode = os.EX_OK
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        entries, load_errors, options = books_loader.load_fy_range(pre_range.start, post_range.stop)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
+    books_load = books.Loader.dispatch(
+        config.books_loader(), pre_range.start, post_range.stop,
+    )
+    books_load.load_account_metadata()
+    returncode = books_load.returncode()

-    data.Account.load_from_books(entries, options)
    real_accounts: Set[data.Account] = set()
    for account_spec in args.accounts:
        new_accounts = frozenset(
@ -432,14 +425,10 @@ def main(arglist: Optional[Sequence[str]]=None,
            logger.critical("account %r did not match any open accounts", account_spec)
            return 2

-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
+    books_load.print_errors(stderr)
    rt_wrapper = config.rt_wrapper()
    if rt_wrapper is None:
        logger.warning("could not initialize RT client; spreadsheet links will be broken")
-    postings = data.Posting.from_entries(entries)
-    for search_term in args.search_terms:
-        postings = search_term.filter_postings(postings)

    report = StatementReconciliation(
        rt_wrapper,
@ -450,7 +439,7 @@ def main(arglist: Optional[Sequence[str]]=None,
        args.statement_metadata_key,
        args.id_metadata_key,
    )
-    report.write(postings)
+    report.write(books_load.iter_postings((), args.search_terms))
    if args.output_file is None:
        out_dir_path = config.repository_path() or Path()
        args.output_file = out_dir_path / 'ReconciliationReport_{}_{}.ods'.format(
--- a/conservancy_beancount/reports/accrual.py
+++ b/conservancy_beancount/reports/accrual.py
@ -624,15 +624,6 @@ class ReportType(enum.Enum):
    OUTGOINGS = OUTGOING


-def filter_search(postings: Iterable[data.Posting],
-                  search_terms: Iterable[cliutil.SearchTerm],
-) -> Iterable[data.Posting]:
-    accounts = tuple(AccrualAccount.account_names())
-    postings = (post for post in postings if post.account.is_under(*accounts))
-    for query in search_terms:
-        postings = query.filter_postings(postings)
-    return postings
-
 def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(prog=PROGNAME)
    cliutil.add_version_argument(parser)
@ -705,37 +696,22 @@ def main(arglist: Optional[Sequence[str]]=None,
        config = configmod.Config()
        config.load_file()

-    returncode = 0
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, _ = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        load_since = None if args.report_type == ReportType.AGING else args.since
-        entries, load_errors, _ = books_loader.load_all(load_since)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    filters.remove_opening_balance_txn(entries)
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    stop_date = args.stop_date or datetime.date(datetime.MAXYEAR, 12, 31)
-    postings_src: Iterator[data.Posting] = (
-        posting
-        for posting in data.Posting.from_entries(entries)
-        if posting.meta.date < stop_date
+    books_load = books.Loader.dispatch(
+        config.books_loader(),
+        None if args.report_type is ReportType.AGING else args.since,
+    )
+    books_load.print_errors(stderr)
+    returncode = books_load.returncode()
+
+    filters.remove_opening_balance_txn(books_load.entries)
+    stop_date = args.stop_date or datetime.date(datetime.MAXYEAR, 12, 31)
+    accrual_accounts = tuple(AccrualAccount.account_names())
+    postings = list(
+        post
+        for post in books_load.iter_postings(args.rewrite_rules, args.search_terms)
+        if post.meta.date < stop_date
+        and post.account.is_under(*accrual_accounts)
    )
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings_src = ruleset.rewrite(postings_src)
-    postings = list(filter_search(postings_src, args.search_terms))
    if not postings:
        logger.warning("no matching entries found to report")
        returncode = returncode or cliutil.ExitCode.NoDataFiltered
--- a/conservancy_beancount/reports/balance_sheet.py
+++ b/conservancy_beancount/reports/balance_sheet.py
@ -523,34 +523,17 @@ def main(arglist: Optional[Sequence[str]]=None,
    if args.start_date is None:
        args.start_date = cliutil.diff_year(args.stop_date, -1)

-    returncode = 0
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, options_map = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        start_fy = config.fiscal_year_begin().for_date(args.start_date) - 1
-        entries, load_errors, options_map = books_loader.load_fy_range(start_fy, args.stop_date)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    data.Account.load_from_books(entries, options_map)
-    postings = data.Posting.from_entries(entries)
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings = ruleset.rewrite(postings)
+    books_load = books.Loader.dispatch(
+        config.books_loader(),
+        config.fiscal_year_begin().for_date(args.start_date) - 1,
+        args.stop_date,
+    )
+    books_load.print_errors(stderr)
+    books_load.load_account_metadata()
+    returncode = books_load.returncode()

    balances = core.Balances(
-        postings,
+        books_load.iter_postings(args.rewrite_rules),
        args.start_date,
        args.stop_date,
        'expense-type',
--- a/conservancy_beancount/reports/budget.py
+++ b/conservancy_beancount/reports/budget.py
@ -214,37 +214,15 @@ def main(arglist: Optional[Sequence[str]]=None,
        config = configmod.Config()
        config.load_file()

-    returncode = 0
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, options_map = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        entries, load_errors, options_map = books_loader.load_fy_range(
-            args.start_date, args.stop_date,
+    books_load = books.Loader.dispatch(
+        config.books_loader(), args.start_date, args.stop_date,
    )
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    data.Account.load_from_books(entries, options_map)
-    postings = data.Posting.from_entries(entries)
-    for search_term in args.search_terms:
-        postings = search_term.filter_postings(postings)
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings = ruleset.rewrite(postings)
+    books_load.print_errors(stderr)
+    books_load.load_account_metadata()
+    returncode = books_load.returncode()

    balances = Balances(
-        postings,
+        books_load.iter_postings(args.rewrite_rules, args.search_terms),
        args.start_date,
        args.stop_date,
        'expense-type',
--- a/conservancy_beancount/reports/fund.py
+++ b/conservancy_beancount/reports/fund.py
@ -383,36 +383,17 @@ def main(arglist: Optional[Sequence[str]]=None,
    if args.start_date is None:
        args.start_date = cliutil.diff_year(args.stop_date, -1)

-    returncode = 0
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        entries, load_errors, options = books_loader.load_fy_range(args.start_date, args.stop_date)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    data.Account.load_from_books(entries, options)
-    postings = iter(
+    books_load = books.Loader.dispatch(
+        config.books_loader(), args.start_date, args.stop_date,
+    )
+    books_load.print_errors(stderr)
+    returncode = books_load.returncode()
+    books_load.load_account_metadata()
+    postings = (
        post
-        for post in data.Posting.from_entries(entries)
+        for post in books_load.iter_postings(args.rewrite_rules, args.search_terms)
        if post.meta.date < args.stop_date
    )
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings = ruleset.rewrite(postings)
-    for search_term in args.search_terms:
-        postings = search_term.filter_postings(postings)
    balances = core.Balances(postings, args.start_date, args.stop_date, 'project')
    funds = sorted(balances.meta_values(), key=lambda s: locale.strxfrm(s.casefold()))
    if not funds:
--- a/conservancy_beancount/reports/ledger.py
+++ b/conservancy_beancount/reports/ledger.py
@ -827,32 +827,12 @@ def main(arglist: Optional[Sequence[str]]=None,
    elif args.stop_date is None:
        args.stop_date = cliutil.diff_year(args.start_date, 1)

-    returncode = 0
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        entries, load_errors, options = books_loader.load_fy_range(args.start_date, args.stop_date)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
-
-    data.Account.load_from_books(entries, options)
-    postings = data.Posting.from_entries(entries)
-    for rewrite_path in args.rewrite_rules:
-        try:
-            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
-        except ValueError as error:
-            logger.critical("failed loading rewrite rules from %s: %s",
-                            rewrite_path, error.args[0])
-            return cliutil.ExitCode.RewriteRulesError
-        postings = ruleset.rewrite(postings)
-    for search_term in args.search_terms:
-        postings = search_term.filter_postings(postings)
+    books_load = books.Loader.dispatch(
+        config.books_loader(), args.start_date, args.stop_date,
+    )
+    returncode = books_load.returncode()
+    books_load.print_errors(stderr)
+    books_load.load_account_metadata()

    rt_wrapper = config.rt_wrapper()
    if rt_wrapper is None:
@ -889,7 +869,7 @@ def main(arglist: Optional[Sequence[str]]=None,
        logger.error("%s: %r", *error.args)
        return 2
    report.set_common_properties(config.books_repo())
-    report.write(postings)
+    report.write(books_load.iter_postings(args.rewrite_rules, args.search_terms))
    if not any(report.account_groups.values()):
        logger.warning("no matching postings found to report")
        returncode = returncode or cliutil.ExitCode.NoDataFiltered
--- a/conservancy_beancount/tools/opening_balances.py
+++ b/conservancy_beancount/tools/opening_balances.py
@ -162,22 +162,12 @@ def main(arglist: Optional[Sequence[str]]=None,
    if isinstance(args.as_of_date, int):
        args.as_of_date = fy.first_date(args.as_of_date)

-    returncode = 0
-    books_loader = config.books_loader()
-    if books_loader is None:
-        entries, load_errors, _ = books.Loader.load_none(config.config_file_path())
-        returncode = cliutil.ExitCode.NoConfiguration
-    else:
-        entries, load_errors, _ = books_loader.load_fy_range(0, args.as_of_date)
-        if load_errors:
-            returncode = cliutil.ExitCode.BeancountErrors
-        elif not entries:
-            returncode = cliutil.ExitCode.NoDataLoaded
-    for error in load_errors:
-        bc_printer.print_error(error, file=stderr)
+    books_load = books.Loader.dispatch(config.books_loader(), 0, args.as_of_date)
+    returncode = books_load.returncode()
+    books_load.print_errors(stderr)

    inventories: Mapping[AccountWithFund, Inventory] = collections.defaultdict(Inventory)
-    for post in Posting.from_entries(entries):
+    for post in books_load.iter_postings():
        if post.meta.date >= args.as_of_date:
            continue
        account = post.account
--- a/tests/test_reports_accrual.py
+++ b/tests/test_reports_accrual.py
@ -221,27 +221,6 @@ def check_aging_ods(ods_file, date, recv_rows=AGING_AR, pay_rows=AGING_AP):
    check_aging_sheet(sheets[0], recv_rows, date)
    check_aging_sheet(sheets[1], pay_rows, date)

-@pytest.mark.parametrize('search_terms,expect_count,check_func', [
-    ([], ACCRUALS_COUNT, lambda post: post.account.is_under(
-        'Assets:Receivable:', 'Liabilities:Payable:',
-    )),
-    ([('rt-id', '^rt:505$')], 2, lambda post: post.meta['entity'] == 'DonorA'),
-    ([('invoice', r'^rt:\D+515/')], 1, lambda post: post.meta['entity'] == 'MatchingProgram'),
-    ([('entity', '^Lawyer$')], 3, lambda post: post.meta['rt-id'] == 'rt:510'),
-    ([('entity', '^Lawyer$'), ('contract', '^rt:510/')], 2,
-     lambda post: post.meta['invoice'].startswith('rt:510/')),
-    ([('rt-id', '^rt:510$'), ('approval', '.')], 0, lambda post: False),
-])
-def test_filter_search(accrual_postings, search_terms, expect_count, check_func):
-    search_terms = [cliutil.SearchTerm._make(query) for query in search_terms]
-    actual = list(accrual.filter_search(accrual_postings, search_terms))
-    if expect_count < ACCRUALS_COUNT:
-        assert ACCRUALS_COUNT > len(actual) >= expect_count
-    else:
-        assert len(actual) == ACCRUALS_COUNT
-    for post in actual:
-        assert check_func(post)
-
@pytest.mark.parametrize('acct_name,invoice,day', testutil.combine_values(
    INVOICE_ACCOUNTS,
    ['FIXME', '', None, *testutil.NON_STRING_METADATA_VALUES],
--- a/tests/testutil.py
+++ b/tests/testutil.py
@ -238,7 +238,7 @@ class TestBooksLoader(books.Loader):
        self.source = source

    def load_all(self, from_year=None):
-        return bc_loader.load_file(self.source)
+        return books.LoadResult._make(bc_loader.load_file(self.source))

    def load_fy_range(self, from_fy, to_fy=None):
        return self.load_all()