From ee2bd6c096935fdb6dde041a0e400366772375da Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Fri, 19 Feb 2021 18:52:44 -0500 Subject: [PATCH] books: All tools use books.Loader.dispatch() and LoadResult. This is significant code deduplication, as seen in the diffstat. --- conservancy_beancount/reconcile/paypal.py | 24 ++++----- conservancy_beancount/reconcile/statement.py | 25 +++------ conservancy_beancount/reports/accrual.py | 54 ++++++------------- .../reports/balance_sheet.py | 35 ++++-------- conservancy_beancount/reports/budget.py | 36 +++---------- conservancy_beancount/reports/fund.py | 35 +++--------- conservancy_beancount/reports/ledger.py | 34 +++--------- .../tools/opening_balances.py | 18 ++----- tests/test_reports_accrual.py | 21 -------- tests/testutil.py | 2 +- 10 files changed, 67 insertions(+), 217 deletions(-) diff --git a/conservancy_beancount/reconcile/paypal.py b/conservancy_beancount/reconcile/paypal.py index 9bff2ca..69d9c2f 100644 --- a/conservancy_beancount/reconcile/paypal.py +++ b/conservancy_beancount/reconcile/paypal.py @@ -371,23 +371,17 @@ def main(arglist: Optional[Sequence[str]]=None, for post in rec.statement_posts ) + datetime.timedelta(days=1) - returncode = os.EX_OK - books_loader = config.books_loader() - if books_loader is None: - entries, load_errors, options = books.Loader.load_none(config.config_file_path()) - returncode = cliutil.ExitCode.NoConfiguration - else: - date_fuzz = datetime.timedelta(days=args.date_fuzz) - entries, load_errors, options = books_loader.load_fy_range( - args.start_date - date_fuzz, args.stop_date + date_fuzz, - ) - if load_errors: - returncode = cliutil.ExitCode.BeancountErrors - for error in load_errors: - bc_printer.print_error(error, file=stderr) + date_fuzz = datetime.timedelta(days=args.date_fuzz) + books_load = books.Loader.dispatch( + config.books_loader(), + args.start_date - date_fuzz, + args.stop_date + date_fuzz, + ) + books_load.print_errors(stderr) + returncode = books_load.returncode() date_range = DateRange(args.start_date, args.stop_date) - for bean_post in data.Posting.from_entries(entries): + for bean_post in books_load.iter_postings(): if bean_post.account != args.account: continue paypal_post = PayPalPosting.from_books(bean_post) diff --git a/conservancy_beancount/reconcile/statement.py b/conservancy_beancount/reconcile/statement.py index 4de7c5f..4c08643 100644 --- a/conservancy_beancount/reconcile/statement.py +++ b/conservancy_beancount/reconcile/statement.py @@ -406,19 +406,12 @@ def main(arglist: Optional[Sequence[str]]=None, rec_range = DateRange(args.start_date, args.stop_date) post_range = DateRange(args.stop_date, args.stop_date + days_diff) - returncode = os.EX_OK - books_loader = config.books_loader() - if books_loader is None: - entries, load_errors, options = books.Loader.load_none(config.config_file_path()) - returncode = cliutil.ExitCode.NoConfiguration - else: - entries, load_errors, options = books_loader.load_fy_range(pre_range.start, post_range.stop) - if load_errors: - returncode = cliutil.ExitCode.BeancountErrors - elif not entries: - returncode = cliutil.ExitCode.NoDataLoaded + books_load = books.Loader.dispatch( + config.books_loader(), pre_range.start, post_range.stop, + ) + books_load.load_account_metadata() + returncode = books_load.returncode() - data.Account.load_from_books(entries, options) real_accounts: Set[data.Account] = set() for account_spec in args.accounts: new_accounts = frozenset( @@ -432,14 +425,10 @@ def main(arglist: Optional[Sequence[str]]=None, logger.critical("account %r did not match any open accounts", account_spec) return 2 - for error in load_errors: - bc_printer.print_error(error, file=stderr) + books_load.print_errors(stderr) rt_wrapper = config.rt_wrapper() if rt_wrapper is None: logger.warning("could not initialize RT client; spreadsheet links will be broken") - postings = data.Posting.from_entries(entries) - for search_term in args.search_terms: - postings = search_term.filter_postings(postings) report = StatementReconciliation( rt_wrapper, @@ -450,7 +439,7 @@ def main(arglist: Optional[Sequence[str]]=None, args.statement_metadata_key, args.id_metadata_key, ) - report.write(postings) + report.write(books_load.iter_postings((), args.search_terms)) if args.output_file is None: out_dir_path = config.repository_path() or Path() args.output_file = out_dir_path / 'ReconciliationReport_{}_{}.ods'.format( diff --git a/conservancy_beancount/reports/accrual.py b/conservancy_beancount/reports/accrual.py index 9c960fc..209b8a7 100644 --- a/conservancy_beancount/reports/accrual.py +++ b/conservancy_beancount/reports/accrual.py @@ -624,15 +624,6 @@ class ReportType(enum.Enum): OUTGOINGS = OUTGOING -def filter_search(postings: Iterable[data.Posting], - search_terms: Iterable[cliutil.SearchTerm], -) -> Iterable[data.Posting]: - accounts = tuple(AccrualAccount.account_names()) - postings = (post for post in postings if post.account.is_under(*accounts)) - for query in search_terms: - postings = query.filter_postings(postings) - return postings - def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace: parser = argparse.ArgumentParser(prog=PROGNAME) cliutil.add_version_argument(parser) @@ -705,37 +696,22 @@ def main(arglist: Optional[Sequence[str]]=None, config = configmod.Config() config.load_file() - returncode = 0 - books_loader = config.books_loader() - if books_loader is None: - entries, load_errors, _ = books.Loader.load_none(config.config_file_path()) - returncode = cliutil.ExitCode.NoConfiguration - else: - load_since = None if args.report_type == ReportType.AGING else args.since - entries, load_errors, _ = books_loader.load_all(load_since) - if load_errors: - returncode = cliutil.ExitCode.BeancountErrors - elif not entries: - returncode = cliutil.ExitCode.NoDataLoaded - filters.remove_opening_balance_txn(entries) - for error in load_errors: - bc_printer.print_error(error, file=stderr) - - stop_date = args.stop_date or datetime.date(datetime.MAXYEAR, 12, 31) - postings_src: Iterator[data.Posting] = ( - posting - for posting in data.Posting.from_entries(entries) - if posting.meta.date < stop_date + books_load = books.Loader.dispatch( + config.books_loader(), + None if args.report_type is ReportType.AGING else args.since, + ) + books_load.print_errors(stderr) + returncode = books_load.returncode() + + filters.remove_opening_balance_txn(books_load.entries) + stop_date = args.stop_date or datetime.date(datetime.MAXYEAR, 12, 31) + accrual_accounts = tuple(AccrualAccount.account_names()) + postings = list( + post + for post in books_load.iter_postings(args.rewrite_rules, args.search_terms) + if post.meta.date < stop_date + and post.account.is_under(*accrual_accounts) ) - for rewrite_path in args.rewrite_rules: - try: - ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path) - except ValueError as error: - logger.critical("failed loading rewrite rules from %s: %s", - rewrite_path, error.args[0]) - return cliutil.ExitCode.RewriteRulesError - postings_src = ruleset.rewrite(postings_src) - postings = list(filter_search(postings_src, args.search_terms)) if not postings: logger.warning("no matching entries found to report") returncode = returncode or cliutil.ExitCode.NoDataFiltered diff --git a/conservancy_beancount/reports/balance_sheet.py b/conservancy_beancount/reports/balance_sheet.py index 25f1454..3125b4e 100644 --- a/conservancy_beancount/reports/balance_sheet.py +++ b/conservancy_beancount/reports/balance_sheet.py @@ -523,34 +523,17 @@ def main(arglist: Optional[Sequence[str]]=None, if args.start_date is None: args.start_date = cliutil.diff_year(args.stop_date, -1) - returncode = 0 - books_loader = config.books_loader() - if books_loader is None: - entries, load_errors, options_map = books.Loader.load_none(config.config_file_path()) - returncode = cliutil.ExitCode.NoConfiguration - else: - start_fy = config.fiscal_year_begin().for_date(args.start_date) - 1 - entries, load_errors, options_map = books_loader.load_fy_range(start_fy, args.stop_date) - if load_errors: - returncode = cliutil.ExitCode.BeancountErrors - elif not entries: - returncode = cliutil.ExitCode.NoDataLoaded - for error in load_errors: - bc_printer.print_error(error, file=stderr) - - data.Account.load_from_books(entries, options_map) - postings = data.Posting.from_entries(entries) - for rewrite_path in args.rewrite_rules: - try: - ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path) - except ValueError as error: - logger.critical("failed loading rewrite rules from %s: %s", - rewrite_path, error.args[0]) - return cliutil.ExitCode.RewriteRulesError - postings = ruleset.rewrite(postings) + books_load = books.Loader.dispatch( + config.books_loader(), + config.fiscal_year_begin().for_date(args.start_date) - 1, + args.stop_date, + ) + books_load.print_errors(stderr) + books_load.load_account_metadata() + returncode = books_load.returncode() balances = core.Balances( - postings, + books_load.iter_postings(args.rewrite_rules), args.start_date, args.stop_date, 'expense-type', diff --git a/conservancy_beancount/reports/budget.py b/conservancy_beancount/reports/budget.py index d6ad7db..e42f22f 100644 --- a/conservancy_beancount/reports/budget.py +++ b/conservancy_beancount/reports/budget.py @@ -214,37 +214,15 @@ def main(arglist: Optional[Sequence[str]]=None, config = configmod.Config() config.load_file() - returncode = 0 - books_loader = config.books_loader() - if books_loader is None: - entries, load_errors, options_map = books.Loader.load_none(config.config_file_path()) - returncode = cliutil.ExitCode.NoConfiguration - else: - entries, load_errors, options_map = books_loader.load_fy_range( - args.start_date, args.stop_date, - ) - if load_errors: - returncode = cliutil.ExitCode.BeancountErrors - elif not entries: - returncode = cliutil.ExitCode.NoDataLoaded - for error in load_errors: - bc_printer.print_error(error, file=stderr) - - data.Account.load_from_books(entries, options_map) - postings = data.Posting.from_entries(entries) - for search_term in args.search_terms: - postings = search_term.filter_postings(postings) - for rewrite_path in args.rewrite_rules: - try: - ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path) - except ValueError as error: - logger.critical("failed loading rewrite rules from %s: %s", - rewrite_path, error.args[0]) - return cliutil.ExitCode.RewriteRulesError - postings = ruleset.rewrite(postings) + books_load = books.Loader.dispatch( + config.books_loader(), args.start_date, args.stop_date, + ) + books_load.print_errors(stderr) + books_load.load_account_metadata() + returncode = books_load.returncode() balances = Balances( - postings, + books_load.iter_postings(args.rewrite_rules, args.search_terms), args.start_date, args.stop_date, 'expense-type', diff --git a/conservancy_beancount/reports/fund.py b/conservancy_beancount/reports/fund.py index 637ac34..de76b9a 100644 --- a/conservancy_beancount/reports/fund.py +++ b/conservancy_beancount/reports/fund.py @@ -383,36 +383,17 @@ def main(arglist: Optional[Sequence[str]]=None, if args.start_date is None: args.start_date = cliutil.diff_year(args.stop_date, -1) - returncode = 0 - books_loader = config.books_loader() - if books_loader is None: - entries, load_errors, options = books.Loader.load_none(config.config_file_path()) - returncode = cliutil.ExitCode.NoConfiguration - else: - entries, load_errors, options = books_loader.load_fy_range(args.start_date, args.stop_date) - if load_errors: - returncode = cliutil.ExitCode.BeancountErrors - elif not entries: - returncode = cliutil.ExitCode.NoDataLoaded - for error in load_errors: - bc_printer.print_error(error, file=stderr) - - data.Account.load_from_books(entries, options) - postings = iter( + books_load = books.Loader.dispatch( + config.books_loader(), args.start_date, args.stop_date, + ) + books_load.print_errors(stderr) + returncode = books_load.returncode() + books_load.load_account_metadata() + postings = ( post - for post in data.Posting.from_entries(entries) + for post in books_load.iter_postings(args.rewrite_rules, args.search_terms) if post.meta.date < args.stop_date ) - for rewrite_path in args.rewrite_rules: - try: - ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path) - except ValueError as error: - logger.critical("failed loading rewrite rules from %s: %s", - rewrite_path, error.args[0]) - return cliutil.ExitCode.RewriteRulesError - postings = ruleset.rewrite(postings) - for search_term in args.search_terms: - postings = search_term.filter_postings(postings) balances = core.Balances(postings, args.start_date, args.stop_date, 'project') funds = sorted(balances.meta_values(), key=lambda s: locale.strxfrm(s.casefold())) if not funds: diff --git a/conservancy_beancount/reports/ledger.py b/conservancy_beancount/reports/ledger.py index 1001851..9af320c 100644 --- a/conservancy_beancount/reports/ledger.py +++ b/conservancy_beancount/reports/ledger.py @@ -827,32 +827,12 @@ def main(arglist: Optional[Sequence[str]]=None, elif args.stop_date is None: args.stop_date = cliutil.diff_year(args.start_date, 1) - returncode = 0 - books_loader = config.books_loader() - if books_loader is None: - entries, load_errors, options = books.Loader.load_none(config.config_file_path()) - returncode = cliutil.ExitCode.NoConfiguration - else: - entries, load_errors, options = books_loader.load_fy_range(args.start_date, args.stop_date) - if load_errors: - returncode = cliutil.ExitCode.BeancountErrors - elif not entries: - returncode = cliutil.ExitCode.NoDataLoaded - for error in load_errors: - bc_printer.print_error(error, file=stderr) - - data.Account.load_from_books(entries, options) - postings = data.Posting.from_entries(entries) - for rewrite_path in args.rewrite_rules: - try: - ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path) - except ValueError as error: - logger.critical("failed loading rewrite rules from %s: %s", - rewrite_path, error.args[0]) - return cliutil.ExitCode.RewriteRulesError - postings = ruleset.rewrite(postings) - for search_term in args.search_terms: - postings = search_term.filter_postings(postings) + books_load = books.Loader.dispatch( + config.books_loader(), args.start_date, args.stop_date, + ) + returncode = books_load.returncode() + books_load.print_errors(stderr) + books_load.load_account_metadata() rt_wrapper = config.rt_wrapper() if rt_wrapper is None: @@ -889,7 +869,7 @@ def main(arglist: Optional[Sequence[str]]=None, logger.error("%s: %r", *error.args) return 2 report.set_common_properties(config.books_repo()) - report.write(postings) + report.write(books_load.iter_postings(args.rewrite_rules, args.search_terms)) if not any(report.account_groups.values()): logger.warning("no matching postings found to report") returncode = returncode or cliutil.ExitCode.NoDataFiltered diff --git a/conservancy_beancount/tools/opening_balances.py b/conservancy_beancount/tools/opening_balances.py index e11301c..db0a08e 100644 --- a/conservancy_beancount/tools/opening_balances.py +++ b/conservancy_beancount/tools/opening_balances.py @@ -162,22 +162,12 @@ def main(arglist: Optional[Sequence[str]]=None, if isinstance(args.as_of_date, int): args.as_of_date = fy.first_date(args.as_of_date) - returncode = 0 - books_loader = config.books_loader() - if books_loader is None: - entries, load_errors, _ = books.Loader.load_none(config.config_file_path()) - returncode = cliutil.ExitCode.NoConfiguration - else: - entries, load_errors, _ = books_loader.load_fy_range(0, args.as_of_date) - if load_errors: - returncode = cliutil.ExitCode.BeancountErrors - elif not entries: - returncode = cliutil.ExitCode.NoDataLoaded - for error in load_errors: - bc_printer.print_error(error, file=stderr) + books_load = books.Loader.dispatch(config.books_loader(), 0, args.as_of_date) + returncode = books_load.returncode() + books_load.print_errors(stderr) inventories: Mapping[AccountWithFund, Inventory] = collections.defaultdict(Inventory) - for post in Posting.from_entries(entries): + for post in books_load.iter_postings(): if post.meta.date >= args.as_of_date: continue account = post.account diff --git a/tests/test_reports_accrual.py b/tests/test_reports_accrual.py index 2f48915..147ae46 100644 --- a/tests/test_reports_accrual.py +++ b/tests/test_reports_accrual.py @@ -221,27 +221,6 @@ def check_aging_ods(ods_file, date, recv_rows=AGING_AR, pay_rows=AGING_AP): check_aging_sheet(sheets[0], recv_rows, date) check_aging_sheet(sheets[1], pay_rows, date) -@pytest.mark.parametrize('search_terms,expect_count,check_func', [ - ([], ACCRUALS_COUNT, lambda post: post.account.is_under( - 'Assets:Receivable:', 'Liabilities:Payable:', - )), - ([('rt-id', '^rt:505$')], 2, lambda post: post.meta['entity'] == 'DonorA'), - ([('invoice', r'^rt:\D+515/')], 1, lambda post: post.meta['entity'] == 'MatchingProgram'), - ([('entity', '^Lawyer$')], 3, lambda post: post.meta['rt-id'] == 'rt:510'), - ([('entity', '^Lawyer$'), ('contract', '^rt:510/')], 2, - lambda post: post.meta['invoice'].startswith('rt:510/')), - ([('rt-id', '^rt:510$'), ('approval', '.')], 0, lambda post: False), -]) -def test_filter_search(accrual_postings, search_terms, expect_count, check_func): - search_terms = [cliutil.SearchTerm._make(query) for query in search_terms] - actual = list(accrual.filter_search(accrual_postings, search_terms)) - if expect_count < ACCRUALS_COUNT: - assert ACCRUALS_COUNT > len(actual) >= expect_count - else: - assert len(actual) == ACCRUALS_COUNT - for post in actual: - assert check_func(post) - @pytest.mark.parametrize('acct_name,invoice,day', testutil.combine_values( INVOICE_ACCOUNTS, ['FIXME', '', None, *testutil.NON_STRING_METADATA_VALUES], diff --git a/tests/testutil.py b/tests/testutil.py index a460ad6..11a9439 100644 --- a/tests/testutil.py +++ b/tests/testutil.py @@ -238,7 +238,7 @@ class TestBooksLoader(books.Loader): self.source = source def load_all(self, from_year=None): - return bc_loader.load_file(self.source) + return books.LoadResult._make(bc_loader.load_file(self.source)) def load_fy_range(self, from_fy, to_fy=None): return self.load_all()