From 887102ea92084f60369c0bbc643e9470a22554d1 Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Sun, 21 Jun 2020 15:27:43 -0400 Subject: [PATCH] fund: New report. --- conservancy_beancount/reports/fund.py | 359 ++++++++++++++++++++++++++ setup.py | 3 +- tests/books/fund.beancount | 71 +++++ tests/test_reports_fund.py | 227 ++++++++++++++++ 4 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 conservancy_beancount/reports/fund.py create mode 100644 tests/books/fund.beancount create mode 100644 tests/test_reports_fund.py diff --git a/conservancy_beancount/reports/fund.py b/conservancy_beancount/reports/fund.py new file mode 100644 index 0000000..f55e2ab --- /dev/null +++ b/conservancy_beancount/reports/fund.py @@ -0,0 +1,359 @@ +"""funds.py - Funds report from Beancount + +This tool produces a report of account balances for restricted funds, +including income, expenses, and realized gain/loss over a given time period. +Restricted funds are designated with the "project" metadata. + +Specify the date range you want to report with the ``--begin`` and ``--end`` +options. + +Select the accounts you want to report with the ``--account`` option. You can +specify this option multiple times. The report will include at least one sheet +for each account you specify. Subaccounts will be reported on that sheet as +well. + +Select the postings you want to report by passing metadata search terms in +``name=value`` format. + +Run ``ledger-report --help`` for abbreviations and other options. + +Examples +-------- + +Generate a report of all restricted funds in ODS format over the last year:: + + fund-report + +Query a specific restricted fund and get a quick balance on the terminal:: + + fund-report +""" +# Copyright © 2020 Brett Smith +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import argparse +import collections +import datetime +import enum +import locale +import logging +import sys + +from typing import ( + Dict, + Iterable, + Iterator, + Mapping, + Optional, + Sequence, + TextIO, + Tuple, +) +from ..beancount_types import ( + MetaValue, +) + +from pathlib import Path + +import odf.table # type:ignore[import] + +from beancount.parser import printer as bc_printer + +from . import core +from .. import books +from .. import cliutil +from .. import config as configmod +from .. import data + +AccountsMap = Mapping[data.Account, core.PeriodPostings] +FundPosts = Tuple[MetaValue, AccountsMap] + +BALANCE_ACCOUNTS = ['Equity', 'Income', 'Expenses'] +PROGNAME = 'fund-report' +UNRESTRICTED_FUND = 'Conservancy' +logger = logging.getLogger('conservancy_beancount.reports.fund') + +class ODSReport(core.BaseODS[FundPosts, None]): + def __init__(self, start_date: datetime.date, stop_date: datetime.date) -> None: + super().__init__() + self.start_date = start_date + self.stop_date = stop_date + self.unrestricted: AccountsMap = {} + + def section_key(self, row: FundPosts) -> None: + return None + + def start_spreadsheet(self) -> None: + self.use_sheet("Fund Report") + for width in [2.5, 1.5, 1.2, 1.2, 1.2, 1.5]: + col_style = self.column_style(width) + self.sheet.addElement(odf.table.TableColumn(stylename=col_style)) + center_bold = self.merge_styles(self.style_centertext, self.style_bold) + self.add_row( + self.string_cell( + "Fund", stylename=self.merge_styles(self.style_endtext, self.style_bold), + ), + self.multiline_cell(["Balance as of", self.start_date.isoformat()], + stylename=center_bold), + self.string_cell("Income", stylename=center_bold), + self.string_cell("Expenses", stylename=center_bold), + self.multiline_cell(["Realized", "Gain/Loss"], stylename=center_bold), + self.multiline_cell(["Balance as of", self.stop_date.isoformat()], + stylename=center_bold), + ) + self.lock_first_row() + self.add_row() + self.add_row(self.string_cell( + f"Fund Report From {self.start_date.isoformat()} To {self.stop_date.isoformat()}", + stylename=center_bold, + numbercolumnsspanned=6, + )) + self.add_row() + + def _row_balances(self, accounts_map: AccountsMap) -> Iterable[core.Balance]: + acct_order = ['Income', 'Expenses', 'Equity'] + key_order = [core.OPENING_BALANCE_NAME, *acct_order, core.ENDING_BALANCE_NAME] + balances: Dict[str, core.Balance] = {key: core.MutableBalance() for key in key_order} + for acct_s, balance in core.account_balances(accounts_map, acct_order): + if acct_s in balances: + balances[acct_s] = balance + else: + acct_root, _, _ = acct_s.partition(':') + balances[acct_root] += balance + for key in key_order: + if key == 'Expenses': + yield balances[key] + else: + yield -balances[key] + + def write_row(self, row: FundPosts) -> None: + fund, accounts_map = row + if fund == UNRESTRICTED_FUND: + assert not self.unrestricted + self.unrestricted = accounts_map + return + self.add_row( + self.string_cell(fund, stylename=self.style_endtext), + *(self.balance_cell(bal) for bal in self._row_balances(accounts_map)), + ) + + def write(self, rows: Iterable[FundPosts]) -> None: + super().write(rows) + if self.unrestricted: + self.add_row() + self.write_row(("Unrestricted", self.unrestricted)) + + +class TextReport: + def __init__(self, + start_date: datetime.date, + stop_date: datetime.date, + out_file: TextIO) -> None: + self.start_date = start_date + self.stop_date = stop_date + self.out_file = out_file + + def _account_balances(self, + fund: str, + account_map: AccountsMap, + ) -> Iterator[Tuple[str, Sequence[str]]]: + total_fmt = f'{fund} balance as of {{}}' + for acct_s, balance in core.account_balances(account_map, BALANCE_ACCOUNTS): + if acct_s is core.OPENING_BALANCE_NAME: + acct_s = total_fmt.format(self.start_date.isoformat()) + elif acct_s is core.ENDING_BALANCE_NAME: + acct_s = total_fmt.format(self.stop_date.isoformat()) + yield acct_s, (-balance).format(None, sep='\0').split('\0') + + def write(self, rows: Iterable[FundPosts]) -> None: + output = [ + line + for fund, account_map in rows + for line in self._account_balances(fund, account_map) + ] + acct_width = max(len(acct_s) for acct_s, _ in output) + 2 + bal_width = max(len(s) for _, bal_s in output for s in bal_s) + bal_width = max(bal_width, 8) + line_fmt = f'{{:>{acct_width}}} {{:>{bal_width}}}' + print(line_fmt.replace('{:>', '{:^').format("ACCOUNT", "BALANCE"), + file=self.out_file) + fund_start = f' balance as of {self.start_date.isoformat()}' + for acct_s, bal_seq in output: + if acct_s.endswith(fund_start): + print(line_fmt.format('―' * acct_width, '―' * bal_width), + file=self.out_file) + bal_iter = iter(bal_seq) + print(line_fmt.format(acct_s, next(bal_iter)), file=self.out_file) + for bal_s in bal_iter: + print(line_fmt.format('', bal_s), file=self.out_file) + + +class ReportType(enum.Enum): + TEXT = TextReport + ODS = ODSReport + TXT = TEXT + SPREADSHEET = ODS + + @classmethod + def from_arg(cls, s: str) -> 'ReportType': + try: + return cls[s.upper()] + except KeyError: + raise ValueError(f"no report type matches {s!r}") from None + + +class ReturnFlag(enum.IntFlag): + LOAD_ERRORS = 1 + NOTHING_TO_REPORT = 8 + + +def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace: + parser = argparse.ArgumentParser(prog=PROGNAME) + cliutil.add_version_argument(parser) + parser.add_argument( + '--begin', '--start', '-b', + dest='start_date', + metavar='DATE', + type=cliutil.date_arg, + help="""Date to start reporting entries, inclusive, in YYYY-MM-DD format. +The default is one year ago. +""") + parser.add_argument( + '--end', '--stop', '-e', + dest='stop_date', + metavar='DATE', + type=cliutil.date_arg, + help="""Date to stop reporting entries, exclusive, in YYYY-MM-DD format. +The default is a year after the start date. +""") + parser.add_argument( + '--report-type', '-t', + metavar='TYPE', + type=ReportType.from_arg, + help="""Type of report to generate. `text` gives a plain two-column text +report listing accounts and balances over the period, and is the default when +you search for a specific project/fund. `ods` produces a higher-level +spreadsheet, meant to provide an overview of all funds, and is the default when +you don't specify a project/fund. +""") + parser.add_argument( + '--output-file', '-O', + metavar='PATH', + type=Path, + help="""Write the report to this file, or stdout when PATH is `-`. +The default is stdout for text reports, and a generated filename for ODS +reports. +""") + cliutil.add_loglevel_argument(parser) + parser.add_argument( + 'search_terms', + metavar='FILTER', + type=cliutil.SearchTerm.arg_parser('project', 'rt-id'), + nargs=argparse.ZERO_OR_MORE, + help="""Report on postings that match this criteria. The format is +NAME=TERM. TERM is a link or word that must exist in a posting's NAME +metadata to match. A single ticket number is a shortcut for +`rt-id=rt:NUMBER`. Any other word is a shortcut for `project=TERM`. +""") + args = parser.parse_args(arglist) + if args.report_type is None: + if any(term.meta_key == 'project' for term in args.search_terms): + args.report_type = ReportType.TEXT + else: + args.report_type = ReportType.ODS + return args + +def diff_year(date: datetime.date, diff: int) -> datetime.date: + new_year = date.year + diff + try: + return date.replace(year=new_year) + except ValueError: + # The original date is Feb 29, which doesn't exist in the new year. + if diff < 0: + return datetime.date(new_year, 2, 28) + else: + return datetime.date(new_year, 3, 1) + +def main(arglist: Optional[Sequence[str]]=None, + stdout: TextIO=sys.stdout, + stderr: TextIO=sys.stderr, + config: Optional[configmod.Config]=None, +) -> int: + args = parse_arguments(arglist) + cliutil.set_loglevel(logger, args.loglevel) + if config is None: + config = configmod.Config() + config.load_file() + + if args.stop_date is None: + if args.start_date is None: + args.stop_date = datetime.date.today() + else: + args.stop_date = diff_year(args.start_date, 1) + if args.start_date is None: + args.start_date = diff_year(args.stop_date, -1) + + returncode = 0 + books_loader = config.books_loader() + if books_loader is None: + entries, load_errors, _ = books.Loader.load_none(config.config_file_path()) + else: + entries, load_errors, _ = books_loader.load_fy_range(args.start_date, args.stop_date) + for error in load_errors: + bc_printer.print_error(error, file=stderr) + returncode |= ReturnFlag.LOAD_ERRORS + + postings = ( + post + for post in data.Posting.from_entries(entries) + if post.meta.date < args.stop_date + and post.account.is_under(*BALANCE_ACCOUNTS) + ) + for search_term in args.search_terms: + postings = search_term.filter_postings(postings) + fund_postings = { + key: related + for key, related in core.RelatedPostings.group_by_meta(postings, 'project') + if isinstance(key, str) + } + period_cls = core.PeriodPostings.with_start_date(args.start_date) + fund_map = collections.OrderedDict( + (fund, dict(period_cls.group_by_account(fund_postings[fund]))) + for fund in sorted(fund_postings, key=lambda s: locale.strxfrm(s.casefold())) + ) + if not fund_map: + logger.warning("no matching postings found to report") + returncode |= ReturnFlag.NOTHING_TO_REPORT + elif args.report_type is ReportType.TEXT: + out_file = cliutil.text_output(args.output_file, stdout) + report = TextReport(args.start_date, args.stop_date, out_file) + report.write(fund_map.items()) + else: + ods_report = ODSReport(args.start_date, args.stop_date) + ods_report.write(fund_map.items()) + if args.output_file is None: + out_dir_path = config.repository_path() or Path() + args.output_file = out_dir_path / 'FundReport_{}_{}.ods'.format( + args.start_date.isoformat(), args.stop_date.isoformat(), + ) + logger.info("Writing report to %s", args.output_file) + ods_file = cliutil.bytes_output(args.output_file, stdout) + ods_report.save_file(ods_file) + return 0 if returncode == 0 else 16 + returncode + +entry_point = cliutil.make_entry_point(__name__, PROGNAME) + +if __name__ == '__main__': + exit(entry_point()) diff --git a/setup.py b/setup.py index 911a3da..60c10d6 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup setup( name='conservancy_beancount', description="Plugin, library, and reports for reading Conservancy's books", - version='1.2.6', + version='1.3.0', author='Software Freedom Conservancy', author_email='info@sfconservancy.org', license='GNU AGPLv3+', @@ -36,6 +36,7 @@ setup( entry_points={ 'console_scripts': [ 'accrual-report = conservancy_beancount.reports.accrual:entry_point', + 'fund-report = conservancy_beancount.reports.fund:entry_point', 'ledger-report = conservancy_beancount.reports.ledger:entry_point', ], }, diff --git a/tests/books/fund.beancount b/tests/books/fund.beancount new file mode 100644 index 0000000..d452cf5 --- /dev/null +++ b/tests/books/fund.beancount @@ -0,0 +1,71 @@ +option "inferred_tolerance_default" "USD:0.01" + +2018-01-01 open Equity:Funds:Restricted +2018-01-01 open Equity:Funds:Unrestricted +2018-01-01 open Equity:Realized:CurrencyConversion +2018-01-01 open Assets:Checking +2018-01-01 open Expenses:Other +2018-01-01 open Income:Other + +2018-02-28 * "Opening balances" + Equity:Funds:Unrestricted -4,000 USD + project: "Conservancy" + Equity:Funds:Restricted -3,000 USD + project: "Alpha" + Equity:Funds:Restricted -2,000 USD + project: "Bravo" + Equity:Funds:Restricted -1,000 USD + project: "Charlie" + Assets:Checking 10,000 USD + +2018-03-03 * "Conservancy income 2018" + project: "Conservancy" + Income:Other -40 USD + Assets:Checking 40 USD + +2018-03-06 * "Conservancy expense 2018" + project: "Conservancy" + Expenses:Other 4 USD + Assets:Checking -4 USD + +2018-06-03 * "Alpha income 2018A" + project: "Alpha" + Income:Other -30 USD + Assets:Checking 30 USD + +2018-06-06 * "Alpha income 2018B" + project: "Alpha" + Income:Other -30 USD + Assets:Checking 30 USD + +2018-09-03 * "Bravo expense" + project: "Bravo" + Expenses:Other 20 USD + Assets:Checking -20 USD + +2019-03-03 * "Conservancy income 2019" + project: "Conservancy" + Income:Other -44 EUR {1.0 USD} + Assets:Checking 40 USD + Equity:Realized:CurrencyConversion + +2019-03-06 * "Conservancy expense 2019" + project: "Conservancy" + Expenses:Other 4.40 EUR {1.0 USD} + Assets:Checking -4.00 USD + Equity:Realized:CurrencyConversion + +2019-06-03 * "Alpha expense 2019A" + project: "Alpha" + Expenses:Other 3 USD + Assets:Checking -3 USD + +2019-06-06 * "Alpha expense 2019B" + project: "Alpha" + Expenses:Other 3 USD + Assets:Checking -3 USD + +2019-09-03 * "Bravo income" + project: "Bravo" + Income:Other -200 USD + Assets:Checking 200 USD diff --git a/tests/test_reports_fund.py b/tests/test_reports_fund.py new file mode 100644 index 0000000..a57318d --- /dev/null +++ b/tests/test_reports_fund.py @@ -0,0 +1,227 @@ +"""test_reports_fund.py - Unit tests for fund report""" +# Copyright © 2020 Brett Smith +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import collections +import copy +import datetime +import io +import itertools + +import pytest + +from . import testutil + +import babel.numbers +import odf.opendocument +import odf.table + +from beancount import loader as bc_loader +from conservancy_beancount import data +from conservancy_beancount.reports import core +from conservancy_beancount.reports import fund + +from decimal import Decimal + +_ledger_load = bc_loader.load_file(testutil.test_path('books/fund.beancount')) +START_DATE = datetime.date(2018, 3, 1) +MID_DATE = datetime.date(2019, 3, 1) +STOP_DATE = datetime.date(2020, 3, 1) + +OPENING_BALANCES = { + 'Alpha': 3000, + 'Bravo': 2000, + 'Charlie': 1000, + 'Conservancy': 4000, +} + +BALANCES_BY_YEAR = { + ('Conservancy', 2018): [ + ('Income:Other', 40), + ('Expenses:Other', -4), + ], + ('Conservancy', 2019): [ + ('Income:Other', 44), + ('Expenses:Other', Decimal('-4.40')), + ('Equity:Realized:CurrencyConversion', Decimal('-3.60')), + ], + ('Alpha', 2018): [ + ('Income:Other', 60), + ], + ('Alpha', 2019): [ + ('Expenses:Other', -6), + ], + ('Bravo', 2018): [ + ('Expenses:Other', -20), + ], + ('Bravo', 2019): [ + ('Income:Other', 200), + ], +} + +@pytest.fixture +def fund_entries(): + return copy.deepcopy(_ledger_load[0]) + +def fund_postings(entries, project, stop_date): + return ( + post for post in data.Posting.from_entries(entries) + if post.meta.date < stop_date + and post.account.is_under('Equity', 'Income', 'Expenses') + and post.meta.get('project') == project + ) + +def split_text_lines(output): + for line in output: + account, amount = line.rsplit(None, 1) + yield account.strip(), amount + +def format_amount(amount, currency='USD'): + return babel.numbers.format_currency( + amount, currency, format_type='accounting', + ) + +def check_text_report(output, project, start_date, stop_date): + _, _, project = project.rpartition('=') + balance_amount = Decimal(OPENING_BALANCES[project]) + expected = collections.defaultdict(Decimal) + for year in range(2018, stop_date.year): + try: + amounts = BALANCES_BY_YEAR[(project, year)] + except KeyError: + pass + else: + for account, amount in amounts: + if year < start_date.year: + balance_amount += amount + else: + expected[account] += amount + expected.default_factory = None + actual = split_text_lines(output) + next(actual); next(actual) # Discard headers + open_acct, open_amt = next(actual) + assert open_acct == "{} balance as of {}".format( + project, start_date.isoformat(), + ) + assert open_amt == format_amount(balance_amount) + for expect_account in [ + 'Equity:Realized:CurrencyConversion', + 'Income:Other', + 'Expenses:Other', + ]: + try: + expect_amount = expected[expect_account] + except KeyError: + continue + else: + actual_account, actual_amount = next(actual) + assert actual_account == expect_account + assert actual_amount == format_amount(expect_amount) + balance_amount += expect_amount + end_acct, end_amt = next(actual) + assert end_acct == "{} balance as of {}".format( + project, stop_date.isoformat(), + ) + assert end_amt == format_amount(balance_amount) + assert next(actual, None) is None + +def check_ods_report(ods, start_date, stop_date): + account_bals = collections.OrderedDict((key, { + 'opening': Decimal(amount), + 'Income': Decimal(0), + 'Expenses': Decimal(0), + 'Equity': Decimal(0), + }) for key, amount in sorted(OPENING_BALANCES.items())) + for fund, year in itertools.product(account_bals, range(2018, stop_date.year)): + try: + amounts = BALANCES_BY_YEAR[(fund, year)] + except KeyError: + pass + else: + for account, amount in amounts: + if year < start_date.year: + acct_key = 'opening' + else: + acct_key, _, _ = account.partition(':') + account_bals[fund][acct_key] += amount + account_bals['Unrestricted'] = account_bals.pop('Conservancy') + for row in ods.getElementsByType(odf.table.TableRow): + cells = iter(testutil.ODSCell.from_row(row)) + try: + fund = next(cells).firstChild.text + except (AttributeError, StopIteration): + fund = None + if fund in account_bals: + balances = account_bals.pop(fund) + assert next(cells).value == balances['opening'] + assert next(cells).value == balances['Income'] + assert next(cells).value == -balances['Expenses'] + if balances['Equity']: + assert next(cells).value == balances['Equity'] + else: + assert not next(cells).value + assert next(cells).value == sum(balances.values()) + assert not account_bals, "did not see all funds in report" + +def run_main(out_type, arglist, config=None): + if config is None: + config = testutil.TestConfig( + books_path=testutil.test_path('books/fund.beancount'), + ) + arglist.insert(0, '--output-file=-') + output = out_type() + errors = io.StringIO() + retcode = fund.main(arglist, output, errors, config) + output.seek(0) + return retcode, output, errors + +@pytest.mark.parametrize('project,start_date,stop_date', [ + ('Conservancy', START_DATE, STOP_DATE), + ('project=Conservancy', MID_DATE, STOP_DATE), + ('Conservancy', START_DATE, MID_DATE), + ('Alpha', START_DATE, STOP_DATE), + ('project=Alpha', MID_DATE, STOP_DATE), + ('Alpha', START_DATE, MID_DATE), + ('Bravo', START_DATE, STOP_DATE), + ('project=Bravo', MID_DATE, STOP_DATE), + ('Bravo', START_DATE, MID_DATE), + ('project=Charlie', START_DATE, STOP_DATE), +]) +def test_text_report(project, start_date, stop_date): + retcode, output, errors = run_main(io.StringIO, [ + '-b', start_date.isoformat(), '-e', stop_date.isoformat(), project, + ]) + assert not errors.getvalue() + assert retcode == 0 + check_text_report(output, project, start_date, stop_date) + +@pytest.mark.parametrize('start_date,stop_date', [ + (START_DATE, STOP_DATE), + (MID_DATE, STOP_DATE), + (START_DATE, MID_DATE), +]) +def test_ods_report(start_date, stop_date): + retcode, output, errors = run_main(io.BytesIO, [ + '--begin', start_date.isoformat(), '--end', stop_date.isoformat(), + ]) + assert not errors.getvalue() + assert retcode == 0 + ods = odf.opendocument.load(output) + check_ods_report(ods, start_date, stop_date) + +def test_main_no_postings(caplog): + retcode, output, errors = run_main(io.StringIO, ['NonexistentProject']) + assert retcode == 24 + assert any(log.levelname == 'WARNING' for log in caplog.records)