fund: New report.

conservancy_beancount/reports/fund.py

@@ -0,0 +1,359 @@
+"""funds.py - Funds report from Beancount
+
+This tool produces a report of account balances for restricted funds,
+including income, expenses, and realized gain/loss over a given time period.
+Restricted funds are designated with the "project" metadata.
+
+Specify the date range you want to report with the ``--begin`` and ``--end``
+options.
+
+Select the accounts you want to report with the ``--account`` option. You can
+specify this option multiple times. The report will include at least one sheet
+for each account you specify. Subaccounts will be reported on that sheet as
+well.
+
+Select the postings you want to report by passing metadata search terms in
+``name=value`` format.
+
+Run ``ledger-report --help`` for abbreviations and other options.
+
+Examples
+--------
+
+Generate a report of all restricted funds in ODS format over the last year::
+
+      fund-report
+
+Query a specific restricted fund and get a quick balance on the terminal::
+
+      fund-report <PROJECT>
+"""
+# Copyright © 2020  Brett Smith
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import argparse
+import collections
+import datetime
+import enum
+import locale
+import logging
+import sys
+
+from typing import (
+    Dict,
+    Iterable,
+    Iterator,
+    Mapping,
+    Optional,
+    Sequence,
+    TextIO,
+    Tuple,
+)
+from ..beancount_types import (
+    MetaValue,
+)
+
+from pathlib import Path
+
+import odf.table  # type:ignore[import]
+
+from beancount.parser import printer as bc_printer
+
+from . import core
+from .. import books
+from .. import cliutil
+from .. import config as configmod
+from .. import data
+
+AccountsMap = Mapping[data.Account, core.PeriodPostings]
+FundPosts = Tuple[MetaValue, AccountsMap]
+
+BALANCE_ACCOUNTS = ['Equity', 'Income', 'Expenses']
+PROGNAME = 'fund-report'
+UNRESTRICTED_FUND = 'Conservancy'
+logger = logging.getLogger('conservancy_beancount.reports.fund')
+
+class ODSReport(core.BaseODS[FundPosts, None]):
+    def __init__(self, start_date: datetime.date, stop_date: datetime.date) -> None:
+        super().__init__()
+        self.start_date = start_date
+        self.stop_date = stop_date
+        self.unrestricted: AccountsMap = {}
+
+    def section_key(self, row: FundPosts) -> None:
+        return None
+
+    def start_spreadsheet(self) -> None:
+        self.use_sheet("Fund Report")
+        for width in [2.5, 1.5, 1.2, 1.2, 1.2, 1.5]:
+            col_style = self.column_style(width)
+            self.sheet.addElement(odf.table.TableColumn(stylename=col_style))
+        center_bold = self.merge_styles(self.style_centertext, self.style_bold)
+        self.add_row(
+            self.string_cell(
+                "Fund", stylename=self.merge_styles(self.style_endtext, self.style_bold),
+            ),
+            self.multiline_cell(["Balance as of", self.start_date.isoformat()],
+                                stylename=center_bold),
+            self.string_cell("Income", stylename=center_bold),
+            self.string_cell("Expenses", stylename=center_bold),
+            self.multiline_cell(["Realized", "Gain/Loss"], stylename=center_bold),
+            self.multiline_cell(["Balance as of", self.stop_date.isoformat()],
+                                stylename=center_bold),
+        )
+        self.lock_first_row()
+        self.add_row()
+        self.add_row(self.string_cell(
+            f"Fund Report From {self.start_date.isoformat()} To {self.stop_date.isoformat()}",
+            stylename=center_bold,
+            numbercolumnsspanned=6,
+        ))
+        self.add_row()
+
+    def _row_balances(self, accounts_map: AccountsMap) -> Iterable[core.Balance]:
+        acct_order = ['Income', 'Expenses', 'Equity']
+        key_order = [core.OPENING_BALANCE_NAME, *acct_order, core.ENDING_BALANCE_NAME]
+        balances: Dict[str, core.Balance] = {key: core.MutableBalance() for key in key_order}
+        for acct_s, balance in core.account_balances(accounts_map, acct_order):
+            if acct_s in balances:
+                balances[acct_s] = balance
+            else:
+                acct_root, _, _ = acct_s.partition(':')
+                balances[acct_root] += balance
+        for key in key_order:
+            if key == 'Expenses':
+                yield balances[key]
+            else:
+                yield -balances[key]
+
+    def write_row(self, row: FundPosts) -> None:
+        fund, accounts_map = row
+        if fund == UNRESTRICTED_FUND:
+            assert not self.unrestricted
+            self.unrestricted = accounts_map
+            return
+        self.add_row(
+            self.string_cell(fund, stylename=self.style_endtext),
+            *(self.balance_cell(bal) for bal in self._row_balances(accounts_map)),
+        )
+
+    def write(self, rows: Iterable[FundPosts]) -> None:
+        super().write(rows)
+        if self.unrestricted:
+            self.add_row()
+            self.write_row(("Unrestricted", self.unrestricted))
+
+
+class TextReport:
+    def __init__(self,
+                 start_date: datetime.date,
+                 stop_date: datetime.date,
+                 out_file: TextIO) -> None:
+        self.start_date = start_date
+        self.stop_date = stop_date
+        self.out_file = out_file
+
+    def _account_balances(self,
+                          fund: str,
+                          account_map: AccountsMap,
+    ) -> Iterator[Tuple[str, Sequence[str]]]:
+        total_fmt = f'{fund} balance as of {{}}'
+        for acct_s, balance in core.account_balances(account_map, BALANCE_ACCOUNTS):
+            if acct_s is core.OPENING_BALANCE_NAME:
+                acct_s = total_fmt.format(self.start_date.isoformat())
+            elif acct_s is core.ENDING_BALANCE_NAME:
+                acct_s = total_fmt.format(self.stop_date.isoformat())
+            yield acct_s, (-balance).format(None, sep='\0').split('\0')
+
+    def write(self, rows: Iterable[FundPosts]) -> None:
+        output = [
+            line
+            for fund, account_map in rows
+            for line in self._account_balances(fund, account_map)
+        ]
+        acct_width = max(len(acct_s) for acct_s, _ in output) + 2
+        bal_width = max(len(s) for _, bal_s in output for s in bal_s)
+        bal_width = max(bal_width, 8)
+        line_fmt = f'{{:>{acct_width}}}  {{:>{bal_width}}}'
+        print(line_fmt.replace('{:>', '{:^').format("ACCOUNT", "BALANCE"),
+              file=self.out_file)
+        fund_start = f' balance as of {self.start_date.isoformat()}'
+        for acct_s, bal_seq in output:
+            if acct_s.endswith(fund_start):
+                print(line_fmt.format('―' * acct_width, '―' * bal_width),
+                      file=self.out_file)
+            bal_iter = iter(bal_seq)
+            print(line_fmt.format(acct_s, next(bal_iter)), file=self.out_file)
+            for bal_s in bal_iter:
+                print(line_fmt.format('', bal_s), file=self.out_file)
+
+
+class ReportType(enum.Enum):
+    TEXT = TextReport
+    ODS = ODSReport
+    TXT = TEXT
+    SPREADSHEET = ODS
+
+    @classmethod
+    def from_arg(cls, s: str) -> 'ReportType':
+        try:
+            return cls[s.upper()]
+        except KeyError:
+            raise ValueError(f"no report type matches {s!r}") from None
+
+
+class ReturnFlag(enum.IntFlag):
+    LOAD_ERRORS = 1
+    NOTHING_TO_REPORT = 8
+
+
+def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(prog=PROGNAME)
+    cliutil.add_version_argument(parser)
+    parser.add_argument(
+        '--begin', '--start', '-b',
+        dest='start_date',
+        metavar='DATE',
+        type=cliutil.date_arg,
+        help="""Date to start reporting entries, inclusive, in YYYY-MM-DD format.
+The default is one year ago.
+""")
+    parser.add_argument(
+        '--end', '--stop', '-e',
+        dest='stop_date',
+        metavar='DATE',
+        type=cliutil.date_arg,
+        help="""Date to stop reporting entries, exclusive, in YYYY-MM-DD format.
+The default is a year after the start date.
+""")
+    parser.add_argument(
+        '--report-type', '-t',
+        metavar='TYPE',
+        type=ReportType.from_arg,
+        help="""Type of report to generate. `text` gives a plain two-column text
+report listing accounts and balances over the period, and is the default when
+you search for a specific project/fund. `ods` produces a higher-level
+spreadsheet, meant to provide an overview of all funds, and is the default when
+you don't specify a project/fund.
+""")
+    parser.add_argument(
+        '--output-file', '-O',
+        metavar='PATH',
+        type=Path,
+        help="""Write the report to this file, or stdout when PATH is `-`.
+The default is stdout for text reports, and a generated filename for ODS
+reports.
+""")
+    cliutil.add_loglevel_argument(parser)
+    parser.add_argument(
+        'search_terms',
+        metavar='FILTER',
+        type=cliutil.SearchTerm.arg_parser('project', 'rt-id'),
+        nargs=argparse.ZERO_OR_MORE,
+        help="""Report on postings that match this criteria. The format is
+NAME=TERM. TERM is a link or word that must exist in a posting's NAME
+metadata to match. A single ticket number is a shortcut for
+`rt-id=rt:NUMBER`. Any other word is a shortcut for `project=TERM`.
+""")
+    args = parser.parse_args(arglist)
+    if args.report_type is None:
+        if any(term.meta_key == 'project' for term in args.search_terms):
+            args.report_type = ReportType.TEXT
+        else:
+            args.report_type = ReportType.ODS
+    return args
+
+def diff_year(date: datetime.date, diff: int) -> datetime.date:
+    new_year = date.year + diff
+    try:
+        return date.replace(year=new_year)
+    except ValueError:
+        # The original date is Feb 29, which doesn't exist in the new year.
+        if diff < 0:
+            return datetime.date(new_year, 2, 28)
+        else:
+            return datetime.date(new_year, 3, 1)
+
+def main(arglist: Optional[Sequence[str]]=None,
+         stdout: TextIO=sys.stdout,
+         stderr: TextIO=sys.stderr,
+         config: Optional[configmod.Config]=None,
+) -> int:
+    args = parse_arguments(arglist)
+    cliutil.set_loglevel(logger, args.loglevel)
+    if config is None:
+        config = configmod.Config()
+        config.load_file()
+
+    if args.stop_date is None:
+        if args.start_date is None:
+            args.stop_date = datetime.date.today()
+        else:
+            args.stop_date = diff_year(args.start_date, 1)
+    if args.start_date is None:
+        args.start_date = diff_year(args.stop_date, -1)
+
+    returncode = 0
+    books_loader = config.books_loader()
+    if books_loader is None:
+        entries, load_errors, _ = books.Loader.load_none(config.config_file_path())
+    else:
+        entries, load_errors, _ = books_loader.load_fy_range(args.start_date, args.stop_date)
+    for error in load_errors:
+        bc_printer.print_error(error, file=stderr)
+        returncode |= ReturnFlag.LOAD_ERRORS
+
+    postings = (
+        post
+        for post in data.Posting.from_entries(entries)
+        if post.meta.date < args.stop_date
+        and post.account.is_under(*BALANCE_ACCOUNTS)
+    )
+    for search_term in args.search_terms:
+        postings = search_term.filter_postings(postings)
+    fund_postings = {
+        key: related
+        for key, related in core.RelatedPostings.group_by_meta(postings, 'project')
+        if isinstance(key, str)
+    }
+    period_cls = core.PeriodPostings.with_start_date(args.start_date)
+    fund_map = collections.OrderedDict(
+        (fund, dict(period_cls.group_by_account(fund_postings[fund])))
+        for fund in sorted(fund_postings, key=lambda s: locale.strxfrm(s.casefold()))
+    )
+    if not fund_map:
+        logger.warning("no matching postings found to report")
+        returncode |= ReturnFlag.NOTHING_TO_REPORT
+    elif args.report_type is ReportType.TEXT:
+        out_file = cliutil.text_output(args.output_file, stdout)
+        report = TextReport(args.start_date, args.stop_date, out_file)
+        report.write(fund_map.items())
+    else:
+        ods_report = ODSReport(args.start_date, args.stop_date)
+        ods_report.write(fund_map.items())
+        if args.output_file is None:
+            out_dir_path = config.repository_path() or Path()
+            args.output_file = out_dir_path / 'FundReport_{}_{}.ods'.format(
+                args.start_date.isoformat(), args.stop_date.isoformat(),
+            )
+            logger.info("Writing report to %s", args.output_file)
+        ods_file = cliutil.bytes_output(args.output_file, stdout)
+        ods_report.save_file(ods_file)
+    return 0 if returncode == 0 else 16 + returncode
+
+entry_point = cliutil.make_entry_point(__name__, PROGNAME)
+
+if __name__ == '__main__':
+    exit(entry_point())

setup.py

@@ -5,7 +5,7 @@ from setuptools import setup
 setup(
     name='conservancy_beancount',
     description="Plugin, library, and reports for reading Conservancy's books",
-    version='1.2.6',
+    version='1.3.0',
     author='Software Freedom Conservancy',
     author_email='info@sfconservancy.org',
     license='GNU AGPLv3+',
@@ -36,6 +36,7 @@ setup(
     entry_points={
         'console_scripts': [
             'accrual-report = conservancy_beancount.reports.accrual:entry_point',
+            'fund-report = conservancy_beancount.reports.fund:entry_point',
             'ledger-report = conservancy_beancount.reports.ledger:entry_point',
         ],
     },

tests/books/fund.beancount

@@ -0,0 +1,71 @@
+option "inferred_tolerance_default" "USD:0.01"
+
+2018-01-01 open Equity:Funds:Restricted
+2018-01-01 open Equity:Funds:Unrestricted
+2018-01-01 open Equity:Realized:CurrencyConversion
+2018-01-01 open Assets:Checking
+2018-01-01 open Expenses:Other
+2018-01-01 open Income:Other
+
+2018-02-28 * "Opening balances"
+  Equity:Funds:Unrestricted  -4,000 USD
+  project: "Conservancy"
+  Equity:Funds:Restricted    -3,000 USD
+  project: "Alpha"
+  Equity:Funds:Restricted    -2,000 USD
+  project: "Bravo"
+  Equity:Funds:Restricted    -1,000 USD
+  project: "Charlie"
+  Assets:Checking            10,000 USD
+
+2018-03-03 * "Conservancy income 2018"
+  project: "Conservancy"
+  Income:Other    -40 USD
+  Assets:Checking  40 USD
+
+2018-03-06 * "Conservancy expense 2018"
+  project: "Conservancy"
+  Expenses:Other    4 USD
+  Assets:Checking  -4 USD
+
+2018-06-03 * "Alpha income 2018A"
+  project: "Alpha"
+  Income:Other    -30 USD
+  Assets:Checking  30 USD
+
+2018-06-06 * "Alpha income 2018B"
+  project: "Alpha"
+  Income:Other    -30 USD
+  Assets:Checking  30 USD
+
+2018-09-03 * "Bravo expense"
+  project: "Bravo"
+  Expenses:Other    20 USD
+  Assets:Checking  -20 USD
+
+2019-03-03 * "Conservancy income 2019"
+  project: "Conservancy"
+  Income:Other    -44 EUR {1.0 USD}
+  Assets:Checking  40 USD
+  Equity:Realized:CurrencyConversion
+
+2019-03-06 * "Conservancy expense 2019"
+  project: "Conservancy"
+  Expenses:Other    4.40 EUR {1.0 USD}
+  Assets:Checking  -4.00 USD
+  Equity:Realized:CurrencyConversion
+
+2019-06-03 * "Alpha expense 2019A"
+  project: "Alpha"
+  Expenses:Other    3 USD
+  Assets:Checking  -3 USD
+
+2019-06-06 * "Alpha expense 2019B"
+  project: "Alpha"
+  Expenses:Other    3 USD
+  Assets:Checking  -3 USD
+
+2019-09-03 * "Bravo income"
+  project: "Bravo"
+  Income:Other    -200 USD
+  Assets:Checking  200 USD

tests/test_reports_fund.py

@@ -0,0 +1,227 @@
+"""test_reports_fund.py - Unit tests for fund report"""
+# Copyright © 2020  Brett Smith
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import collections
+import copy
+import datetime
+import io
+import itertools
+
+import pytest
+
+from . import testutil
+
+import babel.numbers
+import odf.opendocument
+import odf.table
+
+from beancount import loader as bc_loader
+from conservancy_beancount import data
+from conservancy_beancount.reports import core
+from conservancy_beancount.reports import fund
+
+from decimal import Decimal
+
+_ledger_load = bc_loader.load_file(testutil.test_path('books/fund.beancount'))
+START_DATE = datetime.date(2018, 3, 1)
+MID_DATE = datetime.date(2019, 3, 1)
+STOP_DATE = datetime.date(2020, 3, 1)
+
+OPENING_BALANCES = {
+    'Alpha': 3000,
+    'Bravo': 2000,
+    'Charlie': 1000,
+    'Conservancy': 4000,
+}
+
+BALANCES_BY_YEAR = {
+    ('Conservancy', 2018): [
+        ('Income:Other', 40),
+        ('Expenses:Other', -4),
+    ],
+    ('Conservancy', 2019): [
+        ('Income:Other', 44),
+        ('Expenses:Other', Decimal('-4.40')),
+        ('Equity:Realized:CurrencyConversion', Decimal('-3.60')),
+    ],
+    ('Alpha', 2018): [
+        ('Income:Other', 60),
+    ],
+    ('Alpha', 2019): [
+        ('Expenses:Other', -6),
+    ],
+    ('Bravo', 2018): [
+        ('Expenses:Other', -20),
+    ],
+    ('Bravo', 2019): [
+        ('Income:Other', 200),
+    ],
+}
+
+@pytest.fixture
+def fund_entries():
+    return copy.deepcopy(_ledger_load[0])
+
+def fund_postings(entries, project, stop_date):
+    return (
+        post for post in data.Posting.from_entries(entries)
+        if post.meta.date < stop_date
+        and post.account.is_under('Equity', 'Income', 'Expenses')
+        and post.meta.get('project') == project
+    )
+
+def split_text_lines(output):
+    for line in output:
+        account, amount = line.rsplit(None, 1)
+        yield account.strip(), amount
+
+def format_amount(amount, currency='USD'):
+    return babel.numbers.format_currency(
+        amount, currency, format_type='accounting',
+    )
+
+def check_text_report(output, project, start_date, stop_date):
+    _, _, project = project.rpartition('=')
+    balance_amount = Decimal(OPENING_BALANCES[project])
+    expected = collections.defaultdict(Decimal)
+    for year in range(2018, stop_date.year):
+        try:
+            amounts = BALANCES_BY_YEAR[(project, year)]
+        except KeyError:
+            pass
+        else:
+            for account, amount in amounts:
+                if year < start_date.year:
+                    balance_amount += amount
+                else:
+                    expected[account] += amount
+    expected.default_factory = None
+    actual = split_text_lines(output)
+    next(actual); next(actual)  # Discard headers
+    open_acct, open_amt = next(actual)
+    assert open_acct == "{} balance as of {}".format(
+        project, start_date.isoformat(),
+    )
+    assert open_amt == format_amount(balance_amount)
+    for expect_account in [
+            'Equity:Realized:CurrencyConversion',
+            'Income:Other',
+            'Expenses:Other',
+    ]:
+        try:
+            expect_amount = expected[expect_account]
+        except KeyError:
+            continue
+        else:
+            actual_account, actual_amount = next(actual)
+            assert actual_account == expect_account
+            assert actual_amount == format_amount(expect_amount)
+            balance_amount += expect_amount
+    end_acct, end_amt = next(actual)
+    assert end_acct == "{} balance as of {}".format(
+        project, stop_date.isoformat(),
+    )
+    assert end_amt == format_amount(balance_amount)
+    assert next(actual, None) is None
+
+def check_ods_report(ods, start_date, stop_date):
+    account_bals = collections.OrderedDict((key, {
+        'opening': Decimal(amount),
+        'Income': Decimal(0),
+        'Expenses': Decimal(0),
+        'Equity': Decimal(0),
+    }) for key, amount in sorted(OPENING_BALANCES.items()))
+    for fund, year in itertools.product(account_bals, range(2018, stop_date.year)):
+        try:
+            amounts = BALANCES_BY_YEAR[(fund, year)]
+        except KeyError:
+            pass
+        else:
+            for account, amount in amounts:
+                if year < start_date.year:
+                    acct_key = 'opening'
+                else:
+                    acct_key, _, _ = account.partition(':')
+                account_bals[fund][acct_key] += amount
+    account_bals['Unrestricted'] = account_bals.pop('Conservancy')
+    for row in ods.getElementsByType(odf.table.TableRow):
+        cells = iter(testutil.ODSCell.from_row(row))
+        try:
+            fund = next(cells).firstChild.text
+        except (AttributeError, StopIteration):
+            fund = None
+        if fund in account_bals:
+            balances = account_bals.pop(fund)
+            assert next(cells).value == balances['opening']
+            assert next(cells).value == balances['Income']
+            assert next(cells).value == -balances['Expenses']
+            if balances['Equity']:
+                assert next(cells).value == balances['Equity']
+            else:
+                assert not next(cells).value
+            assert next(cells).value == sum(balances.values())
+    assert not account_bals, "did not see all funds in report"
+
+def run_main(out_type, arglist, config=None):
+    if config is None:
+        config = testutil.TestConfig(
+            books_path=testutil.test_path('books/fund.beancount'),
+        )
+    arglist.insert(0, '--output-file=-')
+    output = out_type()
+    errors = io.StringIO()
+    retcode = fund.main(arglist, output, errors, config)
+    output.seek(0)
+    return retcode, output, errors
+
+@pytest.mark.parametrize('project,start_date,stop_date', [
+    ('Conservancy', START_DATE, STOP_DATE),
+    ('project=Conservancy', MID_DATE, STOP_DATE),
+    ('Conservancy', START_DATE, MID_DATE),
+    ('Alpha', START_DATE, STOP_DATE),
+    ('project=Alpha', MID_DATE, STOP_DATE),
+    ('Alpha', START_DATE, MID_DATE),
+    ('Bravo', START_DATE, STOP_DATE),
+    ('project=Bravo', MID_DATE, STOP_DATE),
+    ('Bravo', START_DATE, MID_DATE),
+    ('project=Charlie', START_DATE, STOP_DATE),
+])
+def test_text_report(project, start_date, stop_date):
+    retcode, output, errors = run_main(io.StringIO, [
+        '-b', start_date.isoformat(), '-e', stop_date.isoformat(), project,
+    ])
+    assert not errors.getvalue()
+    assert retcode == 0
+    check_text_report(output, project, start_date, stop_date)
+
+@pytest.mark.parametrize('start_date,stop_date', [
+    (START_DATE, STOP_DATE),
+    (MID_DATE, STOP_DATE),
+    (START_DATE, MID_DATE),
+])
+def test_ods_report(start_date, stop_date):
+    retcode, output, errors = run_main(io.BytesIO, [
+        '--begin', start_date.isoformat(), '--end', stop_date.isoformat(),
+    ])
+    assert not errors.getvalue()
+    assert retcode == 0
+    ods = odf.opendocument.load(output)
+    check_ods_report(ods, start_date, stop_date)
+
+def test_main_no_postings(caplog):
+    retcode, output, errors = run_main(io.StringIO, ['NonexistentProject'])
+    assert retcode == 24
+    assert any(log.levelname == 'WARNING' for log in caplog.records)