books.Loader: New loading strategy.

The old loading strategy didn't load options, which yielded some
spurious errors. It also created awkward duplication of plugin
information in the code as well as the books.

Implement a new loading strategy that works by reading one of the
"main files" under the books/ subdirectory and includes entries
for additional FYs beyond that.

This is still not ideal in a lot of ways. In particular, Beancount can't
cache any results, causing any load to be slower than it theoretically could
be. I expect more commits to follow. But some of them might require
restructuring the books, and that should happen separately.

conservancy_beancount/books.py

@@ -14,7 +14,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import contextlib
 import datetime
+import os
 
 from pathlib import Path
 
@@ -23,6 +25,7 @@ from beancount import loader as bc_loader
 from typing import (
     Any,
     Iterable,
+    Iterator,
     Mapping,
     NamedTuple,
     Optional,
@@ -33,9 +36,17 @@ from .beancount_types import (
 )
 
 PathLike = Union[str, Path]
-PluginsSpec = Mapping[str, Optional[str]]
 Year = Union[int, datetime.date]
 
+@contextlib.contextmanager
+def workdir(path: PathLike) -> Iterator[Path]:
+    old_dir = os.getcwd()
+    os.chdir(path)
+    try:
+        yield Path(old_dir)
+    finally:
+        os.chdir(old_dir)
+
 class FiscalYear(NamedTuple):
     month: int = 3
     day: int = 1
@@ -87,14 +98,9 @@ class FiscalYear(NamedTuple):
 class Loader:
     """Load Beancount books organized by fiscal year"""
 
-    DEFAULT_PLUGINS: PluginsSpec = {
-        'conservancy_beancount.plugin': None,
-    }
-
     def __init__(self,
                  books_root: Path,
                  fiscal_year: FiscalYear,
-                 plugins: Optional[PluginsSpec]=None,
     ) -> None:
         """Set up a books loader
 
@@ -102,69 +108,56 @@ class Loader:
         * books_root: A Path to a Beancount books checkout.
         * fiscal_year: A FiscalYear object, used to determine what books to
           load for a given date range.
-        * plugins: A mapping that specifies what plugins should be loaded
-          before any books. The keys are plugin names, and the values are the
-          configuration parameters string to follow. A value of None means the
-          plugin takes no configuration string. By default, the loader loads
-          conservancy_beancount.plugin.
         """
-        if plugins is None:
-            plugins = self.DEFAULT_PLUGINS
         self.books_root = books_root
+        self.opening_root = books_root / 'books'
         self.fiscal_year = fiscal_year
-        self.plugins = dict(plugins)
 
-    def _format_include(self, year: int, subdir: PathLike='') -> str:
+    def _iter_fy_books(self, fy_range: Iterable[int]) -> Iterator[Path]:
-        file_path = Path(self.books_root, subdir, f'{year}.beancount')
+        dir_path = self.opening_root
-        return f'include "{file_path}"'
+        for year in fy_range:
-
+            path = dir_path / f'{year}.beancount'
-    def _format_plugin(self, name: str, optstring: Optional[str]=None) -> str:
+            if path.exists():
-        if optstring is None:
+                yield path
-            return f'plugin "{name}"'
+                dir_path = self.books_root
-        else:
-            return f'plugin "{name}" "{optstring}"'
 
     def fy_range_string(self,
                         from_fy: Year,
                         to_fy: Optional[Year]=None,
-                        plugins: Optional[PluginsSpec]=None,
     ) -> str:
         """Return a string to load books for a range of fiscal years
 
         This method generates a range of fiscal years by calling
         FiscalYear.range() with its first two arguments. It returns a string of
-        Beancount directives to load all plugins and Beancount files for that
+        Beancount directives to load the books from the first available fiscal
-        range of fiscal years, suitable for passing to
+        year through the end of the range.
-        beancount.loader.load_string().
 
-        You can specify what plugins to load with the plugins argument. If not
+        Pass the string to Loader.load_string() to actually load data from it.
-        specified, the string loads the plugins specified for this instance.
-        See the __init__ docstring for details.
         """
-        if plugins is None:
+        paths = self._iter_fy_books(self.fiscal_year.range(from_fy, to_fy))
-            plugins = self.plugins
-        years = iter(self.fiscal_year.range(from_fy, to_fy))
         try:
-            books_start = self._format_include(next(years), 'books')
+            with next(paths).open() as opening_books:
+                lines = [opening_books.read()]
         except StopIteration:
             return ''
-        return '\n'.join([
+        for path in paths:
-            *(self._format_plugin(name, opts) for name, opts in plugins.items()),
+            lines.append(f'include "../{path.name}"')
-            books_start,
+        return '\n'.join(lines)
-            *(self._format_include(year) for year in years),
+
-        ])
+    def load_string(self, source: str) -> LoadResult:
+        """Load a generated string of Beancount directives
+
+        This method takes a string generated by another Loader method, like
+        fy_range_string, and loads it through Beancount, setting up the
+        environment as necessary to do that.
+        """
+        with workdir(self.opening_root):
+            retval: LoadResult = bc_loader.load_string(source)
+        return retval
 
     def load_fy_range(self,
                       from_fy: Year,
                       to_fy: Optional[Year]=None,
-                      plugins: Optional[PluginsSpec]=None,
     ) -> LoadResult:
-        """Load books for a range of fiscal years
+        """Load books for a range of fiscal years"""
-
+        return self.load_string(self.fy_range_string(from_fy, to_fy))
-        This is a convenience wrapper to call
-        self.fy_range_string(from_fy, to_fy, plugins)
-        and load the result with beancount.loader.load_string.
-        """
-        return bc_loader.load_string(  # type:ignore[no-any-return]
-            self.fy_range_string(from_fy, to_fy, plugins),
-        )

tests/books/2018.beancount

@@ -0,0 +1,3 @@
+2018-04-01 * "2018 donation"
+  Income:Donations  20.18 USD
+  Assets:Checking

tests/books/2019.beancount

@@ -0,0 +1,4 @@
+2019-04-01 * "2019 donation"
+  Income:Donations  20.19 USD
+  Assets:Checking
+

tests/books/2020.beancount

@@ -0,0 +1,3 @@
+2020-04-01 * "2020 donation"
+  Income:Donations  20.20 USD
+  Assets:Checking

tests/books/books/2018.beancount

@@ -0,0 +1,3 @@
+option "title" "Books from 2018"
+plugin "beancount.plugins.auto"
+include "../2018.beancount"

tests/books/books/2019.beancount

@@ -0,0 +1,3 @@
+option "title" "Books from 2019"
+plugin "beancount.plugins.auto"
+include "../2019.beancount"

tests/books/books/2020.beancount

@@ -0,0 +1,3 @@
+option "title" "Books from 2020"
+plugin "beancount.plugins.auto"
+include "../2020.beancount"

tests/test_books_loader.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import hashlib
+import re
 
 from datetime import date
 from pathlib import Path
@@ -25,59 +26,54 @@ from . import testutil
 
 from conservancy_beancount import books
 
-books_path = testutil.test_path('booksroot')
+books_path = testutil.test_path('books')
 
 @pytest.fixture(scope='module')
 def conservancy_loader():
     return books.Loader(books_path, books.FiscalYear(3))
 
-def format_include(year, subdir=''):
+def include_patterns(years, subdir='..'):
-    path = Path(books_path, subdir, f'{year}.beancount')
+    for year in years:
-    return f'include "{path}"'
+        path = Path(subdir, f'{year}.beancount')
-
+        yield rf'^include "{re.escape(str(path))}"$'
-def format_plugin(name, optstring=None):
-    if optstring is None:
-        return f'plugin "{name}"'
-    else:
-        return f'plugin "{name}" "{optstring}"'
-
-def expect_string(years, plugins={'conservancy_beancount.plugin': None}):
-    years = iter(years)
-    year1_s = format_include(next(years), 'books')
-    return '\n'.join([
-        *(format_plugin(name, opts) for name, opts in plugins.items()),
-        year1_s,
-        *(format_include(year) for year in years),
-    ])
 
 @pytest.mark.parametrize('range_start,range_stop,expect_years', [
     (2019, 2020, [2019, 2020]),
     (-1, 2020, [2019, 2020]),
-    (date(2019, 1, 1), date(2020, 6, 1), range(2018, 2021)),
+    (10, 2019, [2019, 2020]),
+    (-10, 2019, [2018, 2019]),
+    (date(2019, 1, 1), date(2020, 6, 1), [2018, 2019, 2020]),
     (-1, date(2020, 2, 1), [2018, 2019]),
 ])
 def test_fy_range_string(conservancy_loader, range_start, range_stop, expect_years):
-    expected = expect_string(expect_years)
+    actual = conservancy_loader.fy_range_string(range_start, range_stop)
-    assert conservancy_loader.fy_range_string(range_start, range_stop) == expected
+    testutil.check_lines_match(actual.splitlines(), [
+        rf'^option "title" "Books from {expect_years[0]}"$',
+        rf'^plugin "beancount\.plugins\.auto"$',
+        *include_patterns(expect_years),
+    ])
 
 @pytest.mark.parametrize('year_offset', range(-3, 1))
-def test_fy_range_string_one_offset(conservancy_loader, year_offset):
+def test_fy_range_string_with_offset(conservancy_loader, year_offset):
-    this_year = date.today().year
+    base_year = 2020
-    expected = expect_string(range(this_year + year_offset, this_year + 1))
+    start_year = max(2018, base_year + year_offset)
-    assert conservancy_loader.fy_range_string(year_offset) == expected
+    expect_years = range(start_year, base_year + 1)
-
+    actual = conservancy_loader.fy_range_string(year_offset, base_year)
-@pytest.mark.parametrize('plugins', [
+    testutil.check_lines_match(actual.splitlines(), include_patterns(expect_years))
-    {},
-    {'conservancy_beancount.plugin': '-all'},
-])
-def test_fy_range_string_plugins_override(conservancy_loader, plugins):
-    expected = expect_string([2019, 2020], plugins)
-    assert conservancy_loader.fy_range_string(2019, 2020, plugins) == expected
 
 def test_fy_range_string_empty_range(conservancy_loader):
     assert conservancy_loader.fy_range_string(2020, 2019) == ''
 
+def test_load_fy_range(conservancy_loader):
+    entries, errors, options_map = conservancy_loader.load_fy_range(2018, 2019)
+    assert not errors
+    narrations = {getattr(entry, 'narration', None) for entry in entries}
+    assert '2018 donation' in narrations
+    assert '2019 donation' in narrations
+    assert '2020 donation' not in narrations
+
 def test_load_fy_range_empty(conservancy_loader):
     entries, errors, options_map = conservancy_loader.load_fy_range(2020, 2019)
+    assert not errors
     assert not entries
     assert options_map.get('input_hash') == hashlib.md5().hexdigest()

tests/test_config.py

@@ -383,12 +383,11 @@ def test_default_fiscal_year_begin():
     assert actual.day == 1
 
 def test_books_loader():
-    books_path = testutil.test_path('bookstest')
+    books_path = testutil.test_path('books')
     config = config_mod.Config()
     config.load_string(f'[Beancount]\nbooks dir = {books_path}\n')
     loader = config.books_loader()
-    expected = 'include "{}"'.format(books_path / 'books/2020.beancount')
+    assert loader.fy_range_string(2020, 2020)
-    assert loader.fy_range_string(0, 2020, {}) == expected
 
 def test_books_loader_without_books():
     assert config_mod.Config().books_loader() is None

tests/test_reports_accrual.py

@@ -273,9 +273,7 @@ def test_consistency_check_when_inconsistent(meta_key, account):
 
 def check_output(output, expect_patterns):
     output.seek(0)
-    for pattern in expect_patterns:
+    testutil.check_lines_match(iter(output), expect_patterns)
-        assert any(re.search(pattern, line) for line in output), \
-            f"{pattern!r} not found in output"
 
 @pytest.mark.parametrize('invoice,expected', [
     ('rt:505/5050', "Zero balance outstanding since 2020-05-05"),

tests/testutil.py

@@ -20,6 +20,7 @@ import re
 
 import beancount.core.amount as bc_amount
 import beancount.core.data as bc_data
+import beancount.loader as bc_loader
 
 from decimal import Decimal
 from pathlib import Path
@@ -33,6 +34,11 @@ FY_MID_DATE = datetime.date(2020, 9, 1)
 PAST_DATE = datetime.date(2000, 1, 1)
 TESTS_DIR = Path(__file__).parent
 
+def check_lines_match(lines, expect_patterns, source='output'):
+    for pattern in expect_patterns:
+        assert any(re.search(pattern, line) for line in lines), \
+            f"{pattern!r} not found in {source}"
+
 def check_post_meta(txn, *expected_meta, default=None):
     assert len(txn.postings) == len(expected_meta)
     for post, expected in zip(txn.postings, expected_meta):
@@ -180,6 +186,8 @@ class TestBooksLoader(books.Loader):
     def fy_range_string(self, from_fy=None, to_fy=None, plugins=None):
         return f'include "{self.source}"'
 
+    load_string = staticmethod(bc_loader.load_string)
+
 
 class TestConfig:
     def __init__(self, *,