import2ledger: First version.

This is a pretty feature-complete version 1.
I don't know why I waited this long to commit anything.

.gitignore

@@ -0,0 +1,6 @@
+build/
+.cache/
+*.egg
+*.egg-info/
+.eggs
+__pycache__/

import2ledger/__main__.py

@@ -0,0 +1,84 @@
+import contextlib
+import logging
+import sys
+
+from . import config, errors, hooks, importers
+
+logger = logging.getLogger('import2ledger')
+
+class FileImporter:
+    def __init__(self, config, stdout):
+        self.config = config
+        self.importers = list(importers.load_all())
+        self.hooks = [hook(config) for hook in hooks.load_all()]
+        self.stdout = stdout
+
+    def import_file(self, in_file):
+        importers = []
+        for importer in self.importers:
+            in_file.seek(0)
+            if importer.can_import(in_file):
+                importers.append(importer)
+        if not importers:
+            raise errors.UserInputFileError("no importers available", in_file.name)
+        with contextlib.ExitStack() as exit_stack:
+            output_path = self.config.get_output_path()
+            if output_path is None:
+                out_file = self.stdout
+            else:
+                out_file = exit_stack.enter_context(output_path.open('a'))
+            for importer in importers:
+                template = self.config.get_template(importer.TEMPLATE_KEY)
+                default_date = self.config.get_default_date()
+                in_file.seek(0)
+                for entry_data in importer(in_file):
+                    for hook in self.hooks:
+                        hook.run(entry_data)
+                    print(template.render(**entry_data), file=out_file, end='')
+
+    def import_path(self, in_path):
+        if in_path is None:
+            raise errors.UserInputFileError("only seekable files are supported", '<stdin>')
+        with in_path.open() as in_file:
+            if not in_file.seekable():
+                raise errors.UserInputFileError("only seekable files are supported", in_path)
+            return self.import_file(in_file)
+
+    def import_paths(self, path_seq):
+        for in_path in path_seq:
+            try:
+                retval = self.import_path(in_path)
+            except (OSError, errors.UserInputError) as error:
+                yield in_path, error
+            else:
+                yield in_path, retval
+
+
+def setup_logger(logger, main_config, stream):
+    formatter = logging.Formatter('%(name)s: %(levelname)s: %(message)s')
+    handler = logging.StreamHandler(stream)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+def main(arglist=None, stdout=sys.stdout, stderr=sys.stderr):
+    try:
+        my_config = config.Configuration(arglist)
+    except errors.UserInputError as error:
+        my_config.error("{}: {!r}".format(error.strerror, error.user_input))
+        return 3
+    setup_logger(logger, my_config, stderr)
+    importer = FileImporter(my_config, stdout)
+    failures = 0
+    for input_path, error in importer.import_paths(my_config.args.input_paths):
+        if error is None:
+            logger.info("%s: imported", input_path)
+        else:
+            logger.warning("%s: failed to import: %s", input_path, error)
+            failures += 1
+    if failures == 0:
+        return 0
+    else:
+        return min(10 + failures, 99)
+
+if __name__ == '__main__':
+    exit(main())

import2ledger/builder.py

@@ -0,0 +1,23 @@
+import decimal
+
+import ledgerlib
+
+class TransactionBuilder:
+    def __init__(self, config):
+        self.config = config
+
+    def _build_one_side(self, entry, amount, tx_provider, tx_type, section):
+        pass
+
+    def build(self, date, payee, amount,
+              from_prov, from_type,
+              to_prov_or_type, to_type=None,
+              section=None):
+        if to_type is None:
+            to_prov = from_prov
+            to_type = to_prov_or_type
+        else:
+            to_prov = to_prov_or_type
+        entry = ledgerlib.Entry(date, payee, date_fmt=self.config.args.date_format)
+        self._build_one_side(entry, -amount, from_prov, from_type, section)
+        self._build_one_side(entry, amount, to_prov, to_type, section)

import2ledger/config.py

@@ -0,0 +1,210 @@
+import argparse
+import configparser
+import contextlib
+import datetime
+import locale
+import logging
+import os.path
+import pathlib
+
+import babel
+import babel.numbers
+from . import errors, template, util
+
+class Configuration:
+    HOME_PATH = pathlib.Path(os.path.expanduser('~'))
+    DEFAULT_CONFIG_PATH = pathlib.Path(HOME_PATH, '.config', 'import2ledger.ini')
+    DEFAULT_ENCODING = locale.getpreferredencoding()
+    LOCALE = babel.core.Locale.default()
+    TODAY = datetime.date.today()
+    CONFIG_DEFAULTS = {
+        'date_format': '%%Y/%%m/%%d',
+        'loglevel': 'WARNING',
+        'output_path': '-',
+        'signed_currencies': ','.join(babel.numbers.get_territory_currencies(
+            LOCALE.territory, start_date=TODAY)),
+        'signed_currency_format': '¤#,##0.###;¤-#,##0.###',
+        'unsigned_currency_format': '#,##0.### ¤¤',
+    }
+
+    def __init__(self, arglist):
+        argparser = self._build_argparser()
+        self.error = argparser.error
+        self.args = argparser.parse_args(arglist)
+
+        if self.args.config_file is None:
+            self.args.config_file = []
+            if self.DEFAULT_CONFIG_PATH.exists():
+                self.args.config_file.append(self.DEFAULT_CONFIG_PATH)
+        self.conffile = self._build_conffile()
+        conffile_paths = [path.as_posix() for path in self.args.config_file]
+        read_files = self.conffile.read(conffile_paths)
+        for expected_path, read_path in zip(conffile_paths, read_files):
+            if read_path != expected_path:
+                self.error("failed to read configuration file {!r}".format(expected_path))
+
+        self.finalize()
+
+    def _build_argparser(self):
+        parser = argparse.ArgumentParser()
+        parser.add_argument(
+            '--config-file', '-C', metavar='PATH', type=pathlib.Path,
+            action='append',
+            help="Path of a configuration file to read",
+        )
+        parser.add_argument(
+            '--use-config', '-c', metavar='SECTION',
+            help="Read settings from this section of the configuration file",
+        )
+        parser.add_argument(
+            'input_paths', metavar='PATH',
+            nargs='+',
+            help="Path to generate Ledger entries from.",
+        )
+
+        out_args = parser.add_argument_group(
+            "default overrides",
+            description="These options take priority over settings in the "
+            "[DEFAULT] section of your config file, but not other sections.",
+        )
+        parser.add_argument(
+            '--loglevel', '-L', metavar='LEVEL',
+            choices=['debug', 'info', 'warning', 'error', 'critical'],
+            help="Log messages at this level and above. Default WARNING.",
+        )
+        out_args.add_argument(
+            '--date', '-d', metavar='DATE',
+            help="Date to use in Ledger entries when the source doesn't "
+            "provide one. Write this in your configured date format. "
+            "Default today.",
+        )
+        out_args.add_argument(
+            '--date-format', '-D', metavar='FORMAT',
+            help="Date format to use in Ledger entries",
+        )
+        out_args.add_argument(
+            '--output-path', '-O', metavar='PATH',
+            help="Path of file to append entries to, or '-' for stdout (default).",
+        )
+        out_args.add_argument(
+            '--signed-currency', '--sign', metavar='CODE',
+            action='append', dest='signed_currencies',
+            help="Currency code to use currency sign for in Ledger entry amounts. "
+            "Can be specified multiple times.",
+        )
+        out_args.add_argument(
+            '--signed-currency-format', '--sign-format', '-S', metavar='FORMAT',
+            help="Unicode number pattern to use for signed currencies in Ledger entry amounts",
+        )
+        out_args.add_argument(
+            '--unsigned-currency-format', '--unsign-format', '-U', metavar='FORMAT',
+            help="Unicode number pattern to use for unsigned currencies in Ledger entry amounts",
+        )
+
+        return parser
+
+    def _build_conffile(self):
+        return configparser.ConfigParser(
+            comment_prefixes='#',
+            defaults=self.CONFIG_DEFAULTS,
+        )
+
+    def _s_to_path(self, s):
+        return None if s == '-' else pathlib.Path(s)
+
+    def _strpdate(self, date_s, date_fmt):
+        try:
+            return util.strpdate(date_s, date_fmt)
+        except ValueError as error:
+            raise errors.UserInputConfigurationError(error.args[0], date_s)
+
+    def _parse_section_date(self, section_name, default=TODAY):
+        section = self.conffile[section_name]
+        try:
+            return self._strpdate(section['date'], section['date_format'])
+        except KeyError:
+            return default
+
+    def finalize(self):
+        if self.args.use_config is None:
+            self.args.use_config = self.conffile.default_section
+        elif not self.conffile.has_section(self.args.use_config):
+            self.error("section {!r} not found in config file".format(self.args.use_config))
+        self.args.input_paths = [self._s_to_path(s) for s in self.args.input_paths]
+
+        defaults = self.conffile[self.conffile.default_section]
+        for key in self.CONFIG_DEFAULTS:
+            value = getattr(self.args, key)
+            if value is None:
+                pass
+            elif key == 'signed_currencies':
+                defaults[key] = ','.join(value)
+            else:
+                defaults[key] = value
+
+        # We parse all the dates now to make sure they're valid.
+        if self.args.date is not None:
+            default_date = self._strpdate(self.args.date, defaults['date_format'])
+        elif 'date' in defaults:
+            default_date = self._strpdate(defaults['date'], defaults['date_format'])
+        else:
+            default_date = self.TODAY
+
+        self.dates = {secname: self._parse_section_date(secname, default_date)
+                      for secname in self.conffile}
+        self.dates[self.conffile.default_section] = default_date
+
+    @contextlib.contextmanager
+    def from_section(self, section_name):
+        prev_section = self.args.use_config
+        self.args.use_config = section_name
+        try:
+            yield self
+        finally:
+            self.args.use_config = prev_section
+
+    def _get_section(self, section_name):
+        if section_name is None:
+            section_name = self.args.use_config
+        return self.conffile[section_name]
+
+    def get_default_date(self, section_name=None):
+        if section_name is None:
+            section_name = self.args.use_config
+        try:
+            return self.dates[section_name]
+        except KeyError:
+            return self.dates[self.conffile.default_section]
+
+    def get_loglevel(self, section_name=None):
+        section_config = self._get_section(section_name)
+        level_name = section_config['loglevel']
+        try:
+            return getattr(logging, level_name.upper())
+        except AttributeError:
+            raise errors.UserInputConfigurationError("not a valid loglevel", level_name)
+
+    def get_output_path(self, section_name=None):
+        section_config = self._get_section(section_name)
+        return self._s_to_path(section_config['output_path'])
+
+    def get_template(self, config_key, section_name=None, factory=template.Template):
+        section_config = self._get_section(section_name)
+        try:
+            template_s = section_config[config_key]
+        except KeyError:
+            raise errors.UserInputConfigurationError(
+                "template not defined in [{}]".format(section_name or self.args.use_config),
+                config_key,
+            )
+        return factory(
+            template_s,
+            date_fmt=section_config['date_format'],
+            signed_currencies=[code.strip().upper() for code in section_config['signed_currencies'].split(',')],
+            signed_currency_fmt=section_config['signed_currency_format'],
+            unsigned_currency_fmt=section_config['unsigned_currency_format'],
+        )
+
+    def setup_logger(self, logger, section_name=None):
+        logger.setLevel(self.get_loglevel(section_name))
+

import2ledger/errors.py

@@ -0,0 +1,15 @@
+class UserInputError(Exception):
+    def __init__(self, strerror, user_input):
+        super().__init__(strerror, user_input)
+        self.strerror = strerror
+        self.user_input = user_input
+
+
+class UserInputConfigurationError(UserInputError):
+    pass
+
+
+class UserInputFileError(UserInputError):
+    def __init__(self, strerror, path):
+        super().__init__(strerror, path)
+        self.path = path

import2ledger/hooks/__init__.py

@@ -0,0 +1,6 @@
+import operator
+
+from .. import util
+
+def load_all():
+    return util.submodule_items_named(__file__, operator.methodcaller('endswith', 'Hook'))

import2ledger/hooks/add_entity.py

@@ -0,0 +1,42 @@
+import re
+import unicodedata
+
+class AddEntityHook:
+    NONASCII_RE = re.compile(r'[^-A-Za-z0-9]')
+    NONALNUM_RE = re.compile(r'[^-\w]')
+    OPEN_PARENS = ['\\(', '\\[', '\\{']
+    CLOSE_PARENS = ['\\)', '\\]', '\\}']
+    NO_PARENS = '[^{}]*'.format(''.join(OPEN_PARENS + CLOSE_PARENS))
+
+    def __init__(self, config):
+        pass
+
+    def _remove_parens(self, s):
+        last_s = None
+        while s != last_s:
+            last_s = s
+            for open_c, close_c in zip(self.OPEN_PARENS, self.CLOSE_PARENS):
+                s = re.sub(open_c + self.NO_PARENS + close_c, '', s)
+        return s if s else last_s
+
+    def _entity_parts(self, s, trim_re):
+        for word in s.split():
+            word = unicodedata.normalize('NFKD', word)
+            word = trim_re.sub('', word)
+            if word:
+                yield word
+
+    def _str2entity(self, s, trim_re):
+        parts = list(self._entity_parts(s, trim_re))
+        if not parts:
+            return ''
+        parts.insert(0, parts.pop())
+        return '-'.join(parts)
+
+    def run(self, data):
+        if ('payee' in data) and ('entity' not in data):
+            payee = self._remove_parens(data['payee'])
+            entity = self._str2entity(payee, self.NONASCII_RE)
+            if not entity:
+                entity = self._str2entity(payee, self.NONALNUM_RE)
+            data['entity'] = entity

import2ledger/hooks/default_date.py

@@ -0,0 +1,7 @@
+class DefaultDateHook:
+    def __init__(self, config):
+        self.config = config
+
+    def run(self, entry_data):
+        if 'date' not in entry_data:
+            entry_data['date'] = self.config.get_default_date()

import2ledger/importers/__init__.py

@@ -0,0 +1,6 @@
+import operator
+
+from .. import util
+
+def load_all():
+    return util.submodule_items_named(__file__, operator.methodcaller('endswith', 'Importer'))

import2ledger/importers/patreon.py

@@ -0,0 +1,72 @@
+import csv
+import datetime
+import pathlib
+import re
+
+from .. import util
+
+class ImporterBase:
+    @classmethod
+    def can_import(cls, input_file):
+        in_csv = csv.reader(input_file)
+        fields = next(iter(in_csv), [])
+        return cls.NEEDED_FIELDS.issubset(fields)
+
+    def __init__(self, input_file):
+        self.in_csv = csv.DictReader(input_file)
+        self.start_data = {'currency': 'USD'}
+
+    def __iter__(self):
+        for row in self.in_csv:
+            row_data = self._read_row(row)
+            if row_data is not None:
+                retval = self.start_data.copy()
+                retval.update(row_data)
+                yield retval
+
+
+class IncomeImporter(ImporterBase):
+    NEEDED_FIELDS = frozenset([
+        'FirstName',
+        'LastName',
+        'Pledge',
+        'Status',
+    ])
+    TEMPLATE_KEY = 'template patreon income'
+
+    def __init__(self, input_file):
+        super().__init__(input_file)
+        match = re.search(r'(?:\b|_)(\d{4}-\d{2}-\d{2})(?:\b|_)',
+                          pathlib.Path(input_file.name).name)
+        if match:
+            self.start_data['date'] = util.strpdate(match.group(1), '%Y-%m-%d')
+
+    def _read_row(self, row):
+        if row['Status'] != 'Processed':
+            return None
+        else:
+            return {
+                'amount': row['Pledge'],
+                'payee': '{0[FirstName]} {0[LastName]}'.format(row),
+            }
+
+
+class FeeImporterBase(ImporterBase):
+    def _read_row(self, row):
+        return {
+            'amount': row[self.AMOUNT_FIELD].lstrip('$'),
+            'date': util.strpdate(row['Month'], '%Y-%m'),
+            'payee': "Patreon",
+        }
+
+
+class PatreonFeeImporter(FeeImporterBase):
+    AMOUNT_FIELD = 'Patreon Fee'
+    NEEDED_FIELDS = frozenset(['Month', AMOUNT_FIELD])
+    TEMPLATE_KEY = 'template patreon svcfees'
+
+
+class CardFeeImporter(FeeImporterBase):
+    AMOUNT_FIELD = 'Processing Fees'
+    NEEDED_FIELDS = frozenset(['Month', AMOUNT_FIELD])
+    TEMPLATE_KEY = 'template patreon cardfees'

import2ledger/template.py

@@ -0,0 +1,98 @@
+import collections
+import datetime
+import decimal
+
+import babel.numbers
+
+class AccountSplitter:
+    Split = collections.namedtuple('Split', ('account', 'percentage'))
+
+    def __init__(self, sign, signed_currencies, signed_currency_fmt, unsigned_currency_fmt):
+        self.splits = []
+        self.sign = sign
+        self.signed_currency_fmt = signed_currency_fmt
+        self.unsigned_currency_fmt = unsigned_currency_fmt
+        self.signed_currencies = set(signed_currencies)
+        self.unsplit = sign * 100
+
+    def add(self, account, percentage_s):
+        int_pctage = decimal.Decimal(percentage_s.rstrip('%'))
+        self.unsplit -= int_pctage
+        percentage = None if self.unsplit == 0 else (int_pctage / 100)
+        self.splits.append(self.Split(account, percentage))
+
+    def render_next(self, template_vars):
+        try:
+            account, percentage = next(self._split_iter)
+        except (AttributeError, StopIteration):
+            self._split_iter = iter(self.splits)
+            account, percentage = next(self._split_iter)
+            self._remainder = self.sign * template_vars['amount']
+            self._currency = template_vars['currency']
+            if self._currency in self.signed_currencies:
+                self._amt_fmt = self.signed_currency_fmt
+            else:
+                self._amt_fmt = self.unsigned_currency_fmt
+        if percentage is None:
+            amount = self._remainder
+        else:
+            amount = decimal.Decimal(babel.numbers.format_currency(
+                template_vars['amount'] * percentage, self._currency, '###0.###'))
+        self._remainder -= amount
+        amt_s = babel.numbers.format_currency(amount, self._currency, self._amt_fmt)
+        return '    {:45}  {:>19}\n'.format(account, amt_s)
+
+
+class Template:
+    DATE_FMT = '%Y/%m/%d'
+    SIGNED_CURRENCY_FMT = '¤#,##0.###;¤-#,##0.###'
+    UNSIGNED_CURRENCY_FMT = '#,##0.### ¤¤'
+
+    def __init__(self, template_s, signed_currencies=frozenset(),
+                 date_fmt=DATE_FMT,
+                 signed_currency_fmt=SIGNED_CURRENCY_FMT,
+                 unsigned_currency_fmt=UNSIGNED_CURRENCY_FMT):
+        self.date_fmt = date_fmt
+        self.pos_splits = AccountSplitter(
+            1, signed_currencies, signed_currency_fmt, unsigned_currency_fmt)
+        self.neg_splits = AccountSplitter(
+            -1, signed_currencies, signed_currency_fmt, unsigned_currency_fmt)
+
+        lines = [s.lstrip() for s in template_s.splitlines(True)]
+        for index, line in enumerate(lines):
+            if line:
+                break
+        del lines[:index]
+        self.format_funcs = [
+            '\n'.format_map,
+            '{date} {payee}\n'.format_map,
+        ]
+        start_index = 0
+        for index, line in enumerate(lines):
+            if line[0].isalpha():
+                if start_index < index:
+                    self._add_str_func(lines[start_index:index])
+                account, percentage_s = line.rsplit(None, 1)
+                splitter = self.neg_splits if percentage_s.startswith('-') else self.pos_splits
+                splitter.add(account, percentage_s)
+                self.format_funcs.append(splitter.render_next)
+                start_index = index + 1
+        if start_index <= index:
+            self._add_str_func(lines[start_index:])
+            if not line.endswith('\n'):
+                self.format_funcs.append('\n'.format_map)
+
+    def _add_str_func(self, str_seq):
+        str_flat = ''.join('      ' + s for s in str_seq)
+        self.format_funcs.append(str_flat.format_map)
+
+    def render(self, payee, amount, currency, date=None, **template_vars):
+        if date is None:
+            date = datetime.date.today()
+        template_vars.update(
+            date=date.strftime(self.date_fmt),
+            payee=payee,
+            amount=decimal.Decimal(amount),
+            currency=currency,
+        )
+        return ''.join(f(template_vars) for f in self.format_funcs)

import2ledger/util.py

@@ -0,0 +1,34 @@
+import datetime
+import importlib
+import logging
+import pathlib
+
+logger = logging.getLogger('import2ledger')
+
+def load_modules(src_dir_path):
+    rel_path = src_dir_path.relative_to(pathlib.Path(__file__).parent)
+    import_prefix = 'import2ledger.{}.'.format('.'.join(rel_path.parts))
+    for py_path in src_dir_path.glob('*.py'):
+        mod_name = py_path.name[:-3]
+        if mod_name.startswith(('.', '_')):
+            continue
+        try:
+            module = importlib.import_module(import_prefix + mod_name)
+        except ImportError as error:
+            logger.info("failed to import %s: %s", py_path, error,
+                        exc_info=logger.isEnabledFor(logging.DEBUG))
+        else:
+            yield module
+
+def module_contents(module):
+    for name in dir(module):
+        yield name, getattr(module, name)
+
+def submodule_items_named(file_path, name_test):
+    for module in load_modules(pathlib.Path(file_path).parent):
+        for name, item in module_contents(module):
+            if name_test(name):
+                yield item
+
+def strpdate(date_s, date_fmt):
+    return datetime.datetime.strptime(date_s, date_fmt).date()

setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

setup.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+from setuptools import setup
+
+setup(
+    name='import2ledger',
+    description="Import different sources of financial data to Ledger",
+    version='0.1',
+    author='Brett Smith',
+    author_email='brettcsmith@brettcsmith.org',
+    license='GNU AGPLv3+',
+
+    install_requires=['babel'],
+    setup_requires=['pytest-runner'],
+    tests_require=['pytest', 'PyYAML'],
+
+    packages=['import2ledger'],
+    entry_points={},
+)

tests/__init__.py

@@ -0,0 +1,7 @@
+import pathlib
+import re
+
+DATA_DIR = pathlib.Path(__file__).with_name('data')
+
+def normalize_whitespace(s):
+    return re.sub(r'(\t| {3,})', '  ', s)

tests/data/PatreonEarnings.csv

@@ -0,0 +1,3 @@
+Month,Processed Pledges,Creator's Share,Processing Fees,Patreon Fee,Creator's Share (%),Processing Fees (%),Patreon Fee (%)
+2017-09,"$1234.50","$1,120.30",$52.47,$61.73,90.75%,4.25%,5.0%
+2017-10,"$2340.50","$2,124.00",$99.47,$117.03,90.75%,4.25%,5.0%

tests/data/PatreonPatronReport_2017-09-01.csv

@@ -0,0 +1,6 @@
+FirstName,LastName,Email,Pledge,Lifetime,Status,Twitter,Street,City,State,Zip,Country,Start,MaxAmount,Complete
+100 + Reward,Description You donate a lot of money!,,,,,,,,,,,,,
+Alex,Jones,alex@example.org,150,300,Processed,,,,,,,2017-08-11 11:28:06.166065,200,0
+5 + Reward,Description You’re nice!,,,,,,,,,,,,,
+Brett,Smith,brett@example.org,10,30,Declined,Brett20XX,,,,,,2017-08-10 13:24:15.955782,10,0
+Dakota,Doe,ddoe@example.org,12,48,Processed,,,,,,,2017-08-10 12:58:31.919341,12,0

tests/data/imports.yml

@@ -0,0 +1,35 @@
+- source: PatreonPatronReport_2017-09-01.csv
+  importer: patreon.IncomeImporter
+  expect:
+    - payee: Alex Jones
+      date: [2017, 9, 1]
+      amount: "150"
+      currency: USD
+    - payee: Dakota Doe
+      date: [2017, 9, 1]
+      amount: "12"
+      currency: USD
+
+- source: PatreonEarnings.csv
+  importer: patreon.PatreonFeeImporter
+  expect:
+    - payee: Patreon
+      date: [2017, 9, 1]
+      amount: "61.73"
+      currency: USD
+    - payee: Patreon
+      date: [2017, 10, 1]
+      amount: "117.03"
+      currency: USD
+
+- source: PatreonEarnings.csv
+  importer: patreon.CardFeeImporter
+  expect:
+    - payee: Patreon
+      date: [2017, 9, 1]
+      amount: "52.47"
+      currency: USD
+    - payee: Patreon
+      date: [2017, 10, 1]
+      amount: "99.47"
+      currency: USD

tests/data/templates.ini

@@ -0,0 +1,22 @@
+[Simplest]
+template =
+ Accrued:Accounts Receivable  100%%
+ Income:Donations  -100%%
+
+[FiftyFifty]
+template =
+ Accrued:Accounts Receivable  100%%
+ Income:Donations  -50%%
+ Income:Sales  -50%%
+
+[Complex]
+template =
+ ;Tag: Value
+ ;TransactionID: {txid}
+ Accrued:Accounts Receivable  100%%
+ ;Entity: Supplier
+ Income:Donations:Specific  -95.5%%
+ ;Program: Specific
+ ;Entity: {entity}
+ Income:Donations:General  -4.5%%
+ ;Entity: {entity}

tests/data/test_config.ini

@@ -0,0 +1,24 @@
+[DEFAULT]
+date_format = %%Y-%%m-%%d
+signed_currencies = EUR
+
+[Templates]
+output_path = Template.output
+one =
+ Accrued:Accounts Receivable  100%%
+ Income:Donations  -100%%
+two =
+ ;Tag1: {value}
+ Income:Donations  -100%%
+ ;IncomeTag: Donations
+ Expenses:Fundraising  5%%
+ ;ExpenseTag: Fundraising
+ Accrued:Accounts Receivable  95%%
+ ;AccrualTag: Receivable
+
+[Date]
+date_format = %%Y|%%m|%%d
+date = 2017|10|08
+
+[Bad Loglevel]
+loglevel = wraning

tests/data/test_main.ini

@@ -0,0 +1,12 @@
+[DEFAULT]
+default_date = 2016/04/04
+loglevel = critical
+signed_currencies = USD
+
+[One]
+template patreon cardfees =
+ Accrued:Accounts Receivable  -100%%
+ Expenses:Fees:Credit Card  100%%
+template patreon svcfees =
+ Accrued:Accounts Receivable  -100%%
+ Expenses:Fundraising  100%%

tests/data/test_main_fees_import.ledger

@@ -0,0 +1,15 @@
+2017/09/01 Patreon
+  Accrued:Accounts Receivable  $-52.47
+  Expenses:Fees:Credit Card  $52.47
+
+2017/09/01 Patreon
+  Accrued:Accounts Receivable  $-61.73
+  Expenses:Fundraising  $61.73
+
+2017/10/01 Patreon
+  Accrued:Accounts Receivable  $-99.47
+  Expenses:Fees:Credit Card  $99.47
+
+2017/10/01 Patreon
+  Accrued:Accounts Receivable  $-117.03
+  Expenses:Fundraising  $117.03

tests/test_config.py

@@ -0,0 +1,109 @@
+import contextlib
+import datetime
+import logging
+import os
+import pathlib
+
+START_DATE = datetime.date.today()
+
+from unittest import mock
+
+import pytest
+from import2ledger import config, errors
+
+from . import DATA_DIR
+
+def config_from_file(path, arglist=[]):
+    path = pathlib.Path(path)
+    if not path.is_absolute():
+        path = DATA_DIR / path
+    arglist = ['-C', path.as_posix(), *arglist, os.devnull]
+    return config.Configuration(arglist)
+
+def test_defaults():
+    config = config_from_file('test_config.ini', ['--sign', 'GBP', '-O', 'out_arg'])
+    factory = mock.Mock(name='Template')
+    template = config.get_template('one', 'Templates', factory)
+    assert factory.called
+    kwargs = factory.call_args[1]
+    assert list(kwargs.pop('signed_currencies', '')) == ['GBP']
+    assert kwargs == {
+        'date_fmt': '%Y-%m-%d',
+        'signed_currency_fmt': kwargs['signed_currency_fmt'],
+        'unsigned_currency_fmt': kwargs['unsigned_currency_fmt'],
+    }
+
+def test_template_parsing():
+    config = config_from_file('test_config.ini')
+    factory = mock.Mock(name='Template')
+    template = config.get_template('two', 'Templates', factory)
+    try:
+        tmpl_s = factory.call_args[0][0]
+    except IndexError as error:
+        assert False, error
+    assert "\n;Tag1: {value}\n" in tmpl_s
+    assert "\nIncome:Donations  -100%\n" in tmpl_s
+    assert "\n;IncomeTag: Donations\n" in tmpl_s
+
+@pytest.mark.parametrize('arg_s', [None, '-', 'output.ledger'])
+def test_output_path(arg_s):
+    arglist = [] if arg_s is None else ['-O', arg_s]
+    config = config_from_file(os.devnull, arglist)
+    output_path = config.get_output_path()
+    if (arg_s is None) or (arg_s == '-'):
+        assert output_path is None
+    else:
+        assert output_path == pathlib.Path(arg_s)
+
+def test_output_path_from_section():
+    expected_path = pathlib.Path('Template.output')
+    config = config_from_file('test_config.ini', ['-O', 'output.ledger'])
+    assert config.get_output_path('Templates') == expected_path
+    assert config.get_output_path() != expected_path
+    with config.from_section('Templates'):
+        assert config.get_output_path() == expected_path
+
+@pytest.mark.parametrize('arglist,expect_date', [
+    ([], None),
+    (['-d', '2017-10-12'], datetime.date(2017, 10, 12)),
+    (['-c', 'Date'], datetime.date(2017, 10, 8)),
+])
+def test_default_date(arglist, expect_date):
+    config = config_from_file('test_config.ini', arglist)
+    default_date = config.get_default_date()
+    if expect_date is None:
+        assert START_DATE <= default_date <= datetime.date.today()
+    else:
+        assert default_date == expect_date
+    assert config.get_default_date('Date') == datetime.date(2017, 10, 8)
+
+@pytest.mark.parametrize('level_s,expect_level', [
+    (s, getattr(logging, s.upper()))
+    for s in ['critical', 'debug', 'error', 'info', 'warning']
+])
+def test_loglevel(level_s, expect_level):
+    config = config_from_file(os.devnull, ['--loglevel', level_s])
+    assert config.get_loglevel() == expect_level
+
+@contextlib.contextmanager
+def bad_config(expect_input):
+    with pytest.raises(errors.UserInputConfigurationError) as exc_info:
+        yield exc_info
+    assert exc_info.value.user_input == expect_input
+
+def test_bad_default_date():
+    date_s = '2017-10-06'
+    with bad_config(date_s):
+        config = config_from_file(os.devnull, ['--date', date_s])
+        config.get_default_date()
+
+def test_bad_loglevel():
+    with bad_config('wraning'):
+        config = config_from_file('test_config.ini', ['-c', 'Bad Loglevel'])
+        config.get_loglevel()
+
+def test_undefined_template():
+    template_name = 'template nonexistent'
+    config = config_from_file(os.devnull)
+    with bad_config(template_name):
+        config.get_template(template_name)

tests/test_hooks.py

@@ -0,0 +1,66 @@
+import argparse
+import datetime
+import itertools
+
+import pytest
+
+from import2ledger import hooks
+from import2ledger.hooks import add_entity, default_date
+
+def test_load_all():
+    all_hooks = list(hooks.load_all())
+    assert add_entity.AddEntityHook in all_hooks
+
+@pytest.mark.parametrize('payee,expected', [
+    ('Alex Smith', 'Smith-Alex'),
+    ('Dakota D.  Doe', 'Doe-Dakota-D'),
+    ('Björk', 'Bjork'),
+    ('Fran Doe-Smith', 'Doe-Smith-Fran'),
+    ('Alex(Nickname) Smith', 'Smith-Alex'),
+    ('稲荷', '稲荷'),
+])
+def test_add_entity(payee, expected):
+    data = {'payee': payee}
+    hook = add_entity.AddEntityHook(argparse.Namespace())
+    hook.run(data)
+    assert data['entity'] == expected
+
+
+class DefaultDateConfig:
+    ONE_DAY = datetime.timedelta(days=1)
+
+    def __init__(self, start_date=None):
+        if start_date is None:
+            start_date = datetime.date(2016, 3, 5)
+        self.date = start_date - self.ONE_DAY
+
+    def get_default_date(self, section_name=None):
+        self.date += self.ONE_DAY
+        return self.date
+
+
+class TestDefaultDate:
+    def test_simple_case(self):
+        expect_date = datetime.date(2016, 2, 4)
+        config = DefaultDateConfig(expect_date)
+        data = {}
+        hook = default_date.DefaultDateHook(config)
+        hook.run(data)
+        assert data['date'] == expect_date
+
+    def test_no_caching(self):
+        config = DefaultDateConfig()
+        hook = default_date.DefaultDateHook(config)
+        d1 = {}
+        d2 = {}
+        hook.run(d1)
+        hook.run(d2)
+        assert d1['date'] != d2['date']
+
+    def test_no_override(self):
+        expect_date = datetime.date(2016, 2, 6)
+        config = DefaultDateConfig(expect_date + datetime.timedelta(days=300))
+        hook = default_date.DefaultDateHook(config)
+        data = {'date': expect_date}
+        hook.run(data)
+        assert data['date'] is expect_date

tests/test_import_patreon_income.py

@@ -0,0 +1,2 @@
+import datetime
+

tests/test_importers.py

@@ -0,0 +1,47 @@
+import datetime
+import importlib
+import itertools
+import pathlib
+
+import pytest
+import yaml
+from import2ledger import importers
+
+from . import DATA_DIR
+
+class TestImporters:
+    with pathlib.Path(DATA_DIR, 'imports.yml').open() as yaml_file:
+        test_data = yaml.load(yaml_file)
+    for test in test_data:
+        test['source'] = DATA_DIR / test['source']
+
+        module_name, class_name = test['importer'].rsplit('.', 1)
+        module = importlib.import_module('.' + module_name, 'import2ledger.importers')
+        test['importer'] = getattr(module, class_name)
+
+        for expect_result in test['expect']:
+            try:
+                expect_result['date'] = datetime.date(*expect_result['date'])
+            except KeyError:
+                pass
+
+    @pytest.mark.parametrize('source_path,importer', [
+        (t['source'], t['importer']) for t in test_data
+    ])
+    def test_can_import(self, source_path, importer):
+        with source_path.open() as source_file:
+            assert importer.can_import(source_file)
+
+    @pytest.mark.parametrize('source_path,import_class,expect_results', [
+        (t['source'], t['importer'], t['expect']) for t in test_data
+    ])
+    def test_import(self, source_path, import_class, expect_results):
+        with source_path.open() as source_file:
+            importer = import_class(source_file)
+            for actual, expected in itertools.zip_longest(importer, expect_results):
+                assert actual == expected
+
+    def test_loader(self):
+        all_importers = list(importers.load_all())
+        for test in self.test_data:
+            assert test['importer'] in all_importers

tests/test_main.py

@@ -0,0 +1,52 @@
+import io
+import pathlib
+
+import pytest
+from . import DATA_DIR, normalize_whitespace
+
+from import2ledger import __main__ as i2lmain
+
+ARGLIST = [
+    '-C', (DATA_DIR / 'test_main.ini').as_posix(),
+]
+
+def run_main(arglist):
+    stdout = io.StringIO()
+    stderr = io.StringIO()
+    exitcode = i2lmain.main(arglist, stdout, stderr)
+    stdout.seek(0)
+    stderr.seek(0)
+    return exitcode, stdout, stderr
+
+def iter_entries(in_file):
+    lines = []
+    for line in in_file:
+        if line == '\n':
+            if lines:
+                yield ''.join(lines)
+            lines = []
+        else:
+            lines.append(line)
+    if lines:
+        yield ''.join(lines)
+
+def entries2set(in_file):
+    return set(normalize_whitespace(e) for e in iter_entries(in_file))
+
+def expected_entries(path):
+    path = pathlib.Path(path)
+    if not path.is_absolute():
+        path = DATA_DIR / path
+    with path.open() as in_file:
+        return entries2set(in_file)
+
+def test_fees_import():
+    arglist = ARGLIST + [
+        '-c', 'One',
+        pathlib.Path(DATA_DIR, 'PatreonEarnings.csv').as_posix(),
+    ]
+    exitcode, stdout, _ = run_main(arglist)
+    print(_.getvalue())
+    assert exitcode == 0
+    actual = entries2set(stdout)
+    assert actual == expected_entries('test_main_fees_import.ledger')

tests/test_templates.py

@@ -0,0 +1,75 @@
+import configparser
+import datetime
+import decimal
+import pathlib
+
+import pytest
+from import2ledger import template
+
+from . import DATA_DIR, normalize_whitespace
+
+DATE = datetime.date(2015, 3, 14)
+
+config = configparser.ConfigParser(comment_prefixes='#')
+with pathlib.Path(DATA_DIR, 'templates.ini').open() as conffile:
+    config.read_file(conffile)
+
+def template_from(section_name, *args, **kwargs):
+    return template.Template(config[section_name]['template'], *args, **kwargs)
+
+def assert_easy_render(tmpl, entity, amount, currency, expect_date, expect_amt):
+    rendered = tmpl.render(entity, decimal.Decimal(amount), currency, DATE)
+    print(repr(rendered))
+    lines = [normalize_whitespace(s) for s in rendered.splitlines()]
+    assert lines == [
+        "",
+        "{} {}".format(expect_date, entity),
+        "  Accrued:Accounts Receivable  " + expect_amt,
+        "  Income:Donations  " + expect_amt.replace(amount, "-" + amount),
+    ]
+
+def test_easy_template():
+    tmpl = template_from('Simplest')
+    assert_easy_render(tmpl, 'JJ', '5.99', 'CAD', '2015/03/14', '5.99 CAD')
+
+def test_date_formatting():
+    tmpl = template_from('Simplest', date_fmt='%Y-%m-%d')
+    assert_easy_render(tmpl, 'KK', '6.99', 'CAD', '2015-03-14', '6.99 CAD')
+
+def test_currency_formatting():
+    tmpl = template_from('Simplest', signed_currencies=['USD'])
+    assert_easy_render(tmpl, 'CC', '7.99', 'USD', '2015/03/14', '$7.99')
+
+def test_complex_template():
+    template_vars = {
+        'entity': 'T-T',
+        'txid': 'ABCDEF',
+    }
+    tmpl = template_from('Complex', date_fmt='%Y-%m-%d', signed_currencies=['USD'])
+    rendered = tmpl.render('TT', decimal.Decimal('125.50'), 'USD', DATE, **template_vars)
+    lines = [normalize_whitespace(s) for s in rendered.splitlines()]
+    assert lines == [
+        "",
+        "2015-03-14 TT",
+        "  ;Tag: Value",
+        "  ;TransactionID: ABCDEF",
+        "  Accrued:Accounts Receivable  $125.50",
+        "  ;Entity: Supplier",
+        "  Income:Donations:Specific  $-119.85",
+        "  ;Program: Specific",
+        "  ;Entity: T-T",
+        "  Income:Donations:General  $-5.65",
+        "  ;Entity: T-T",
+    ]
+
+def test_balancing():
+    tmpl = template_from('FiftyFifty')
+    rendered = tmpl.render('FF', decimal.Decimal('1.01'), 'USD', DATE)
+    lines = [normalize_whitespace(s) for s in rendered.splitlines()]
+    assert lines == [
+        "",
+        "2015/03/14 FF",
+        "  Accrued:Accounts Receivable  1.01 USD",
+        "  Income:Donations  -0.50 USD",
+        "  Income:Sales  -0.51 USD",
+    ]