From 5c73c40bccfeb6d088df52814c14bd3ff2560935 Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Sun, 22 Oct 2017 13:38:53 -0400 Subject: [PATCH] import2ledger: First version. This is a pretty feature-complete version 1. I don't know why I waited this long to commit anything. --- .gitignore | 6 + import2ledger/__init__.py | 0 import2ledger/__main__.py | 84 +++++++ import2ledger/builder.py | 23 ++ import2ledger/config.py | 210 ++++++++++++++++++ import2ledger/errors.py | 15 ++ import2ledger/hooks/__init__.py | 6 + import2ledger/hooks/add_entity.py | 42 ++++ import2ledger/hooks/default_date.py | 7 + import2ledger/importers/__init__.py | 6 + import2ledger/importers/patreon.py | 72 ++++++ import2ledger/template.py | 98 ++++++++ import2ledger/util.py | 34 +++ setup.cfg | 2 + setup.py | 19 ++ tests/__init__.py | 7 + tests/data/PatreonEarnings.csv | 3 + tests/data/PatreonPatronReport_2017-09-01.csv | 6 + tests/data/imports.yml | 35 +++ tests/data/templates.ini | 22 ++ tests/data/test_config.ini | 24 ++ tests/data/test_main.ini | 12 + tests/data/test_main_fees_import.ledger | 15 ++ tests/test_config.py | 109 +++++++++ tests/test_hooks.py | 66 ++++++ tests/test_import_patreon_income.py | 2 + tests/test_importers.py | 47 ++++ tests/test_main.py | 52 +++++ tests/test_templates.py | 75 +++++++ 29 files changed, 1099 insertions(+) create mode 100644 .gitignore create mode 100644 import2ledger/__init__.py create mode 100644 import2ledger/__main__.py create mode 100644 import2ledger/builder.py create mode 100644 import2ledger/config.py create mode 100644 import2ledger/errors.py create mode 100644 import2ledger/hooks/__init__.py create mode 100644 import2ledger/hooks/add_entity.py create mode 100644 import2ledger/hooks/default_date.py create mode 100644 import2ledger/importers/__init__.py create mode 100644 import2ledger/importers/patreon.py create mode 100644 import2ledger/template.py create mode 100644 import2ledger/util.py create mode 100644 setup.cfg create mode 100755 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/data/PatreonEarnings.csv create mode 100644 tests/data/PatreonPatronReport_2017-09-01.csv create mode 100644 tests/data/imports.yml create mode 100644 tests/data/templates.ini create mode 100644 tests/data/test_config.ini create mode 100644 tests/data/test_main.ini create mode 100644 tests/data/test_main_fees_import.ledger create mode 100644 tests/test_config.py create mode 100644 tests/test_hooks.py create mode 100644 tests/test_import_patreon_income.py create mode 100644 tests/test_importers.py create mode 100644 tests/test_main.py create mode 100644 tests/test_templates.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f7ddf50 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +build/ +.cache/ +*.egg +*.egg-info/ +.eggs +__pycache__/ diff --git a/import2ledger/__init__.py b/import2ledger/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/import2ledger/__main__.py b/import2ledger/__main__.py new file mode 100644 index 0000000..c1184e2 --- /dev/null +++ b/import2ledger/__main__.py @@ -0,0 +1,84 @@ +import contextlib +import logging +import sys + +from . import config, errors, hooks, importers + +logger = logging.getLogger('import2ledger') + +class FileImporter: + def __init__(self, config, stdout): + self.config = config + self.importers = list(importers.load_all()) + self.hooks = [hook(config) for hook in hooks.load_all()] + self.stdout = stdout + + def import_file(self, in_file): + importers = [] + for importer in self.importers: + in_file.seek(0) + if importer.can_import(in_file): + importers.append(importer) + if not importers: + raise errors.UserInputFileError("no importers available", in_file.name) + with contextlib.ExitStack() as exit_stack: + output_path = self.config.get_output_path() + if output_path is None: + out_file = self.stdout + else: + out_file = exit_stack.enter_context(output_path.open('a')) + for importer in importers: + template = self.config.get_template(importer.TEMPLATE_KEY) + default_date = self.config.get_default_date() + in_file.seek(0) + for entry_data in importer(in_file): + for hook in self.hooks: + hook.run(entry_data) + print(template.render(**entry_data), file=out_file, end='') + + def import_path(self, in_path): + if in_path is None: + raise errors.UserInputFileError("only seekable files are supported", '') + with in_path.open() as in_file: + if not in_file.seekable(): + raise errors.UserInputFileError("only seekable files are supported", in_path) + return self.import_file(in_file) + + def import_paths(self, path_seq): + for in_path in path_seq: + try: + retval = self.import_path(in_path) + except (OSError, errors.UserInputError) as error: + yield in_path, error + else: + yield in_path, retval + + +def setup_logger(logger, main_config, stream): + formatter = logging.Formatter('%(name)s: %(levelname)s: %(message)s') + handler = logging.StreamHandler(stream) + handler.setFormatter(formatter) + logger.addHandler(handler) + +def main(arglist=None, stdout=sys.stdout, stderr=sys.stderr): + try: + my_config = config.Configuration(arglist) + except errors.UserInputError as error: + my_config.error("{}: {!r}".format(error.strerror, error.user_input)) + return 3 + setup_logger(logger, my_config, stderr) + importer = FileImporter(my_config, stdout) + failures = 0 + for input_path, error in importer.import_paths(my_config.args.input_paths): + if error is None: + logger.info("%s: imported", input_path) + else: + logger.warning("%s: failed to import: %s", input_path, error) + failures += 1 + if failures == 0: + return 0 + else: + return min(10 + failures, 99) + +if __name__ == '__main__': + exit(main()) diff --git a/import2ledger/builder.py b/import2ledger/builder.py new file mode 100644 index 0000000..d087d75 --- /dev/null +++ b/import2ledger/builder.py @@ -0,0 +1,23 @@ +import decimal + +import ledgerlib + +class TransactionBuilder: + def __init__(self, config): + self.config = config + + def _build_one_side(self, entry, amount, tx_provider, tx_type, section): + pass + + def build(self, date, payee, amount, + from_prov, from_type, + to_prov_or_type, to_type=None, + section=None): + if to_type is None: + to_prov = from_prov + to_type = to_prov_or_type + else: + to_prov = to_prov_or_type + entry = ledgerlib.Entry(date, payee, date_fmt=self.config.args.date_format) + self._build_one_side(entry, -amount, from_prov, from_type, section) + self._build_one_side(entry, amount, to_prov, to_type, section) diff --git a/import2ledger/config.py b/import2ledger/config.py new file mode 100644 index 0000000..ea637c8 --- /dev/null +++ b/import2ledger/config.py @@ -0,0 +1,210 @@ +import argparse +import configparser +import contextlib +import datetime +import locale +import logging +import os.path +import pathlib + +import babel +import babel.numbers +from . import errors, template, util + +class Configuration: + HOME_PATH = pathlib.Path(os.path.expanduser('~')) + DEFAULT_CONFIG_PATH = pathlib.Path(HOME_PATH, '.config', 'import2ledger.ini') + DEFAULT_ENCODING = locale.getpreferredencoding() + LOCALE = babel.core.Locale.default() + TODAY = datetime.date.today() + CONFIG_DEFAULTS = { + 'date_format': '%%Y/%%m/%%d', + 'loglevel': 'WARNING', + 'output_path': '-', + 'signed_currencies': ','.join(babel.numbers.get_territory_currencies( + LOCALE.territory, start_date=TODAY)), + 'signed_currency_format': '¤#,##0.###;¤-#,##0.###', + 'unsigned_currency_format': '#,##0.### ¤¤', + } + + def __init__(self, arglist): + argparser = self._build_argparser() + self.error = argparser.error + self.args = argparser.parse_args(arglist) + + if self.args.config_file is None: + self.args.config_file = [] + if self.DEFAULT_CONFIG_PATH.exists(): + self.args.config_file.append(self.DEFAULT_CONFIG_PATH) + self.conffile = self._build_conffile() + conffile_paths = [path.as_posix() for path in self.args.config_file] + read_files = self.conffile.read(conffile_paths) + for expected_path, read_path in zip(conffile_paths, read_files): + if read_path != expected_path: + self.error("failed to read configuration file {!r}".format(expected_path)) + + self.finalize() + + def _build_argparser(self): + parser = argparse.ArgumentParser() + parser.add_argument( + '--config-file', '-C', metavar='PATH', type=pathlib.Path, + action='append', + help="Path of a configuration file to read", + ) + parser.add_argument( + '--use-config', '-c', metavar='SECTION', + help="Read settings from this section of the configuration file", + ) + parser.add_argument( + 'input_paths', metavar='PATH', + nargs='+', + help="Path to generate Ledger entries from.", + ) + + out_args = parser.add_argument_group( + "default overrides", + description="These options take priority over settings in the " + "[DEFAULT] section of your config file, but not other sections.", + ) + parser.add_argument( + '--loglevel', '-L', metavar='LEVEL', + choices=['debug', 'info', 'warning', 'error', 'critical'], + help="Log messages at this level and above. Default WARNING.", + ) + out_args.add_argument( + '--date', '-d', metavar='DATE', + help="Date to use in Ledger entries when the source doesn't " + "provide one. Write this in your configured date format. " + "Default today.", + ) + out_args.add_argument( + '--date-format', '-D', metavar='FORMAT', + help="Date format to use in Ledger entries", + ) + out_args.add_argument( + '--output-path', '-O', metavar='PATH', + help="Path of file to append entries to, or '-' for stdout (default).", + ) + out_args.add_argument( + '--signed-currency', '--sign', metavar='CODE', + action='append', dest='signed_currencies', + help="Currency code to use currency sign for in Ledger entry amounts. " + "Can be specified multiple times.", + ) + out_args.add_argument( + '--signed-currency-format', '--sign-format', '-S', metavar='FORMAT', + help="Unicode number pattern to use for signed currencies in Ledger entry amounts", + ) + out_args.add_argument( + '--unsigned-currency-format', '--unsign-format', '-U', metavar='FORMAT', + help="Unicode number pattern to use for unsigned currencies in Ledger entry amounts", + ) + + return parser + + def _build_conffile(self): + return configparser.ConfigParser( + comment_prefixes='#', + defaults=self.CONFIG_DEFAULTS, + ) + + def _s_to_path(self, s): + return None if s == '-' else pathlib.Path(s) + + def _strpdate(self, date_s, date_fmt): + try: + return util.strpdate(date_s, date_fmt) + except ValueError as error: + raise errors.UserInputConfigurationError(error.args[0], date_s) + + def _parse_section_date(self, section_name, default=TODAY): + section = self.conffile[section_name] + try: + return self._strpdate(section['date'], section['date_format']) + except KeyError: + return default + + def finalize(self): + if self.args.use_config is None: + self.args.use_config = self.conffile.default_section + elif not self.conffile.has_section(self.args.use_config): + self.error("section {!r} not found in config file".format(self.args.use_config)) + self.args.input_paths = [self._s_to_path(s) for s in self.args.input_paths] + + defaults = self.conffile[self.conffile.default_section] + for key in self.CONFIG_DEFAULTS: + value = getattr(self.args, key) + if value is None: + pass + elif key == 'signed_currencies': + defaults[key] = ','.join(value) + else: + defaults[key] = value + + # We parse all the dates now to make sure they're valid. + if self.args.date is not None: + default_date = self._strpdate(self.args.date, defaults['date_format']) + elif 'date' in defaults: + default_date = self._strpdate(defaults['date'], defaults['date_format']) + else: + default_date = self.TODAY + + self.dates = {secname: self._parse_section_date(secname, default_date) + for secname in self.conffile} + self.dates[self.conffile.default_section] = default_date + + @contextlib.contextmanager + def from_section(self, section_name): + prev_section = self.args.use_config + self.args.use_config = section_name + try: + yield self + finally: + self.args.use_config = prev_section + + def _get_section(self, section_name): + if section_name is None: + section_name = self.args.use_config + return self.conffile[section_name] + + def get_default_date(self, section_name=None): + if section_name is None: + section_name = self.args.use_config + try: + return self.dates[section_name] + except KeyError: + return self.dates[self.conffile.default_section] + + def get_loglevel(self, section_name=None): + section_config = self._get_section(section_name) + level_name = section_config['loglevel'] + try: + return getattr(logging, level_name.upper()) + except AttributeError: + raise errors.UserInputConfigurationError("not a valid loglevel", level_name) + + def get_output_path(self, section_name=None): + section_config = self._get_section(section_name) + return self._s_to_path(section_config['output_path']) + + def get_template(self, config_key, section_name=None, factory=template.Template): + section_config = self._get_section(section_name) + try: + template_s = section_config[config_key] + except KeyError: + raise errors.UserInputConfigurationError( + "template not defined in [{}]".format(section_name or self.args.use_config), + config_key, + ) + return factory( + template_s, + date_fmt=section_config['date_format'], + signed_currencies=[code.strip().upper() for code in section_config['signed_currencies'].split(',')], + signed_currency_fmt=section_config['signed_currency_format'], + unsigned_currency_fmt=section_config['unsigned_currency_format'], + ) + + def setup_logger(self, logger, section_name=None): + logger.setLevel(self.get_loglevel(section_name)) + diff --git a/import2ledger/errors.py b/import2ledger/errors.py new file mode 100644 index 0000000..9bf310e --- /dev/null +++ b/import2ledger/errors.py @@ -0,0 +1,15 @@ +class UserInputError(Exception): + def __init__(self, strerror, user_input): + super().__init__(strerror, user_input) + self.strerror = strerror + self.user_input = user_input + + +class UserInputConfigurationError(UserInputError): + pass + + +class UserInputFileError(UserInputError): + def __init__(self, strerror, path): + super().__init__(strerror, path) + self.path = path diff --git a/import2ledger/hooks/__init__.py b/import2ledger/hooks/__init__.py new file mode 100644 index 0000000..c8d6a9a --- /dev/null +++ b/import2ledger/hooks/__init__.py @@ -0,0 +1,6 @@ +import operator + +from .. import util + +def load_all(): + return util.submodule_items_named(__file__, operator.methodcaller('endswith', 'Hook')) diff --git a/import2ledger/hooks/add_entity.py b/import2ledger/hooks/add_entity.py new file mode 100644 index 0000000..b6583f1 --- /dev/null +++ b/import2ledger/hooks/add_entity.py @@ -0,0 +1,42 @@ +import re +import unicodedata + +class AddEntityHook: + NONASCII_RE = re.compile(r'[^-A-Za-z0-9]') + NONALNUM_RE = re.compile(r'[^-\w]') + OPEN_PARENS = ['\\(', '\\[', '\\{'] + CLOSE_PARENS = ['\\)', '\\]', '\\}'] + NO_PARENS = '[^{}]*'.format(''.join(OPEN_PARENS + CLOSE_PARENS)) + + def __init__(self, config): + pass + + def _remove_parens(self, s): + last_s = None + while s != last_s: + last_s = s + for open_c, close_c in zip(self.OPEN_PARENS, self.CLOSE_PARENS): + s = re.sub(open_c + self.NO_PARENS + close_c, '', s) + return s if s else last_s + + def _entity_parts(self, s, trim_re): + for word in s.split(): + word = unicodedata.normalize('NFKD', word) + word = trim_re.sub('', word) + if word: + yield word + + def _str2entity(self, s, trim_re): + parts = list(self._entity_parts(s, trim_re)) + if not parts: + return '' + parts.insert(0, parts.pop()) + return '-'.join(parts) + + def run(self, data): + if ('payee' in data) and ('entity' not in data): + payee = self._remove_parens(data['payee']) + entity = self._str2entity(payee, self.NONASCII_RE) + if not entity: + entity = self._str2entity(payee, self.NONALNUM_RE) + data['entity'] = entity diff --git a/import2ledger/hooks/default_date.py b/import2ledger/hooks/default_date.py new file mode 100644 index 0000000..4e787c3 --- /dev/null +++ b/import2ledger/hooks/default_date.py @@ -0,0 +1,7 @@ +class DefaultDateHook: + def __init__(self, config): + self.config = config + + def run(self, entry_data): + if 'date' not in entry_data: + entry_data['date'] = self.config.get_default_date() diff --git a/import2ledger/importers/__init__.py b/import2ledger/importers/__init__.py new file mode 100644 index 0000000..b540e6d --- /dev/null +++ b/import2ledger/importers/__init__.py @@ -0,0 +1,6 @@ +import operator + +from .. import util + +def load_all(): + return util.submodule_items_named(__file__, operator.methodcaller('endswith', 'Importer')) diff --git a/import2ledger/importers/patreon.py b/import2ledger/importers/patreon.py new file mode 100644 index 0000000..403c16d --- /dev/null +++ b/import2ledger/importers/patreon.py @@ -0,0 +1,72 @@ +import csv +import datetime +import pathlib +import re + +from .. import util + +class ImporterBase: + @classmethod + def can_import(cls, input_file): + in_csv = csv.reader(input_file) + fields = next(iter(in_csv), []) + return cls.NEEDED_FIELDS.issubset(fields) + + def __init__(self, input_file): + self.in_csv = csv.DictReader(input_file) + self.start_data = {'currency': 'USD'} + + def __iter__(self): + for row in self.in_csv: + row_data = self._read_row(row) + if row_data is not None: + retval = self.start_data.copy() + retval.update(row_data) + yield retval + + +class IncomeImporter(ImporterBase): + NEEDED_FIELDS = frozenset([ + 'FirstName', + 'LastName', + 'Pledge', + 'Status', + ]) + TEMPLATE_KEY = 'template patreon income' + + def __init__(self, input_file): + super().__init__(input_file) + match = re.search(r'(?:\b|_)(\d{4}-\d{2}-\d{2})(?:\b|_)', + pathlib.Path(input_file.name).name) + if match: + self.start_data['date'] = util.strpdate(match.group(1), '%Y-%m-%d') + + def _read_row(self, row): + if row['Status'] != 'Processed': + return None + else: + return { + 'amount': row['Pledge'], + 'payee': '{0[FirstName]} {0[LastName]}'.format(row), + } + + +class FeeImporterBase(ImporterBase): + def _read_row(self, row): + return { + 'amount': row[self.AMOUNT_FIELD].lstrip('$'), + 'date': util.strpdate(row['Month'], '%Y-%m'), + 'payee': "Patreon", + } + + +class PatreonFeeImporter(FeeImporterBase): + AMOUNT_FIELD = 'Patreon Fee' + NEEDED_FIELDS = frozenset(['Month', AMOUNT_FIELD]) + TEMPLATE_KEY = 'template patreon svcfees' + + +class CardFeeImporter(FeeImporterBase): + AMOUNT_FIELD = 'Processing Fees' + NEEDED_FIELDS = frozenset(['Month', AMOUNT_FIELD]) + TEMPLATE_KEY = 'template patreon cardfees' diff --git a/import2ledger/template.py b/import2ledger/template.py new file mode 100644 index 0000000..ec4d7bd --- /dev/null +++ b/import2ledger/template.py @@ -0,0 +1,98 @@ +import collections +import datetime +import decimal + +import babel.numbers + +class AccountSplitter: + Split = collections.namedtuple('Split', ('account', 'percentage')) + + def __init__(self, sign, signed_currencies, signed_currency_fmt, unsigned_currency_fmt): + self.splits = [] + self.sign = sign + self.signed_currency_fmt = signed_currency_fmt + self.unsigned_currency_fmt = unsigned_currency_fmt + self.signed_currencies = set(signed_currencies) + self.unsplit = sign * 100 + + def add(self, account, percentage_s): + int_pctage = decimal.Decimal(percentage_s.rstrip('%')) + self.unsplit -= int_pctage + percentage = None if self.unsplit == 0 else (int_pctage / 100) + self.splits.append(self.Split(account, percentage)) + + def render_next(self, template_vars): + try: + account, percentage = next(self._split_iter) + except (AttributeError, StopIteration): + self._split_iter = iter(self.splits) + account, percentage = next(self._split_iter) + self._remainder = self.sign * template_vars['amount'] + self._currency = template_vars['currency'] + if self._currency in self.signed_currencies: + self._amt_fmt = self.signed_currency_fmt + else: + self._amt_fmt = self.unsigned_currency_fmt + if percentage is None: + amount = self._remainder + else: + amount = decimal.Decimal(babel.numbers.format_currency( + template_vars['amount'] * percentage, self._currency, '###0.###')) + self._remainder -= amount + amt_s = babel.numbers.format_currency(amount, self._currency, self._amt_fmt) + return ' {:45} {:>19}\n'.format(account, amt_s) + + +class Template: + DATE_FMT = '%Y/%m/%d' + SIGNED_CURRENCY_FMT = '¤#,##0.###;¤-#,##0.###' + UNSIGNED_CURRENCY_FMT = '#,##0.### ¤¤' + + def __init__(self, template_s, signed_currencies=frozenset(), + date_fmt=DATE_FMT, + signed_currency_fmt=SIGNED_CURRENCY_FMT, + unsigned_currency_fmt=UNSIGNED_CURRENCY_FMT): + self.date_fmt = date_fmt + self.pos_splits = AccountSplitter( + 1, signed_currencies, signed_currency_fmt, unsigned_currency_fmt) + self.neg_splits = AccountSplitter( + -1, signed_currencies, signed_currency_fmt, unsigned_currency_fmt) + + lines = [s.lstrip() for s in template_s.splitlines(True)] + for index, line in enumerate(lines): + if line: + break + del lines[:index] + self.format_funcs = [ + '\n'.format_map, + '{date} {payee}\n'.format_map, + ] + start_index = 0 + for index, line in enumerate(lines): + if line[0].isalpha(): + if start_index < index: + self._add_str_func(lines[start_index:index]) + account, percentage_s = line.rsplit(None, 1) + splitter = self.neg_splits if percentage_s.startswith('-') else self.pos_splits + splitter.add(account, percentage_s) + self.format_funcs.append(splitter.render_next) + start_index = index + 1 + if start_index <= index: + self._add_str_func(lines[start_index:]) + if not line.endswith('\n'): + self.format_funcs.append('\n'.format_map) + + def _add_str_func(self, str_seq): + str_flat = ''.join(' ' + s for s in str_seq) + self.format_funcs.append(str_flat.format_map) + + def render(self, payee, amount, currency, date=None, **template_vars): + if date is None: + date = datetime.date.today() + template_vars.update( + date=date.strftime(self.date_fmt), + payee=payee, + amount=decimal.Decimal(amount), + currency=currency, + ) + return ''.join(f(template_vars) for f in self.format_funcs) diff --git a/import2ledger/util.py b/import2ledger/util.py new file mode 100644 index 0000000..2c4d7de --- /dev/null +++ b/import2ledger/util.py @@ -0,0 +1,34 @@ +import datetime +import importlib +import logging +import pathlib + +logger = logging.getLogger('import2ledger') + +def load_modules(src_dir_path): + rel_path = src_dir_path.relative_to(pathlib.Path(__file__).parent) + import_prefix = 'import2ledger.{}.'.format('.'.join(rel_path.parts)) + for py_path in src_dir_path.glob('*.py'): + mod_name = py_path.name[:-3] + if mod_name.startswith(('.', '_')): + continue + try: + module = importlib.import_module(import_prefix + mod_name) + except ImportError as error: + logger.info("failed to import %s: %s", py_path, error, + exc_info=logger.isEnabledFor(logging.DEBUG)) + else: + yield module + +def module_contents(module): + for name in dir(module): + yield name, getattr(module, name) + +def submodule_items_named(file_path, name_test): + for module in load_modules(pathlib.Path(file_path).parent): + for name, item in module_contents(module): + if name_test(name): + yield item + +def strpdate(date_s, date_fmt): + return datetime.datetime.strptime(date_s, date_fmt).date() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b7e4789 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test=pytest diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..7d7e5da --- /dev/null +++ b/setup.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +from setuptools import setup + +setup( + name='import2ledger', + description="Import different sources of financial data to Ledger", + version='0.1', + author='Brett Smith', + author_email='brettcsmith@brettcsmith.org', + license='GNU AGPLv3+', + + install_requires=['babel'], + setup_requires=['pytest-runner'], + tests_require=['pytest', 'PyYAML'], + + packages=['import2ledger'], + entry_points={}, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..4c5c929 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,7 @@ +import pathlib +import re + +DATA_DIR = pathlib.Path(__file__).with_name('data') + +def normalize_whitespace(s): + return re.sub(r'(\t| {3,})', ' ', s) diff --git a/tests/data/PatreonEarnings.csv b/tests/data/PatreonEarnings.csv new file mode 100644 index 0000000..945044a --- /dev/null +++ b/tests/data/PatreonEarnings.csv @@ -0,0 +1,3 @@ +Month,Processed Pledges,Creator's Share,Processing Fees,Patreon Fee,Creator's Share (%),Processing Fees (%),Patreon Fee (%) +2017-09,"$1234.50","$1,120.30",$52.47,$61.73,90.75%,4.25%,5.0% +2017-10,"$2340.50","$2,124.00",$99.47,$117.03,90.75%,4.25%,5.0% diff --git a/tests/data/PatreonPatronReport_2017-09-01.csv b/tests/data/PatreonPatronReport_2017-09-01.csv new file mode 100644 index 0000000..4ac9ef5 --- /dev/null +++ b/tests/data/PatreonPatronReport_2017-09-01.csv @@ -0,0 +1,6 @@ +FirstName,LastName,Email,Pledge,Lifetime,Status,Twitter,Street,City,State,Zip,Country,Start,MaxAmount,Complete +100 + Reward,Description You donate a lot of money!,,,,,,,,,,,,, +Alex,Jones,alex@example.org,150,300,Processed,,,,,,,2017-08-11 11:28:06.166065,200,0 +5 + Reward,Description You’re nice!,,,,,,,,,,,,, +Brett,Smith,brett@example.org,10,30,Declined,Brett20XX,,,,,,2017-08-10 13:24:15.955782,10,0 +Dakota,Doe,ddoe@example.org,12,48,Processed,,,,,,,2017-08-10 12:58:31.919341,12,0 diff --git a/tests/data/imports.yml b/tests/data/imports.yml new file mode 100644 index 0000000..ffc0041 --- /dev/null +++ b/tests/data/imports.yml @@ -0,0 +1,35 @@ +- source: PatreonPatronReport_2017-09-01.csv + importer: patreon.IncomeImporter + expect: + - payee: Alex Jones + date: [2017, 9, 1] + amount: "150" + currency: USD + - payee: Dakota Doe + date: [2017, 9, 1] + amount: "12" + currency: USD + +- source: PatreonEarnings.csv + importer: patreon.PatreonFeeImporter + expect: + - payee: Patreon + date: [2017, 9, 1] + amount: "61.73" + currency: USD + - payee: Patreon + date: [2017, 10, 1] + amount: "117.03" + currency: USD + +- source: PatreonEarnings.csv + importer: patreon.CardFeeImporter + expect: + - payee: Patreon + date: [2017, 9, 1] + amount: "52.47" + currency: USD + - payee: Patreon + date: [2017, 10, 1] + amount: "99.47" + currency: USD diff --git a/tests/data/templates.ini b/tests/data/templates.ini new file mode 100644 index 0000000..b5216e4 --- /dev/null +++ b/tests/data/templates.ini @@ -0,0 +1,22 @@ +[Simplest] +template = + Accrued:Accounts Receivable 100%% + Income:Donations -100%% + +[FiftyFifty] +template = + Accrued:Accounts Receivable 100%% + Income:Donations -50%% + Income:Sales -50%% + +[Complex] +template = + ;Tag: Value + ;TransactionID: {txid} + Accrued:Accounts Receivable 100%% + ;Entity: Supplier + Income:Donations:Specific -95.5%% + ;Program: Specific + ;Entity: {entity} + Income:Donations:General -4.5%% + ;Entity: {entity} diff --git a/tests/data/test_config.ini b/tests/data/test_config.ini new file mode 100644 index 0000000..4c9fa8d --- /dev/null +++ b/tests/data/test_config.ini @@ -0,0 +1,24 @@ +[DEFAULT] +date_format = %%Y-%%m-%%d +signed_currencies = EUR + +[Templates] +output_path = Template.output +one = + Accrued:Accounts Receivable 100%% + Income:Donations -100%% +two = + ;Tag1: {value} + Income:Donations -100%% + ;IncomeTag: Donations + Expenses:Fundraising 5%% + ;ExpenseTag: Fundraising + Accrued:Accounts Receivable 95%% + ;AccrualTag: Receivable + +[Date] +date_format = %%Y|%%m|%%d +date = 2017|10|08 + +[Bad Loglevel] +loglevel = wraning diff --git a/tests/data/test_main.ini b/tests/data/test_main.ini new file mode 100644 index 0000000..3506f93 --- /dev/null +++ b/tests/data/test_main.ini @@ -0,0 +1,12 @@ +[DEFAULT] +default_date = 2016/04/04 +loglevel = critical +signed_currencies = USD + +[One] +template patreon cardfees = + Accrued:Accounts Receivable -100%% + Expenses:Fees:Credit Card 100%% +template patreon svcfees = + Accrued:Accounts Receivable -100%% + Expenses:Fundraising 100%% diff --git a/tests/data/test_main_fees_import.ledger b/tests/data/test_main_fees_import.ledger new file mode 100644 index 0000000..b7acce4 --- /dev/null +++ b/tests/data/test_main_fees_import.ledger @@ -0,0 +1,15 @@ +2017/09/01 Patreon + Accrued:Accounts Receivable $-52.47 + Expenses:Fees:Credit Card $52.47 + +2017/09/01 Patreon + Accrued:Accounts Receivable $-61.73 + Expenses:Fundraising $61.73 + +2017/10/01 Patreon + Accrued:Accounts Receivable $-99.47 + Expenses:Fees:Credit Card $99.47 + +2017/10/01 Patreon + Accrued:Accounts Receivable $-117.03 + Expenses:Fundraising $117.03 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..d16a89d --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,109 @@ +import contextlib +import datetime +import logging +import os +import pathlib + +START_DATE = datetime.date.today() + +from unittest import mock + +import pytest +from import2ledger import config, errors + +from . import DATA_DIR + +def config_from_file(path, arglist=[]): + path = pathlib.Path(path) + if not path.is_absolute(): + path = DATA_DIR / path + arglist = ['-C', path.as_posix(), *arglist, os.devnull] + return config.Configuration(arglist) + +def test_defaults(): + config = config_from_file('test_config.ini', ['--sign', 'GBP', '-O', 'out_arg']) + factory = mock.Mock(name='Template') + template = config.get_template('one', 'Templates', factory) + assert factory.called + kwargs = factory.call_args[1] + assert list(kwargs.pop('signed_currencies', '')) == ['GBP'] + assert kwargs == { + 'date_fmt': '%Y-%m-%d', + 'signed_currency_fmt': kwargs['signed_currency_fmt'], + 'unsigned_currency_fmt': kwargs['unsigned_currency_fmt'], + } + +def test_template_parsing(): + config = config_from_file('test_config.ini') + factory = mock.Mock(name='Template') + template = config.get_template('two', 'Templates', factory) + try: + tmpl_s = factory.call_args[0][0] + except IndexError as error: + assert False, error + assert "\n;Tag1: {value}\n" in tmpl_s + assert "\nIncome:Donations -100%\n" in tmpl_s + assert "\n;IncomeTag: Donations\n" in tmpl_s + +@pytest.mark.parametrize('arg_s', [None, '-', 'output.ledger']) +def test_output_path(arg_s): + arglist = [] if arg_s is None else ['-O', arg_s] + config = config_from_file(os.devnull, arglist) + output_path = config.get_output_path() + if (arg_s is None) or (arg_s == '-'): + assert output_path is None + else: + assert output_path == pathlib.Path(arg_s) + +def test_output_path_from_section(): + expected_path = pathlib.Path('Template.output') + config = config_from_file('test_config.ini', ['-O', 'output.ledger']) + assert config.get_output_path('Templates') == expected_path + assert config.get_output_path() != expected_path + with config.from_section('Templates'): + assert config.get_output_path() == expected_path + +@pytest.mark.parametrize('arglist,expect_date', [ + ([], None), + (['-d', '2017-10-12'], datetime.date(2017, 10, 12)), + (['-c', 'Date'], datetime.date(2017, 10, 8)), +]) +def test_default_date(arglist, expect_date): + config = config_from_file('test_config.ini', arglist) + default_date = config.get_default_date() + if expect_date is None: + assert START_DATE <= default_date <= datetime.date.today() + else: + assert default_date == expect_date + assert config.get_default_date('Date') == datetime.date(2017, 10, 8) + +@pytest.mark.parametrize('level_s,expect_level', [ + (s, getattr(logging, s.upper())) + for s in ['critical', 'debug', 'error', 'info', 'warning'] +]) +def test_loglevel(level_s, expect_level): + config = config_from_file(os.devnull, ['--loglevel', level_s]) + assert config.get_loglevel() == expect_level + +@contextlib.contextmanager +def bad_config(expect_input): + with pytest.raises(errors.UserInputConfigurationError) as exc_info: + yield exc_info + assert exc_info.value.user_input == expect_input + +def test_bad_default_date(): + date_s = '2017-10-06' + with bad_config(date_s): + config = config_from_file(os.devnull, ['--date', date_s]) + config.get_default_date() + +def test_bad_loglevel(): + with bad_config('wraning'): + config = config_from_file('test_config.ini', ['-c', 'Bad Loglevel']) + config.get_loglevel() + +def test_undefined_template(): + template_name = 'template nonexistent' + config = config_from_file(os.devnull) + with bad_config(template_name): + config.get_template(template_name) diff --git a/tests/test_hooks.py b/tests/test_hooks.py new file mode 100644 index 0000000..1a7aa3c --- /dev/null +++ b/tests/test_hooks.py @@ -0,0 +1,66 @@ +import argparse +import datetime +import itertools + +import pytest + +from import2ledger import hooks +from import2ledger.hooks import add_entity, default_date + +def test_load_all(): + all_hooks = list(hooks.load_all()) + assert add_entity.AddEntityHook in all_hooks + +@pytest.mark.parametrize('payee,expected', [ + ('Alex Smith', 'Smith-Alex'), + ('Dakota D. Doe', 'Doe-Dakota-D'), + ('Björk', 'Bjork'), + ('Fran Doe-Smith', 'Doe-Smith-Fran'), + ('Alex(Nickname) Smith', 'Smith-Alex'), + ('稲荷', '稲荷'), +]) +def test_add_entity(payee, expected): + data = {'payee': payee} + hook = add_entity.AddEntityHook(argparse.Namespace()) + hook.run(data) + assert data['entity'] == expected + + +class DefaultDateConfig: + ONE_DAY = datetime.timedelta(days=1) + + def __init__(self, start_date=None): + if start_date is None: + start_date = datetime.date(2016, 3, 5) + self.date = start_date - self.ONE_DAY + + def get_default_date(self, section_name=None): + self.date += self.ONE_DAY + return self.date + + +class TestDefaultDate: + def test_simple_case(self): + expect_date = datetime.date(2016, 2, 4) + config = DefaultDateConfig(expect_date) + data = {} + hook = default_date.DefaultDateHook(config) + hook.run(data) + assert data['date'] == expect_date + + def test_no_caching(self): + config = DefaultDateConfig() + hook = default_date.DefaultDateHook(config) + d1 = {} + d2 = {} + hook.run(d1) + hook.run(d2) + assert d1['date'] != d2['date'] + + def test_no_override(self): + expect_date = datetime.date(2016, 2, 6) + config = DefaultDateConfig(expect_date + datetime.timedelta(days=300)) + hook = default_date.DefaultDateHook(config) + data = {'date': expect_date} + hook.run(data) + assert data['date'] is expect_date diff --git a/tests/test_import_patreon_income.py b/tests/test_import_patreon_income.py new file mode 100644 index 0000000..c64d3c8 --- /dev/null +++ b/tests/test_import_patreon_income.py @@ -0,0 +1,2 @@ +import datetime + diff --git a/tests/test_importers.py b/tests/test_importers.py new file mode 100644 index 0000000..1a47d56 --- /dev/null +++ b/tests/test_importers.py @@ -0,0 +1,47 @@ +import datetime +import importlib +import itertools +import pathlib + +import pytest +import yaml +from import2ledger import importers + +from . import DATA_DIR + +class TestImporters: + with pathlib.Path(DATA_DIR, 'imports.yml').open() as yaml_file: + test_data = yaml.load(yaml_file) + for test in test_data: + test['source'] = DATA_DIR / test['source'] + + module_name, class_name = test['importer'].rsplit('.', 1) + module = importlib.import_module('.' + module_name, 'import2ledger.importers') + test['importer'] = getattr(module, class_name) + + for expect_result in test['expect']: + try: + expect_result['date'] = datetime.date(*expect_result['date']) + except KeyError: + pass + + @pytest.mark.parametrize('source_path,importer', [ + (t['source'], t['importer']) for t in test_data + ]) + def test_can_import(self, source_path, importer): + with source_path.open() as source_file: + assert importer.can_import(source_file) + + @pytest.mark.parametrize('source_path,import_class,expect_results', [ + (t['source'], t['importer'], t['expect']) for t in test_data + ]) + def test_import(self, source_path, import_class, expect_results): + with source_path.open() as source_file: + importer = import_class(source_file) + for actual, expected in itertools.zip_longest(importer, expect_results): + assert actual == expected + + def test_loader(self): + all_importers = list(importers.load_all()) + for test in self.test_data: + assert test['importer'] in all_importers diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..6f6c438 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,52 @@ +import io +import pathlib + +import pytest +from . import DATA_DIR, normalize_whitespace + +from import2ledger import __main__ as i2lmain + +ARGLIST = [ + '-C', (DATA_DIR / 'test_main.ini').as_posix(), +] + +def run_main(arglist): + stdout = io.StringIO() + stderr = io.StringIO() + exitcode = i2lmain.main(arglist, stdout, stderr) + stdout.seek(0) + stderr.seek(0) + return exitcode, stdout, stderr + +def iter_entries(in_file): + lines = [] + for line in in_file: + if line == '\n': + if lines: + yield ''.join(lines) + lines = [] + else: + lines.append(line) + if lines: + yield ''.join(lines) + +def entries2set(in_file): + return set(normalize_whitespace(e) for e in iter_entries(in_file)) + +def expected_entries(path): + path = pathlib.Path(path) + if not path.is_absolute(): + path = DATA_DIR / path + with path.open() as in_file: + return entries2set(in_file) + +def test_fees_import(): + arglist = ARGLIST + [ + '-c', 'One', + pathlib.Path(DATA_DIR, 'PatreonEarnings.csv').as_posix(), + ] + exitcode, stdout, _ = run_main(arglist) + print(_.getvalue()) + assert exitcode == 0 + actual = entries2set(stdout) + assert actual == expected_entries('test_main_fees_import.ledger') diff --git a/tests/test_templates.py b/tests/test_templates.py new file mode 100644 index 0000000..8dc0243 --- /dev/null +++ b/tests/test_templates.py @@ -0,0 +1,75 @@ +import configparser +import datetime +import decimal +import pathlib + +import pytest +from import2ledger import template + +from . import DATA_DIR, normalize_whitespace + +DATE = datetime.date(2015, 3, 14) + +config = configparser.ConfigParser(comment_prefixes='#') +with pathlib.Path(DATA_DIR, 'templates.ini').open() as conffile: + config.read_file(conffile) + +def template_from(section_name, *args, **kwargs): + return template.Template(config[section_name]['template'], *args, **kwargs) + +def assert_easy_render(tmpl, entity, amount, currency, expect_date, expect_amt): + rendered = tmpl.render(entity, decimal.Decimal(amount), currency, DATE) + print(repr(rendered)) + lines = [normalize_whitespace(s) for s in rendered.splitlines()] + assert lines == [ + "", + "{} {}".format(expect_date, entity), + " Accrued:Accounts Receivable " + expect_amt, + " Income:Donations " + expect_amt.replace(amount, "-" + amount), + ] + +def test_easy_template(): + tmpl = template_from('Simplest') + assert_easy_render(tmpl, 'JJ', '5.99', 'CAD', '2015/03/14', '5.99 CAD') + +def test_date_formatting(): + tmpl = template_from('Simplest', date_fmt='%Y-%m-%d') + assert_easy_render(tmpl, 'KK', '6.99', 'CAD', '2015-03-14', '6.99 CAD') + +def test_currency_formatting(): + tmpl = template_from('Simplest', signed_currencies=['USD']) + assert_easy_render(tmpl, 'CC', '7.99', 'USD', '2015/03/14', '$7.99') + +def test_complex_template(): + template_vars = { + 'entity': 'T-T', + 'txid': 'ABCDEF', + } + tmpl = template_from('Complex', date_fmt='%Y-%m-%d', signed_currencies=['USD']) + rendered = tmpl.render('TT', decimal.Decimal('125.50'), 'USD', DATE, **template_vars) + lines = [normalize_whitespace(s) for s in rendered.splitlines()] + assert lines == [ + "", + "2015-03-14 TT", + " ;Tag: Value", + " ;TransactionID: ABCDEF", + " Accrued:Accounts Receivable $125.50", + " ;Entity: Supplier", + " Income:Donations:Specific $-119.85", + " ;Program: Specific", + " ;Entity: T-T", + " Income:Donations:General $-5.65", + " ;Entity: T-T", + ] + +def test_balancing(): + tmpl = template_from('FiftyFifty') + rendered = tmpl.render('FF', decimal.Decimal('1.01'), 'USD', DATE) + lines = [normalize_whitespace(s) for s in rendered.splitlines()] + assert lines == [ + "", + "2015/03/14 FF", + " Accrued:Accounts Receivable 1.01 USD", + " Income:Donations -0.50 USD", + " Income:Sales -0.51 USD", + ]