meta_receivable_documentation: Start hook.

conservancy_beancount/plugin/meta_receivable_documentation.py

@@ -0,0 +1,92 @@
+"""meta_receivable_documentation - Validate receivables have supporting docs"""
+# Copyright © 2020  Brett Smith
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+
+from . import core
+from .. import config as configmod
+from .. import data
+from .. import errors as errormod
+from ..beancount_types import (
+    MetaKey,
+    Transaction,
+)
+
+from typing import (
+    Dict,
+    Optional,
+)
+
+class MetaReceivableDocumentation(core._RequireLinksPostingMetadataHook):
+    HOOK_GROUPS = frozenset(['network', 'rt'])
+    SUPPORTING_METADATA = frozenset([
+        'approval',
+        'contract',
+        'purchase-order',
+    ])
+    METADATA_KEY = '/'.join(sorted(SUPPORTING_METADATA))
+    # Conservancy invoice filenames have followed two patterns.
+    # The pre-RT pattern: `YYYY-MM-DD_Entity_invoice-YYYYMMDDNN??_as-sent.pdf`
+    # The RT pattern: `ProjectInvoice-30NNNN??.pdf`
+    # This regexp matches both, with a little slack to try to reduce the false
+    # negative rate due to minor renames, etc.
+    ISSUED_INVOICE_RE = re.compile(
+        r'[Ii]nvoice[-_ ]*(?:2[0-9]{9,}|30[0-9]+)[A-Za-z]*[-_ .]',
+    )
+
+    def __init__(self, config: configmod.Config) -> None:
+        rt_wrapper = config.rt_wrapper()
+        # In principle, we could still check for non-RT invoices and enforce
+        # checks on them without an RT wrapper. In practice, that would
+        # provide so little utility today it's not worth bothering with.
+        if rt_wrapper is None:
+            raise errormod.ConfigurationError("can't log in to RT")
+        self.rt = rt_wrapper
+
+    def _run_on_post(self, txn: Transaction, post: data.Posting) -> bool:
+        if not post.account.is_under('Accrued:AccountsReceivable'):
+            return False
+
+        # Get the first invoice, or return False if it doesn't exist.
+        try:
+            invoice_link = post.meta.get_links('invoice')[0]
+        except (IndexError, TypeError):
+            return False
+
+        # Get the filename, following an RT link if necessary.
+        rt_args = self.rt.parse(invoice_link)
+        if rt_args is not None:
+            ticket_id, attachment_id = rt_args
+            invoice_link = self.rt.url(ticket_id, attachment_id) or invoice_link
+        return self.ISSUED_INVOICE_RE.search(invoice_link) is not None
+
+    def post_run(self, txn: Transaction, post: data.Posting) -> errormod.Iter:
+        errors: Dict[MetaKey, Optional[errormod.Error]] = {
+            key: None for key in self.SUPPORTING_METADATA
+        }
+        have_support = False
+        for key in errors:
+            try:
+                self._check_links(txn, post, key)
+            except errormod.Error as key_error:
+                errors[key] = key_error
+            else:
+                have_support = True
+        for key, error in errors.items():
+            if error and not error.message.endswith(f" missing {key}"):
+                yield error
+        if not have_support:
+            yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, None, post)

tests/test_meta_receivable_documentation.py

@@ -0,0 +1,188 @@
+"""Test metadata for receivables includes supporting documentation"""
+# Copyright © 2020  Brett Smith
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import random
+
+import pytest
+
+from . import testutil
+
+from conservancy_beancount import errors as errormod
+from conservancy_beancount.plugin import meta_receivable_documentation
+
+TEST_ACCT = 'Accrued:AccountsReceivable'
+OTHER_ACCT = 'Income:Donations'
+
+SUPPORTING_METADATA = [
+    'approval',
+    'contract',
+    'purchase-order',
+]
+
+NON_SUPPORTING_METADATA = [
+    'check',
+    'receipt',
+    'statement',
+]
+
+ISSUED_INVOICE_LINKS = [
+    'rt:1/6',
+    'rt://ticket/2/attachments/10',
+    'Financial/Invoices/Company_invoice-2020030405_as-sent.pdf',
+    'Projects/Alpha/Invoices/ProjectInvoice-304567.pdf',
+]
+
+RECEIVED_INVOICE_LINKS = [
+    'rt:1/4',
+    'rt://ticket/2/attachments/14',
+    'Financial/Invoices/Invoice-23456789.csv',
+    'Projects/Bravo/Invoices/Statement304567.pdf',
+]
+
+MISSING_MSG = f"{TEST_ACCT} missing approval/contract/purchase-order"
+
+# for supporting links, use the lists from testutil
+
+@pytest.fixture(scope='module')
+def hook():
+    config = testutil.TestConfig(rt_client=testutil.RTClient())
+    return meta_receivable_documentation.MetaReceivableDocumentation(config)
+
+def seed_meta(invoice=ISSUED_INVOICE_LINKS[0],
+              support_values=testutil.NON_LINK_METADATA_STRINGS,
+              **kwargs,
+):
+    meta = dict(testutil.combine_values(
+        SUPPORTING_METADATA,
+        support_values,
+    ))
+    meta.update(kwargs)
+    meta['invoice'] = invoice
+    return meta
+
+def wrong_type_message(key, wrong_value):
+    return  "{} has wrong type of {}: expected str but is a {}".format(
+        TEST_ACCT,
+        key,
+        type(wrong_value).__name__,
+    )
+
+def check(hook, expected, test_acct=TEST_ACCT, other_acct=OTHER_ACCT, *,
+          txn_meta={}, post_meta={}, min_amt=10):
+    amount = '{:.02f}'.format(min_amt + random.random() * 100)
+    txn = testutil.Transaction(**txn_meta, postings=[
+        (test_acct, amount, post_meta),
+        (other_acct, f'-{amount}'),
+    ])
+    actual = {error.message for error in hook.run(txn)}
+    if expected is None:
+        assert not actual
+    elif isinstance(expected, str):
+        assert expected in actual
+    else:
+        assert actual == expected
+
+@pytest.mark.parametrize('invoice,support_key,support_value', testutil.combine_values(
+    ISSUED_INVOICE_LINKS,
+    SUPPORTING_METADATA,
+    testutil.LINK_METADATA_STRINGS,
+))
+def test_valid_docs_in_post(hook, invoice, support_key, support_value):
+    meta = seed_meta(invoice)
+    meta[support_key] = support_value
+    check(hook, None, post_meta=meta)
+
+@pytest.mark.parametrize('invoice,support_key,support_value', testutil.combine_values(
+    ISSUED_INVOICE_LINKS,
+    SUPPORTING_METADATA,
+    testutil.LINK_METADATA_STRINGS,
+))
+def test_valid_docs_in_txn(hook, invoice, support_key, support_value):
+    meta = seed_meta(invoice)
+    meta[support_key] = support_value
+    check(hook, None, txn_meta=meta)
+
+@pytest.mark.parametrize('invoice,meta_type', testutil.combine_values(
+    ISSUED_INVOICE_LINKS,
+    ['post_meta', 'txn_meta'],
+))
+def test_no_valid_docs(hook, invoice, meta_type):
+    meta = seed_meta(invoice)
+    meta.update((key, value) for key, value in testutil.combine_values(
+        NON_SUPPORTING_METADATA,
+        testutil.LINK_METADATA_STRINGS,
+    ))
+    check(hook, {MISSING_MSG}, **{meta_type: meta})
+
+@pytest.mark.parametrize('invoice,meta_type', testutil.combine_values(
+    ISSUED_INVOICE_LINKS,
+    ['post_meta', 'txn_meta'],
+))
+def test_docs_all_bad_type(hook, invoice, meta_type):
+    meta = seed_meta(invoice, testutil.NON_STRING_METADATA_VALUES)
+    expected = {
+        wrong_type_message(key, value)
+        for key, value in meta.items()
+        if key != 'invoice'
+    }
+    expected.add(MISSING_MSG)
+    check(hook, expected, **{meta_type: meta})
+
+@pytest.mark.parametrize('invoice,support_key,support_value', testutil.combine_values(
+    ISSUED_INVOICE_LINKS,
+    SUPPORTING_METADATA,
+    testutil.LINK_METADATA_STRINGS,
+))
+def test_type_errors_reported_with_valid_post_docs(hook, invoice, support_key, support_value):
+    meta = seed_meta(invoice, testutil.NON_STRING_METADATA_VALUES)
+    meta[support_key] = support_value
+    expected = {
+        wrong_type_message(key, value)
+        for key, value in meta.items()
+        if key != 'invoice' and  key != support_key
+    }
+    check(hook, expected, post_meta=meta)
+
+@pytest.mark.parametrize('invoice,support_key,support_value', testutil.combine_values(
+    ISSUED_INVOICE_LINKS,
+    SUPPORTING_METADATA,
+    testutil.LINK_METADATA_STRINGS,
+))
+def test_type_errors_reported_with_valid_txn_docs(hook, invoice, support_key, support_value):
+    meta = seed_meta(invoice, testutil.NON_STRING_METADATA_VALUES)
+    meta[support_key] = support_value
+    expected = {
+        wrong_type_message(key, value)
+        for key, value in meta.items()
+        if key != 'invoice' and  key != support_key
+    }
+    check(hook, expected, txn_meta=meta)
+
+@pytest.mark.parametrize('invoice,meta_type', testutil.combine_values(
+    RECEIVED_INVOICE_LINKS,
+    ['post_meta', 'txn_meta'],
+))
+def test_received_invoices_not_checked(hook, invoice, meta_type):
+    check(hook, None, **{meta_type: {'invoice': invoice}})
+
+def test_does_not_apply_to_payables(hook):
+    meta = seed_meta()
+    check(hook, None, 'Accrued:AccountsPayable', 'Expenses:Other', post_meta=meta)
+
+def test_configuration_error_without_rt():
+    config = testutil.TestConfig()
+    with pytest.raises(errormod.ConfigurationError):
+        meta_receivable_documentation.MetaReceivableDocumentation(config)

tests/test_rtutil.py

@@ -30,7 +30,7 @@ EXPECTED_URLS = [
     (1, 4, 'Ticket/Attachment/1/4/Forwarded%20Message.eml'),
     (1, 99, None),
     (2, 1, None),
-    (2, 10, 'Ticket/Attachment/7/10/screenshot.png'),
+    (2, 10, 'Ticket/Attachment/7/10/Company_invoice-2020030405_as-sent.pdf'),
     (2, 13, 'Ticket/Display.html?id=2#txn-11'),
     (2, 14, 'Ticket/Display.html?id=2#txn-11'),  # statement.txt
     (3, None, 'Ticket/Display.html?id=3'),

tests/testutil.py

@@ -179,9 +179,10 @@ class _TicketBuilder:
     MISC_ATTACHMENTS = [
         ('Forwarded Message.eml', 'message/rfc822', '3.1k'),
         ('photo.jpg', 'image/jpeg', '65.2k'),
-        ('document.pdf', 'application/pdf', '326k'),
-        ('screenshot.png', 'image/png', '1.9m'),
+        ('ConservancyInvoice-301.pdf', 'application/pdf', '326k'),
+        ('Company_invoice-2020030405_as-sent.pdf', 'application/pdf', '50k'),
         ('statement.txt', 'text/plain', '652b'),
+        ('screenshot.png', 'image/png', '1.9m'),
     ]
 
     def __init__(self):