Add importer for Printful Orders.csv file

This reflects the current CSV output from the "Export" button.

import2ledger/hooks/ledger_entry.py

@@ -5,6 +5,7 @@ import functools
 import io
 import logging
 import operator
+import pprint
 import re
 import tokenize
 
@@ -192,8 +193,18 @@ class AccountSplitter:
                 "template {!r}".format(self.template_name)
             ) from error
         if sum(amt for _, amt in amounts) != 0:
-            self._balance_amounts(amounts, template_vars['amount'])
-            self._balance_amounts(amounts, -template_vars['amount'])
+            try:
+                self._balance_amounts(amounts, template_vars['amount'])
+                self._balance_amounts(amounts, -template_vars['amount'])
+            except errors.UserInputConfigurationError as error:
+                printer = pprint.PrettyPrinter()
+                logger.debug(
+                    "%s\n%s\n%s",
+                    error.args[0],
+                    printer.pformat(amounts),
+                    printer.pformat(dict(template_vars)),
+                )
+                raise
         return amounts
 
     def _iter_splits(self, template_vars):

import2ledger/importers/brightfunds.py

@@ -53,7 +53,12 @@ class _BrightFundsMixin:
 
 
 class DonorReportImporter(_BrightFundsMixin, _csv.CSVImporterBase):
-    pass
+    BOOK_KWARGS = {'encoding_override': 'utf-8'}
+    DATE_FMT = '%m/%d/%Y'
+    DATE_FIELD = 'Created'
+    DONOR_FIELD = 'Donor Name'
+    EMAIL_FIELD = 'Donor Email'
+    TYPE_FIELD = 'Type'
 
 
 class DonorReportXLSImporter(_BrightFundsMixin, _xls.XLSImporterBase):

import2ledger/importers/eventbrite.py

@@ -29,7 +29,8 @@ class SalesImporter(_csv.CSVImporterBase):
     ENTRY_SEED = {'currency': 'USD'}
 
     def _read_row(self, row):
-        if row['Order Type'] != 'Eventbrite Completed':
+ #       if row['Order Type'] != 'Eventbrite Completed':
+        if row['Order Type'] == 'Free Order':
             return None
         retval = {
             self.DECIMAL_FIELDS[key]: strparse.currency_decimal(row[key])

import2ledger/importers/github.py

@@ -6,24 +6,27 @@ from .. import strparse
 class SponsorsImporter(_csv.CSVImporterBase):
     DATE_FMT = '%Y-%m-%d %H:%M:%S %z'
     NEEDED_FIELDS = frozenset([
-        'processed amount',
-        'status',
-        'transaction date',
+        'transferred amount',
+        'transaction timestamp',
     ])
     COPIED_FIELDS = {
         'sponsor handle': 'handle',
         # 'sponsor profile name': 'name',
         'sponsor email': 'email',
-        'transaction id': 'transaction_id',
+        'stripe transfer id': 'transaction_id',
     }
     ENTRY_SEED = {'currency': 'USD'}
+    PAYEE_MAP = {
+        'PRIVATE': 'Anonymous',
+    }
 
     def _read_row(self, row):
-        amount = strparse.currency_decimal(row['processed amount'])
-        if (not amount) or row['status'] != 'settled':
+        amount = strparse.currency_decimal(row['transferred amount'])
+        if (not amount):
             return None
+        payee = row.get('sponsor profile name') or row['sponsor handle']
         return {
             'amount': amount,
-            'date': strparse.date(row['transaction date'], self.DATE_FMT),
-            'payee': row.get('sponsor profile name') or row['sponsor handle'],
+            'date': strparse.date(row['transaction timestamp'], self.DATE_FMT),
+            'payee': self.PAYEE_MAP.get(payee, payee),
         }

import2ledger/importers/patreon.py

@@ -4,6 +4,38 @@ import re
 from . import _csv
 from .. import strparse
 
+class EarningsImporter(_csv.CSVImporterBase):
+    AMOUNT_FIELDS = {
+        'Creator Share': 'net_amount',
+        'Creator Platform Fee': 'platform_fee',
+        'Creator Payment Processing Fee': 'processing_fee',
+        'Creator Currency Conversion Fee': 'currency_fee',
+        'Creator Equivalent of Patron Tax': 'tax_amount',
+    }
+    DATE_KEY = 'Date'
+    TYPE_KEY = 'Event Type'
+    NEEDED_FIELDS = frozenset([*AMOUNT_FIELDS, DATE_KEY, TYPE_KEY])
+    COPIED_FIELDS = {
+        'Creator Currency': 'currency',
+        'Event ID': 'event_id',
+        'Patron Email': 'email',
+        'Patron Name': 'payee',
+        'Patron User ID': 'user_id',
+    }
+    DATE_FMT = '%Y-%m-%d %H:%M:%S'
+
+    def _read_row(self, row):
+        if row[self.TYPE_KEY] != 'Payment':
+            return None
+        retval = {
+            ret_key: strparse.currency_decimal(row[src_key] or 0)
+            for src_key, ret_key in self.AMOUNT_FIELDS.items()
+        }
+        retval['amount'] = sum(retval.values()) - retval['tax_amount']
+        retval['date'] = strparse.date(row[self.DATE_KEY], self.DATE_FMT)
+        return retval
+
+
 class IncomeImporter(_csv.CSVImporterBase):
     AMOUNT_KEY = 'Pledge Amount'
     DATE_KEY = 'Last Charge Date'
@@ -118,6 +150,7 @@ class Income2017Importer(_csv.CSVImporterBase):
         'Status',
     ])
     COPIED_FIELDS = {
+        'Email': 'email',
         'Pledge': 'amount',
     }
     ENTRY_SEED = {

import2ledger/importers/printful.py

@@ -0,0 +1,58 @@
+import re
+
+from . import _csv
+from .. import strparse
+
+
+class SalesImporter(_csv.CSVImporterBase):
+    DECIMAL_FIELDS = {
+        'Products': 'products',
+        'Shipping': 'shipping_fees',
+        'Tax': 'tax',
+        'VAT': 'vat',
+        'Total': 'amount',
+        'Discount': 'discount',
+    }
+    NEEDED_FIELDS = {'Printful ID', *DECIMAL_FIELDS}
+    COPIED_FIELDS = {}
+    CURRENCY_MAP = {
+        '$': 'USD',
+        '€': 'EUR',
+        '£': 'GBP',
+        'AU$': 'AUD',
+        'C$': 'CAD',
+    }
+    currency_regexp = re.compile(r'-?(?P<currency>\D*)\d.*')
+
+    def __init__(self, input_file):
+        # Orders.csv file is in UTF-8-SIG encoding and begins with a byte order
+        # mark character \ufeff. This happens *after* NEEDED_FIELDS is
+        # checked. This encoding check is done conditionally to handle the
+        # situation where someone has modified the CSV and saved as UTF-8.
+        first_char = input_file.read(1)
+        input_file.seek(0)
+        if first_char == '\ufeff':
+            input_file.reconfigure(encoding='UTF-8-SIG')
+        super().__init__(input_file)
+
+    def _read_row(self, row):
+        if row['Date'].startswith('Total paid'):
+            return None
+
+        # Parse the currency values. A hyphen represents zero.
+        record = {
+            self.DECIMAL_FIELDS[key]: strparse.currency_decimal(row[key] if row[key] != '-' else '0')
+            for key in self.DECIMAL_FIELDS
+        }
+
+        # Most importers don't deal with multiple currencies. There are
+        # libraries that partly help with currency codes and symbols, but the
+        # codes/symbols Printful use have some quirks, so it's easier to do this
+        # ourselves.
+        currency_code = re.match(self.currency_regexp, row['Total']).group('currency')
+        record['currency'] = self.CURRENCY_MAP.get(currency_code, currency_code).replace('$', '')
+
+        record['date'] = strparse.date(row['Date'], '%B %d, %Y')
+        record['order_id'] = row['Order'].lstrip('Order ')
+        record['payee'] = 'Printful'  # Not used but required to be set
+        return record

import2ledger/importers/stripe.py

@@ -1,15 +1,14 @@
-import decimal
-
 from . import _csv
 from .. import strparse
 
+
 class PaymentImporter(_csv.CSVImporterBase):
     NEEDED_FIELDS = frozenset([
         'Converted Currency',
         'Created (UTC)',
         'Fee',
         'Status',
-        'Tax',
+        'Taxes On Fee',
     ])
     COPIED_FIELDS = {
         'Card Name': 'payee',
@@ -31,7 +30,7 @@ class PaymentImporter(_csv.CSVImporterBase):
                 'currency': row['Converted Currency'].upper(),
                 'date': strparse.date(date_s, self.DATE_FMT),
                 'fee': strparse.currency_decimal(row['Fee']),
-                'tax': strparse.currency_decimal(row['Tax']),
+                'tax': strparse.currency_decimal(row['Taxes On Fee']),
             }
 
 

setup.py

@@ -30,7 +30,7 @@ REQUIREMENTS['tests_require'] = [
 setup(
     name='import2ledger',
     description="Import different sources of financial data to Ledger",
-    version='1.3.1',
+    version='1.4.5',
     author='Brett Smith',
     author_email='brettcsmith@brettcsmith.org',
     license='GNU AGPLv3+',

tests/data/PatreonEarnings2020.csv

@@ -0,0 +1,4 @@
+Date,Event ID,Event Type,Creator Currency,Creator Share,Creator Platform Fee,Creator Payment Processing Fee,Creator Currency Conversion Fee,Creator Equivalent of Patron Tax,Currency Conversion Rate,Patron Currency,Patron Charge Amount,Patron Tax Amount,Patron User ID,Patron Name,Patron Email
+2020-12-03 04:05:06,30123456,Payment,USD,4.31,0.25,0.44,,1.00,,USD,6.00,1.00,32101234,Frances Jones,fjones@example.com
+2020-12-02 03:04:05,34567890,Payment,USD,5.14,0.31,0.42,0.19,1.27,1212240,EUR,6.05,1.05,2345678,Barbara Johnson,bjohnson@example.com
+2020-12-01 02:03:04,456789090,Payment,USD,12.64,0.70,0.66,,,,USD,14.00,,7654321,Alex Jones,ajones@example.com

tests/data/imports.yml

@@ -5,10 +5,12 @@
       date: !!python/object/apply:datetime.date [2017, 9, 1]
       amount: !!python/object/apply:decimal.Decimal ["1500.00"]
       currency: USD
+      email: alex@example.org
     - payee: Dakota Doe
       date: !!python/object/apply:datetime.date [2017, 9, 1]
       amount: !!python/object/apply:decimal.Decimal ["12.00"]
       currency: USD
+      email: ddoe@example.org
 
 - source: PatreonPatronReport_2020-08-01.csv
   importer: patreon.Income2020AugustImporter
@@ -76,6 +78,46 @@
       amount: !!python/object/apply:decimal.Decimal ["117.03"]
       currency: USD
 
+- source: PatreonEarnings2020.csv
+  importer: patreon.EarningsImporter
+  expect:
+    - payee: Frances Jones
+      email: fjones@example.com
+      user_id: "32101234"
+      event_id: "30123456"
+      date: !!python/object/apply:datetime.date [2020, 12, 3]
+      currency: USD
+      amount: !!python/object/apply:decimal.Decimal ["5"]
+      tax_amount: !!python/object/apply:decimal.Decimal ["1"]
+      net_amount: !!python/object/apply:decimal.Decimal ["4.31"]
+      platform_fee: !!python/object/apply:decimal.Decimal ["0.25"]
+      processing_fee: !!python/object/apply:decimal.Decimal ["0.44"]
+      currency_fee: !!python/object/apply:decimal.Decimal ["0"]
+    - payee: Barbara Johnson
+      email: bjohnson@example.com
+      user_id: "2345678"
+      event_id: "34567890"
+      date: !!python/object/apply:datetime.date [2020, 12, 2]
+      currency: USD
+      amount: !!python/object/apply:decimal.Decimal ["6.06"]
+      tax_amount: !!python/object/apply:decimal.Decimal ["1.27"]
+      net_amount: !!python/object/apply:decimal.Decimal ["5.14"]
+      platform_fee: !!python/object/apply:decimal.Decimal ["0.31"]
+      processing_fee: !!python/object/apply:decimal.Decimal ["0.42"]
+      currency_fee: !!python/object/apply:decimal.Decimal ["0.19"]
+    - payee: Alex Jones
+      email: ajones@example.com
+      user_id: "7654321"
+      event_id: "456789090"
+      date: !!python/object/apply:datetime.date [2020, 12, 1]
+      currency: USD
+      amount: !!python/object/apply:decimal.Decimal ["14"]
+      tax_amount: !!python/object/apply:decimal.Decimal ["0"]
+      net_amount: !!python/object/apply:decimal.Decimal ["12.64"]
+      platform_fee: !!python/object/apply:decimal.Decimal [".7"]
+      processing_fee: !!python/object/apply:decimal.Decimal [".66"]
+      currency_fee: !!python/object/apply:decimal.Decimal ["0"]
+
 - source: PatreonEarnings.csv
   importer: patreon.CardFeesImporter
   expect: