benevity: Support new 2019 report format.

README.rst

@@ -127,11 +127,26 @@ Benevity
   ---------------- -----------------------------------------------------------
   donation_amount  The amount donated by the individual donor named
   ---------------- -----------------------------------------------------------
+  donation_fee     The fee taken from the donation.
+                   This information was not available in reports before 2019,
+                   so this amount will always be 0.00 in those reports.
+  ---------------- -----------------------------------------------------------
   frequency        The frequency of this donation as indicated in the report
   ---------------- -----------------------------------------------------------
   match_amount     The amount of the donation match by the participating
                    corporation
   ---------------- -----------------------------------------------------------
+  match_fee        The fee taken from the donation match.
+                   This information was not available in reports before 2019,
+                   so this amount will always be 0.00 in those reports.
+  ---------------- -----------------------------------------------------------
+  merchant_fee     The merchant fee taken from the donation total.
+                   This information was not available in reports before 2019,
+                   so this amount will always be 0.00 in those reports.
+  ---------------- -----------------------------------------------------------
+  net_amount       The net amount Benevity owes the charity for this
+                   transaction: the donation plus the match minus the fees
+  ---------------- -----------------------------------------------------------
   project          The Project named in this row
   ---------------- -----------------------------------------------------------
   transaction_id   The ID of this specific donation

import2ledger/importers/benevity.py

@@ -1,35 +1,12 @@
+import decimal
+
 from . import _csv
 from .. import strparse
 
-class Donations2018Importer(_csv.CSVImporterBase):
+ZERO_DECIMAL = decimal.Decimal(0)
-    ENTRY_SEED = {
+
-        'ledger template': 'benevity donations ledger entry',
+class _DonationsImporterBase(_csv.CSVImporterBase):
-    }
-    HEADER_FIELDS = {
-        'Currency': 'currency',
-        'Disbursement ID': 'disbursement_id',
-        'Reference': 'reference',
-    }
-    DATE_FIELD = 'Date of Donation'
     NAME_FIELDS = ['Donor First Name', 'Donor Last Name']
-    DECIMAL_FIELDS = {
-        'Donation Amount': 'donation_amount',
-        'Matched Amount': 'match_amount',
-        'Total': 'amount',
-    }
-    NEEDED_FIELDS = frozenset([
-        DATE_FIELD,
-        *NAME_FIELDS,
-        *DECIMAL_FIELDS,
-    ])
-    COPIED_FIELDS = {
-        'Participating Corporation': 'corporation',
-        'Project': 'project',
-        'Comment': 'comment',
-        'Transaction ID': 'transaction_id',
-        'Donation Frequency': 'frequency',
-    }
-    DATE_FMT = '%Y-%m-%d'
     NOT_SHARED = 'Not shared by donor'
 
     @classmethod
@@ -43,18 +20,83 @@ class Donations2018Importer(_csv.CSVImporterBase):
             return {}
 
     def _read_row(self, row):
-        try:
+        date_s = row.get(self.DATE_FIELD)
-            date = strparse.date(row[self.DATE_FIELD], self.DATE_FMT)
+        if not date_s:
-        except ValueError:
             return None
         if all(row[key] == self.NOT_SHARED for key in self.NAME_FIELDS):
             payee = 'Anonymous'
         else:
             payee = ' '.join(row[key] for key in self.NAME_FIELDS)
         retval = {
-            'date': date,
+            'date': strparse.date(date_s, self.DATE_FMT),
             'payee': payee,
         }
         retval.update((retkey, strparse.currency_decimal(row[rowkey]))
                       for rowkey, retkey in self.DECIMAL_FIELDS.items())
+        retval['amount'] = retval['donation_amount'] + retval['match_amount']
         return retval
+
+
+class Donations2018Importer(_DonationsImporterBase):
+    ENTRY_SEED = {
+        'ledger template': 'benevity donations ledger entry',
+        'donation_fee': ZERO_DECIMAL,
+        'match_fee': ZERO_DECIMAL,
+        'merchant_fee': ZERO_DECIMAL,
+    }
+    DATE_FIELD = 'Date of Donation'
+    DATE_FMT = '%Y-%m-%d'
+    HEADER_FIELDS = {
+        'Currency': 'currency',
+        'Disbursement ID': 'disbursement_id',
+        'Reference': 'reference',
+    }
+    DECIMAL_FIELDS = {
+        'Donation Amount': 'donation_amount',
+        'Matched Amount': 'match_amount',
+        'Total': 'net_amount',
+    }
+    NEEDED_FIELDS = frozenset([
+        DATE_FIELD,
+        *_DonationsImporterBase.NAME_FIELDS,
+        *DECIMAL_FIELDS,
+    ])
+    COPIED_FIELDS = {
+        'Participating Corporation': 'corporation',
+        'Project': 'project',
+        'Comment': 'comment',
+        'Transaction ID': 'transaction_id',
+        'Donation Frequency': 'frequency',
+    }
+
+
+class Donations2019Importer(_DonationsImporterBase):
+    ENTRY_SEED = {
+        'ledger template': 'benevity donations ledger entry',
+    }
+    DATE_FIELD = 'Donation Date'
+    DATE_FMT = '%Y-%m-%dT%H:%M:%SZ'
+    HEADER_FIELDS = {
+        'Disbursement ID': 'disbursement_id',
+    }
+    DECIMAL_FIELDS = {
+        'Donation Amount': 'donation_amount',
+        'Donation Fee': 'donation_fee',
+        'Match Amount': 'match_amount',
+        'Match Fee': 'match_fee',
+        'Merchant Fee': 'merchant_fee',
+        'Net total': 'net_amount',
+    }
+    NEEDED_FIELDS = frozenset([
+        DATE_FIELD,
+        *_DonationsImporterBase.NAME_FIELDS,
+        *DECIMAL_FIELDS,
+    ])
+    COPIED_FIELDS = {
+        'Company': 'corporation',
+        'Currency': 'currency',
+        'Project': 'project',
+        'Comment': 'comment',
+        'Transaction ID': 'transaction_id',
+        'Donation Frequency': 'frequency',
+    }

tests/data/Benevity2019.csv

@@ -0,0 +1,22 @@
+Donations Report,
+"#-------------------------------------------",
+Charity Name,"Software Freedom Conservancy, Inc."
+Charity ID,850-607080900
+Period Ending,Tue 1 Jan 2019 0:00:00
+Currency,USD
+Payment Method,EFT
+Disbursement ID,ABCDE12345
+Note,Rounding may be applied to some values in this report. Learn more at https://causes.benevity.org/feedback-support
+"#-------------------------------------------",
+
+Company,Project,Donation Date,Donor First Name,Donor Last Name,Email,Address,City,State/Province,Postal Code,Activity,Comment,Transaction ID,Donation Frequency,Currency,Donation Amount,Donation Fee,Match Amount,Match Fee,Merchant Fee,Net total
+Company A,,2017-10-28T12:00:01Z,Dakota,Smith,a@example.com,Not shared by donor,Not shared by donor,Not shared by donor,00000,Volunteering,,67890QWERT,One-time,USD,20.00,0.00,0.00,0.00,0.00,20.00
+Company A,,2017-10-30T12:00:03Z,Dakota,Smith,a@example.com,Not shared by donor,Not shared by donor,Not shared by donor,00000,Volunteering,,67890WERTY,One-time,USD,25.00,0.00,0.00,0.00,0.00,25.00
+Company B,,2017-10-19T12:00:05Z,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,12345,,,67890ERTYU,Unknown,USD,0.00,0.00,10.00,0.00,0.00,10.00
+Company B,,2017-10-19T12:00:07Z,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,12345,,,67890RTYUI,Unknown,USD,0.00,0.00,20.00,0.00,0.00,20.00
+Company B,,2017-10-19T12:00:09Z,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,12345,,,67890TYUIO,Recurring,USD,30.00,0.00,0.00,0.00,0.00,30.00
+Company C,,2017-10-18T12:00:11Z,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,12345,,,67890YUIOP,Unknown,USD,10.00,0.10,10.00,0.20,0.15,19.55
+Company C,,2017-10-21T12:00:13Z,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,Not shared by donor,12345,,,67890ASDFG,Unknown,USD,10.00,0.10,10.00,0.20,0.15,19.55
+Totals,,,,,,,,,,,,,,,95.00,0.20,50.00,0.40,0.30,144.10
+Check Fee,0.00
+Total Donations (Gross),,,,,,,,,,,,,,,,144.10

tests/data/imports.yml

@@ -281,8 +281,13 @@
       currency: USD
       disbursement_id: ABCDE12345
       amount: !!python/object/apply:decimal.Decimal [20]
+      net_amount: !!python/object/apply:decimal.Decimal [20]
       donation_amount: !!python/object/apply:decimal.Decimal [20]
       match_amount: !!python/object/apply:decimal.Decimal [0]
+      match_amount: !!python/object/apply:decimal.Decimal [0]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
       payee: Dakota Smith
       corporation: Company A
       project: ""
@@ -294,8 +299,12 @@
       currency: USD
       disbursement_id: ABCDE12345
       amount: !!python/object/apply:decimal.Decimal [25]
+      net_amount: !!python/object/apply:decimal.Decimal [25]
       donation_amount: !!python/object/apply:decimal.Decimal [25]
       match_amount: !!python/object/apply:decimal.Decimal [0]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
       payee: Dakota Smith
       corporation: Company A
       project: ""
@@ -307,8 +316,12 @@
       currency: USD
       disbursement_id: ABCDE12345
       amount: !!python/object/apply:decimal.Decimal [10]
+      net_amount: !!python/object/apply:decimal.Decimal [10]
       donation_amount: !!python/object/apply:decimal.Decimal [0]
       match_amount: !!python/object/apply:decimal.Decimal [10]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
       payee: Anonymous
       corporation: Company B
       project: ""
@@ -320,8 +333,12 @@
       currency: USD
       disbursement_id: ABCDE12345
       amount: !!python/object/apply:decimal.Decimal [20]
+      net_amount: !!python/object/apply:decimal.Decimal [20]
       donation_amount: !!python/object/apply:decimal.Decimal [0]
       match_amount: !!python/object/apply:decimal.Decimal [20]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
       payee: Anonymous
       corporation: Company B
       project: ""
@@ -333,15 +350,142 @@
       currency: USD
       disbursement_id: ABCDE12345
       amount: !!python/object/apply:decimal.Decimal [30]
+      net_amount: !!python/object/apply:decimal.Decimal [30]
       donation_amount: !!python/object/apply:decimal.Decimal [30]
       match_amount: !!python/object/apply:decimal.Decimal [0]
       payee: Anonymous
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
+      corporation: Company B
+      project: ""
+      comment: ""
+      frequency: Recurring
+      transaction_id: 67890TYUIO
+      ledger template: benevity donations ledger entry
+
+- source: Benevity2019.csv
+  importer: benevity.Donations2019Importer
+  expect:
+    - date: !!python/object/apply:datetime.date [2017, 10, 28]
+      currency: USD
+      disbursement_id: ABCDE12345
+      amount: !!python/object/apply:decimal.Decimal [20]
+      net_amount: !!python/object/apply:decimal.Decimal [20]
+      donation_amount: !!python/object/apply:decimal.Decimal [20]
+      match_amount: !!python/object/apply:decimal.Decimal [0]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
+      payee: Dakota Smith
+      corporation: Company A
+      project: ""
+      comment: ""
+      frequency: One-time
+      transaction_id: 67890QWERT
+      ledger template: benevity donations ledger entry
+    - date: !!python/object/apply:datetime.date [2017, 10, 30]
+      currency: USD
+      disbursement_id: ABCDE12345
+      amount: !!python/object/apply:decimal.Decimal [25]
+      net_amount: !!python/object/apply:decimal.Decimal [25]
+      donation_amount: !!python/object/apply:decimal.Decimal [25]
+      match_amount: !!python/object/apply:decimal.Decimal [0]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
+      payee: Dakota Smith
+      corporation: Company A
+      project: ""
+      comment: ""
+      frequency: One-time
+      transaction_id: 67890WERTY
+      ledger template: benevity donations ledger entry
+    - date: !!python/object/apply:datetime.date [2017, 10, 19]
+      currency: USD
+      disbursement_id: ABCDE12345
+      amount: !!python/object/apply:decimal.Decimal [10]
+      net_amount: !!python/object/apply:decimal.Decimal [10]
+      donation_amount: !!python/object/apply:decimal.Decimal [0]
+      match_amount: !!python/object/apply:decimal.Decimal [10]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
+      payee: Anonymous
+      corporation: Company B
+      project: ""
+      comment: ""
+      frequency: Unknown
+      transaction_id: 67890ERTYU
+      ledger template: benevity donations ledger entry
+    - date: !!python/object/apply:datetime.date [2017, 10, 19]
+      currency: USD
+      disbursement_id: ABCDE12345
+      amount: !!python/object/apply:decimal.Decimal [20]
+      net_amount: !!python/object/apply:decimal.Decimal [20]
+      donation_amount: !!python/object/apply:decimal.Decimal [0]
+      match_amount: !!python/object/apply:decimal.Decimal [20]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
+      payee: Anonymous
+      corporation: Company B
+      project: ""
+      comment: ""
+      frequency: Unknown
+      transaction_id: 67890RTYUI
+      ledger template: benevity donations ledger entry
+    - date: !!python/object/apply:datetime.date [2017, 10, 19]
+      currency: USD
+      disbursement_id: ABCDE12345
+      amount: !!python/object/apply:decimal.Decimal [30]
+      net_amount: !!python/object/apply:decimal.Decimal [30]
+      donation_amount: !!python/object/apply:decimal.Decimal [30]
+      match_amount: !!python/object/apply:decimal.Decimal [0]
+      donation_fee: !!python/object/apply:decimal.Decimal [0]
+      match_fee: !!python/object/apply:decimal.Decimal [0]
+      merchant_fee: !!python/object/apply:decimal.Decimal [0]
+      payee: Anonymous
       corporation: Company B
       project: ""
       comment: ""
       frequency: Recurring
       transaction_id: 67890TYUIO
       ledger template: benevity donations ledger entry
+    - date: !!python/object/apply:datetime.date [2017, 10, 18]
+      currency: USD
+      disbursement_id: ABCDE12345
+      amount: !!python/object/apply:decimal.Decimal [20]
+      net_amount: !!python/object/apply:decimal.Decimal ["19.55"]
+      donation_amount: !!python/object/apply:decimal.Decimal [10]
+      match_amount: !!python/object/apply:decimal.Decimal [10]
+      donation_fee: !!python/object/apply:decimal.Decimal [".1"]
+      match_fee: !!python/object/apply:decimal.Decimal [".2"]
+      merchant_fee: !!python/object/apply:decimal.Decimal [".15"]
+      payee: Anonymous
+      corporation: Company C
+      project: ""
+      comment: ""
+      frequency: Unknown
+      transaction_id: 67890YUIOP
+      ledger template: benevity donations ledger entry
+    - date: !!python/object/apply:datetime.date [2017, 10, 21]
+      currency: USD
+      disbursement_id: ABCDE12345
+      amount: !!python/object/apply:decimal.Decimal [20]
+      net_amount: !!python/object/apply:decimal.Decimal ["19.55"]
+      donation_amount: !!python/object/apply:decimal.Decimal [10]
+      match_amount: !!python/object/apply:decimal.Decimal [10]
+      donation_fee: !!python/object/apply:decimal.Decimal [".1"]
+      match_fee: !!python/object/apply:decimal.Decimal [".2"]
+      merchant_fee: !!python/object/apply:decimal.Decimal [".15"]
+      payee: Anonymous
+      corporation: Company C
+      project: ""
+      comment: ""
+      frequency: Unknown
+      transaction_id: 67890ASDFG
+      ledger template: benevity donations ledger entry
 
 - source: BrightFunds.xls
   importer: brightfunds.DonorReportImporter