Restructure the importers to be more consistent

src/core.clj

@@ -25,7 +25,8 @@
    [nil "--pay-invoice-no REFERENCE" "Payroll invoice number, eg. \"rt:111/222\""
     :default "FIXME"]
    [nil "--total-fees NUM" "Total fee charged by Paychex, eg. \"206.50\""
-    :default (bigdec 0)]
+    :parse-fn bigdec
+    :default 0M]
    [nil "--fees-receipt-no REFERENCE" "Paychex fees receipt number, eg. \"rt:111/222\""
     :default "FIXME"]
    [nil "--fees-invoice-no REFERENCE" "Paychex fees invoice number, eg. \"rt:111/222\""
@@ -46,11 +47,11 @@
       (println summary)
       (System/exit 0))
     (let [grouped-data (import/read-grouped-csv (:csv options))
-          imported (concat [(import/import-monthly-payroll pay-date period pay-receipt-no grouped-data)]
+          imported (concat (import/payroll pay-date period pay-receipt-no grouped-data)
-                           (import/import-individual-taxes pay-date period pay-receipt-no pay-invoice-no grouped-data)
+                           (import/individual-taxes pay-date period pay-receipt-no pay-invoice-no grouped-data)
-                           [(import/import-employer-taxes pay-date period pay-receipt-no grouped-data)]
+                           (import/employer-taxes pay-date period pay-receipt-no grouped-data)
-                           [(import/payroll-fees pay-date period fees-receipt-no fees-invoice-no total-fees grouped-data)]
+                           (import/payroll-fees pay-date period fees-receipt-no fees-invoice-no total-fees grouped-data)
-                           [(import/import-retirement retirement-date period retirement-receipt-no retirement-invoice-no grouped-data)])]
+                           (import/retirement retirement-date period retirement-receipt-no retirement-invoice-no grouped-data))]
       (doseq [i imported]
         (println (import/render-transaction i))))))
 
@@ -58,17 +59,30 @@
   (require '[examples :as examples])
   (def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2023-12-27_Pay-Item-Details_2023-12-2.csv"))
   (def imported
-    (concat [(import/import-monthly-payroll "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)]
+    (concat (import/payroll "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)
-            (import/import-individual-taxes "2023-12-29" "December 2023" "rt:19462/674660" "rt:19403/675431" grouped-data)
+            (import/individual-taxes "2023-12-29" "December 2023" "rt:19462/674660" "rt:19403/675431" grouped-data)
-            [(import/import-employer-taxes "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)]
+            (import/employer-taxes "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)
-            [(import/payroll-fees "2023-12-29" "December 2023" "rt:19459/675387" "rt:19459/674887" (bigdec 206.50) grouped-data)]
+            (import/payroll-fees "2023-12-29" "December 2023" "rt:19459/675387" "rt:19459/674887" 206.50M grouped-data)
-            [(import/import-retirement "2024-01-02" "December 2023" "rt:19403/676724" "rt:19403/675431" grouped-data)]))
+            (import/retirement "2024-01-02" "December 2023" "rt:19403/676724" "rt:19403/675431" grouped-data)))
-
   (dd/pretty-print
    (dd/diff
     (sort-postings (parse examples/human))
     (sort-postings imported)))
 
+  (require '[examples2 :as examples])
+  (def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2024-01-29_Pay-Item-Details_2024-01.csv"))
+  (def imported
+    (concat (import/payroll "2024-01-31" "January 2024" "rt:19462/685751" grouped-data)
+            (import/individual-taxes "2024-01-31" "January 2024" "rt:19462/685751" "rt:19403/685602" grouped-data)
+            ;; TODO: Needs fixes for the correct :entity and :memo
+            (import/employer-taxes "2024-01-31" "January 2024" "rt:19462/685751" grouped-data)
+            #_(import/payroll-fees "2024-01-31" "January 2024" "rt:19459/675387" "rt:19459/674887" 206.50M grouped-data)
+            (import/retirement "2024-01-31" "January 2024" "rt:19403/685929" "rt:19403/685602" grouped-data)))
+  (dd/pretty-print
+   (dd/diff
+    (sort-postings (parse examples2/human))
+    (sort-postings imported)))
+
   (doseq [i imported]
     (println (import/render-transaction i)))
 

src/import.clj

@@ -9,36 +9,6 @@
       (str/replace " " "-")
       keyword))
 
-(defn update-vals-subset [m ks f]
-  (reduce #(update %1 %2 f) m ks))
-
-(defn bigdec-or-zero [s]
-  (if (str/blank? s)
-    (bigdec 0)
-    (bigdec s)))
-
-;; TODO: Consider merging into a single amount column.
-(defn prep-csv-data [records]
-  (let [headers [:org :name :id :category :type :t-earnings :_ :t-earnings-match :_ :t-reimbursement :t-retirement :_ :t-withholding :t-liability :_ :t-net-pay]
-        map-records (map zipmap (repeat headers) records)
-        cleaned-records (->> map-records
-                             (map #(-> %
-                                       (dissoc :org :_)
-                                       (update-vals-subset [:t-earnings
-                                                            :t-earnings-match
-                                                            :t-reimbursement
-                                                            :t-retirement
-                                                            :t-withholding
-                                                            :t-liability
-                                                            :t-net-pay]
-                                                           bigdec-or-zero))))]
-    cleaned-records))
-
-(defn read-grouped-csv [filename]
-  (with-open [reader (io/reader filename)]
-    (doall
-     (group-by :name (import/prep-csv-data (csv/read-csv reader))))))
-
 (defn format-name [name]
   (case name
     "Sharp, Sage A" "Sharp-Sage-A"
@@ -47,14 +17,27 @@
         (str/replace ", " "-")
         (str/replace #" \w$" ""))))
 
-(defn render-transaction [trans]
+(defn read-grouped-csv [filename]
-  (str/join (concat [(format "%s txn \"%s\"\n" (:date trans) (:desc trans))]
+  (with-open [reader (io/reader filename)]
-                    (for [[k v] (seq (:meta trans))]
+    (doall
+     (group-by
+      :name
+      (for [[_ name _ category type & totals] (csv/read-csv reader)]
+        {:name (import/format-name name)
+         :category category
+         :type type
+         :amount (apply max (map bigdec (remove str/blank? totals)))})))))
+
+(defn render-transaction [{:keys [date payee desc meta postings]}]
+  (str/join (concat [(if payee
+                       (format "%s txn \"%s\" \"%s\"\n" date payee desc)
+                       (format "%s txn \"%s\"\n" date desc))]
+                    (for [[k v] meta]
                       (format "  %s: \"%s\"\n" (name k) v))
-                    (for [posting (:postings trans)]
+                    (for [{:keys [amount account currency meta]} postings]
-                      (when (not (zero? (:amount posting)))
+                      (when (not (zero? amount))
-                        (format "  %-40s  %10.2f %s\n%s" (:account posting) (:amount posting) (:currency posting)
+                        (format "  %-40s  %10.2f %s\n%s" account amount currency
-                                (str/join (for [[k v] (seq (:meta posting))]
+                                (str/join (for [[k v] meta]
                                             (format "     %s: \"%s\"\n" (name k) v)))))))))
 
 (defn cat->acct [cat]
@@ -78,196 +61,199 @@
     "IL Unemploy" "IL:Unemployment"
     "NY Unemploy" "NY:Unemployment"
     "OR Unemploy" "OR:Unemployment"
+    "NY Re-empl Svc" "US:NY:Reempt"
     cat))
 
-;; TODO: How do we know we used all the relevant data from the report? Didn't
+(defn employee-entity-meta [name]
-;; miss anything?
+  (if (= name "Sharp-Sage-A")
+    {:entity name
+     :project "Outreachy"}
+    {:entity name}))
 
-(defn import-monthly-payroll [date period receipt-no groups]
+(defn project [name]
-  {:date date :desc (format "Monthly Payroll - %s - Net Pay" period)
+  (if (= name "Sharp-Sage-A")
+    "Outreachy"
+    "Conservancy"))
+
+(defn split-fee
+  "Share a total fee into n groups allocating the remainder as evenly as possible."
+  [total n]
+  (let [total (bigdec total)
+        fee-share (.setScale (bigdec (/ (double total) n)) 2 java.math.RoundingMode/FLOOR)
+        extra-cents (* 100M (- total (* fee-share n)))
+        base-fee-allocation (repeat fee-share)
+        cents-allocation (take n (concat (repeat extra-cents 0.01M) (repeat 0)))]
+    (map + base-fee-allocation cents-allocation)))
+
+(defn payroll
+  "Return a net pay transaction."
+  [date period receipt-no groups]
+  (let [postings (for [[name records] groups]
+                   (let [total-net-pay (->> records
+                                            (filter #(= (:type %) "Net Pay"))
+                                            (map :amount)
+                                            (apply +))
+                         total-reimbursement (->> records
+                                                  (filter #(= (:type %) "Reimbursement"))
+                                                  (map :amount)
+                                                  (apply +))
+                         actual-total-net-pay (- total-net-pay total-reimbursement)]
+                     [{:account "Expenses:Payroll:Salary"
+                       :amount actual-total-net-pay
+                       :currency "USD"
+                       :meta (employee-entity-meta name)}
+                      {:account "Assets:FR:Check2721"
+                       :amount (- actual-total-net-pay)
+                       :currency "USD"
+                       :meta {:entity name}}
+                      {:account "Expenses:Hosting"
+                       :amount total-reimbursement
+                       :currency "USD"
+                       :meta (merge (employee-entity-meta name) {:payroll-type "US:Reimbursement"})}
+                      {:account "Assets:FR:Check2721"
+                       :amount (- total-reimbursement)
+                       :currency "USD"
+                       :meta (merge (employee-entity-meta name) {:tax-implication "Reimbursement"})}]))]
+    [{:date date :desc (format "Monthly Payroll - %s - Net Pay" period)
       :meta {:program "Conservancy:Payroll"
              :project "Conservancy"
              :receipt receipt-no
              :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
              :tax-implication "W2"
              :payroll-type "US:General"}
-   :postings
+      :postings (apply concat postings)}]))
-   (apply concat (for [[name records] groups]
-                   (let [name (format-name name)
-                         total-net-pay (->> records
-                                            (filter #(= (:type %) "Net Pay"))
-                                            (map :t-net-pay)
-                                            (apply +)
-                                            bigdec)
-                         total-reimbursement (->> records
-                                                  (filter #(= (:type %) "Reimbursement"))
-                                                  (map :t-reimbursement)
-                                                  (apply +)
-                                                  bigdec)
-                         total-net-pay-less-reimb (- total-net-pay total-reimbursement)
-                         pay-exp-trans [{:account "Expenses:Payroll:Salary"
-                                         :amount total-net-pay-less-reimb
-                                         :currency "USD"
-                                         :meta (if (= name "Sharp-Sage-A")
-                                                 {:entity name
-                                                  :project "Outreachy"}
-                                                 {:entity name})}
-                                        {:account "Assets:FR:Check2721"
-                                         :amount (- total-net-pay-less-reimb)
-                                         :currency "USD"
-                                         :meta {:entity name}}]
-                         reimbursement-exp-trans [{:account "Expenses:Hosting"
-                                                   :amount total-reimbursement
-                                                   :currency "USD"
-                                                   :meta (if (= name "Sharp-Sage-A")
-                                                           {:entity name
-                                                            :project "Outreachy"
-                                                            :payroll-type "US:Reimbursement"}
-                                                           {:entity name
-                                                            :payroll-type "US:Reimbursement"})}
-                                                  {:account "Assets:FR:Check2721"
-                                                   :amount (- total-reimbursement)
-                                                   :currency "USD"
-                                                   :meta (if (= name "Sharp-Sage-A")
-                                                           {:entity name
-                                                            :project "Outreachy"
-                                                            :tax-implication "Reimbursement"}
-                                                           {:entity name
-                                                            :tax-implication "Reimbursement"})}]]
-                     (concat pay-exp-trans reimbursement-exp-trans))))})
 
-(defn import-individual-taxes [date period receipt-no invoice-no groups]
+(defn individual-taxes
-  ;; Print the individual taxes blocks
+  "Return a transaction of expenses/witholding for each employee."
+  [date period receipt-no invoice-no groups]
   (for [[name records] groups]
-    {:date date :desc (format "Monthly Payroll - %s - TAXES - %s" period (format-name name))
-   :meta {:project (if (= (format-name name) "Sharp-Sage-A") "Outreachy" "Conservancy")
-          :program "Conservancy:Payroll"
-          :entity (format-name name)
-          :receipt receipt-no
-          :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
-     :postings
     (let [super-lines (filter #(str/starts-with? (:category %) "403b") records)
           ;; TODO: Have I got the liability/witholding right? Which is used in which report.
           witholding-lines (filter #(= (:type %) "Withholding") records)
+          ;; TODO: We seem to add these extra insurance lines for Karen (NY) only. Confirm with Rosanne.
           insurance-lines (filter #(and (= (:type %) "Withholding")
                                         (str/includes? (:category %) "NY Disability")) records)
           total-super (->> super-lines
-                            (map :t-retirement)
+                           (map :amount)
-                            (apply +)
+                           (apply +))
-                            bigdec)]
+          super-postings (for [{:keys [category amount]} super-lines]
-       (concat
+                           (if (= category "403b ER match")
-        (for [x super-lines]
-          (if (= (:category x) "403b ER match")
                              {:account "Expenses:Payroll:Salary"
-             :amount (:t-retirement x)
+                              :amount amount
                               :currency "USD"
                               :meta {:payroll-type "US:403b:Match"
                                      :invoice invoice-no}}
                              {:account "Expenses:Payroll:Salary"
-             :amount (:t-retirement x)
+                              :amount amount
                               :currency "USD"
                               :meta {:payroll-type "US:403b:Employee"
                                      :invoice invoice-no}}))
-        [{:account "Liabilities:Payable:Accounts"
+          liability-postings [{:account "Liabilities:Payable:Accounts"
                                :amount (- total-super)
                                :currency "USD"
                                :meta {:invoice invoice-no}}]
-        (conj
+          withholding-postings (for [{:keys [category amount]} witholding-lines]
-         (vec (for [x witholding-lines]
                                  {:account "Expenses:Payroll:Salary"
-                 :amount (:t-withholding x)
+                                  :amount amount
                                   :currency "USD"
-                 :meta {:payroll-type (cat->acct (:category x))}}))
+                                  :meta {:payroll-type (cat->acct category)}})
-         {:account "Assets:FR:Check2721"
+          withholding-asset-postings [{:account "Assets:FR:Check2721"
-          :amount (- (reduce + (map :t-withholding witholding-lines)))
+                                       :amount (- (reduce + (map :amount witholding-lines)))
                                        :currency "USD"
-          :meta {:tax-implication "W2"}})
+                                       :meta {:tax-implication "W2"}}]
-        ;; TODO: We seem to add these extra insurance lines for Karen (only). Confirm with Rosanne.
+          insurance-postings (for [{:keys [category amount]} insurance-lines]
-        (conj
-         (vec (for [x insurance-lines]
                                {:account "Expenses:Insurance"
-                 :amount (- (:t-withholding x))
+                                :amount (- amount)
                                 :currency "USD"
-                 :meta {:payroll-type (cat->acct (:category x))}}))
+                                :meta {:payroll-type (cat->acct category)}})
-         {:account "Assets:FR:Check2721"
+          insurance-asset-postings [{:account "Assets:FR:Check2721"
-          :amount (bigdec (reduce + (map :t-withholding insurance-lines)))
+                                     :amount (reduce + (map :amount insurance-lines))
-          :currency "USD"})))}))
+                                     :currency "USD"}]]
+      {:date date :desc (format "Monthly Payroll - %s - TAXES - %s" period name)
+       :meta {:project (project name)
+              :program "Conservancy:Payroll"
+              :entity name
+              :receipt receipt-no
+              :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
+       :postings (concat
+                  super-postings
+                  liability-postings
+                  withholding-postings
+                  withholding-asset-postings
+                  insurance-postings
+                  insurance-asset-postings)})))
 
-(defn import-employer-taxes [date period receipt-no groups]
+(defn employer-taxes
-  {:date date :desc (format "Monthly Payroll - %s - TAXES - Employer" period)
+  "Return an employer taxes transaction."
+  [date period receipt-no groups]
+  (let [liability-postings (apply concat
+                                  (for [[name records] groups]
+                                    (let [liability-lines (filter #(= (:type %) "Liability") records)]
+                                      (for [{:keys [category amount]} liability-lines]
+                                        {:account "Expenses:Payroll:Tax"
+                                         :amount amount
+                                         :currency "USD"
+                                         :meta (merge
+                                                (employee-entity-meta name)
+                                                ;; TODO: Check lack of ":Tax:" with Rosanne.
+                                                {:payroll-type (str/replace (cat->acct category) "Tax:" "")})}))))
+        total-liabilities (->> liability-postings (map :amount) (reduce +))
+        asset-postings [{:account "Assets:FR:Check2721"
+                         :amount (- total-liabilities)
+                         :currency "USD"
+                         :meta {:entity "Paychex"
+                                :tax-implication "Tax-Payment"}}]]
+    [{:date date :desc (format "Monthly Payroll - %s - TAXES - Employer" period)
       :meta {:program "Conservancy:Payroll"
              :project "Conservancy"
              :receipt receipt-no
              :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
-   :postings
+      :postings (concat liability-postings asset-postings)}]))
-   (let [total-emp-tax (reduce + (map :t-liability (apply concat (vals groups))))]
-     (conj (vec (apply concat (for [[name records] groups]
-                                (let [name (format-name name)
-                                      liability-lines (filter #(= (:type %) "Liability") records)]
-                                  (for [x liability-lines]
-                                    {:account "Expenses:Payroll:Tax"
-                                     :amount (:t-liability x)
-                                     :currency "USD"
-                                     :meta (if (= name "Sharp-Sage-A")
-                                             {:entity name
-                                              :project "Outreachy"
-                                              ;; TODO: Check lack of ":Tax:" with Rosanne.
-                                              :payroll-type (str/replace (cat->acct (:category x)) "Tax:" "")}
-                                             {:entity name
-                                              :payroll-type (str/replace (cat->acct (:category x)) "Tax:" "")})})))))
-           {:account "Assets:FR:Check2721"
-            :amount (- total-emp-tax)
-            :currency "USD"
-            :meta {:entity "Paychex"
-                   :tax-implication "Tax-Payment"}}))})
 
-(defn payroll-fees [date period receipt-no invoice-no total-fees groups]
+(defn payroll-fees
-  (let [employees (map import/format-name (keys groups))
+  "Return a payroll fees transaction."
-        num-employees (count employees)
+  [date period receipt-no invoice-no total-fees groups]
-        fee-share (.setScale (/ total-fees num-employees) 2 java.math.RoundingMode/FLOOR)
+  (let [employees (keys groups)
-        extra-cents (* 100 (- total-fees (* fee-share num-employees)))
+        exact-fee-allocation (split-fee total-fees (count employees))
-        base-fee-allocation (repeat fee-share)
+        employee-fees (map vector employees exact-fee-allocation)
-        cents-allocation (concat (repeat extra-cents (bigdec 0.01)) (repeat 0))
+        expense-postings (for [[name fee] employee-fees]
-        exact-fee-allocation (reverse (take num-employees (map + base-fee-allocation cents-allocation)))
-        expense-postings (for [[name fee] (map vector employees exact-fee-allocation)]
                            {:account "Expenses:Payroll:Fees"
                             :amount fee
                             :currency "USD"
-                            :meta {:entity name}})
+                            :meta (employee-entity-meta name)})
-        asset-posting {:account "Assets:FR:Check2721"
+        asset-postings [{:account "Assets:FR:Check2721"
                          :amount (- total-fees)
-                       :currency "USD"}
+                         :currency "USD"}]]
-        all-postings (conj (vec expense-postings) asset-posting)]
+    [{:date date :payee "Paychex" :desc (format "Monthly Payroll - %s - Fee" period)
-    {:date date :payee "Paychex" :desc (format "Monthly Payroll - %s - Fee" period)
       :meta {:program "Conservancy:Payroll"
              :project "Conservancy"
              :receipt receipt-no
              :invoice invoice-no
              :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
              :tax-implication "USA-Corporation"}
-     :postings all-postings}))
+      :postings (concat expense-postings asset-postings)}]))
 
-(defn import-retirement [date period receipt-no invoice-no groups]
+(defn retirement
-  {:date date :desc (format "ASCENSUS TRUST RET PLAN - ACH DEBIT - Vanguard 403(b) - %s" period)
+  "Return a retirement transaction."
+  [date period receipt-no invoice-no groups]
+  (let [liability-postings (for [[name records] groups]
+                             (let [total-retirement (->> records
+                                                         (filter #(= (:type %) "Retirement"))
+                                                         (map :amount)
+                                                         (reduce +))]
+                               {:account "Liabilities:Payable:Accounts",
+                                :amount total-retirement,
+                                :currency "USD",
+                                :meta {:entity name}}))
+        total-liabilities (->> liability-postings (map :amount) (reduce +))
+        asset-postings [{:account "Assets:FR:Check1345"
+                         :amount (- total-liabilities)
+                         :currency "USD"}]]
+    [{:date date :desc (format "ASCENSUS TRUST RET PLAN - ACH DEBIT - Vanguard 403(b) - %s" period)
       :meta {:program "Conservancy:Payroll"
              :project "Conservancy"
              :receipt receipt-no
              :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
              :tax-implication "Retirement-Pretax"
              :invoice invoice-no}
-   :postings
+      :postings (concat liability-postings asset-postings)}]))
-   (let [total-retirement (reduce + (map :t-retirement (apply concat (vals groups))))]
-     (conj (vec (for [[name records] groups]
-                  (let [name (format-name name)
-                        total-retirement (->> records
-                                              (filter #(= (:type %) "Retirement"))
-                                              (map :t-retirement)
-                                              (apply +)
-                                              bigdec)]
-                    {:account "Liabilities:Payable:Accounts",
-                     :amount total-retirement,
-                     :currency "USD",
-                     :meta {:entity name}})))
-           {:account "Assets:FR:Check1345"
-            :amount (- total-retirement)
-            :currency "USD"}))})