Restructure the importers to be more consistent

This commit is contained in:
Ben Sturmfels 2024-02-22 01:02:38 +11:00
parent 18eb076df0
commit d77967d61b
Signed by: bsturmfels
GPG key ID: 023C05E2C9C068F0
2 changed files with 228 additions and 228 deletions

View file

@ -25,7 +25,8 @@
[nil "--pay-invoice-no REFERENCE" "Payroll invoice number, eg. \"rt:111/222\""
:default "FIXME"]
[nil "--total-fees NUM" "Total fee charged by Paychex, eg. \"206.50\""
:default (bigdec 0)]
:parse-fn bigdec
:default 0M]
[nil "--fees-receipt-no REFERENCE" "Paychex fees receipt number, eg. \"rt:111/222\""
:default "FIXME"]
[nil "--fees-invoice-no REFERENCE" "Paychex fees invoice number, eg. \"rt:111/222\""
@ -46,11 +47,11 @@
(println summary)
(System/exit 0))
(let [grouped-data (import/read-grouped-csv (:csv options))
imported (concat [(import/import-monthly-payroll pay-date period pay-receipt-no grouped-data)]
(import/import-individual-taxes pay-date period pay-receipt-no pay-invoice-no grouped-data)
[(import/import-employer-taxes pay-date period pay-receipt-no grouped-data)]
[(import/payroll-fees pay-date period fees-receipt-no fees-invoice-no total-fees grouped-data)]
[(import/import-retirement retirement-date period retirement-receipt-no retirement-invoice-no grouped-data)])]
imported (concat (import/payroll pay-date period pay-receipt-no grouped-data)
(import/individual-taxes pay-date period pay-receipt-no pay-invoice-no grouped-data)
(import/employer-taxes pay-date period pay-receipt-no grouped-data)
(import/payroll-fees pay-date period fees-receipt-no fees-invoice-no total-fees grouped-data)
(import/retirement retirement-date period retirement-receipt-no retirement-invoice-no grouped-data))]
(doseq [i imported]
(println (import/render-transaction i))))))
@ -58,17 +59,30 @@
(require '[examples :as examples])
(def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2023-12-27_Pay-Item-Details_2023-12-2.csv"))
(def imported
(concat [(import/import-monthly-payroll "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)]
(import/import-individual-taxes "2023-12-29" "December 2023" "rt:19462/674660" "rt:19403/675431" grouped-data)
[(import/import-employer-taxes "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)]
[(import/payroll-fees "2023-12-29" "December 2023" "rt:19459/675387" "rt:19459/674887" (bigdec 206.50) grouped-data)]
[(import/import-retirement "2024-01-02" "December 2023" "rt:19403/676724" "rt:19403/675431" grouped-data)]))
(concat (import/payroll "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)
(import/individual-taxes "2023-12-29" "December 2023" "rt:19462/674660" "rt:19403/675431" grouped-data)
(import/employer-taxes "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)
(import/payroll-fees "2023-12-29" "December 2023" "rt:19459/675387" "rt:19459/674887" 206.50M grouped-data)
(import/retirement "2024-01-02" "December 2023" "rt:19403/676724" "rt:19403/675431" grouped-data)))
(dd/pretty-print
(dd/diff
(sort-postings (parse examples/human))
(sort-postings imported)))
(require '[examples2 :as examples])
(def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2024-01-29_Pay-Item-Details_2024-01.csv"))
(def imported
(concat (import/payroll "2024-01-31" "January 2024" "rt:19462/685751" grouped-data)
(import/individual-taxes "2024-01-31" "January 2024" "rt:19462/685751" "rt:19403/685602" grouped-data)
;; TODO: Needs fixes for the correct :entity and :memo
(import/employer-taxes "2024-01-31" "January 2024" "rt:19462/685751" grouped-data)
#_(import/payroll-fees "2024-01-31" "January 2024" "rt:19459/675387" "rt:19459/674887" 206.50M grouped-data)
(import/retirement "2024-01-31" "January 2024" "rt:19403/685929" "rt:19403/685602" grouped-data)))
(dd/pretty-print
(dd/diff
(sort-postings (parse examples2/human))
(sort-postings imported)))
(doseq [i imported]
(println (import/render-transaction i)))

View file

@ -9,36 +9,6 @@
(str/replace " " "-")
keyword))
(defn update-vals-subset [m ks f]
(reduce #(update %1 %2 f) m ks))
(defn bigdec-or-zero [s]
(if (str/blank? s)
(bigdec 0)
(bigdec s)))
;; TODO: Consider merging into a single amount column.
(defn prep-csv-data [records]
(let [headers [:org :name :id :category :type :t-earnings :_ :t-earnings-match :_ :t-reimbursement :t-retirement :_ :t-withholding :t-liability :_ :t-net-pay]
map-records (map zipmap (repeat headers) records)
cleaned-records (->> map-records
(map #(-> %
(dissoc :org :_)
(update-vals-subset [:t-earnings
:t-earnings-match
:t-reimbursement
:t-retirement
:t-withholding
:t-liability
:t-net-pay]
bigdec-or-zero))))]
cleaned-records))
(defn read-grouped-csv [filename]
(with-open [reader (io/reader filename)]
(doall
(group-by :name (import/prep-csv-data (csv/read-csv reader))))))
(defn format-name [name]
(case name
"Sharp, Sage A" "Sharp-Sage-A"
@ -47,14 +17,27 @@
(str/replace ", " "-")
(str/replace #" \w$" ""))))
(defn render-transaction [trans]
(str/join (concat [(format "%s txn \"%s\"\n" (:date trans) (:desc trans))]
(for [[k v] (seq (:meta trans))]
(defn read-grouped-csv [filename]
(with-open [reader (io/reader filename)]
(doall
(group-by
:name
(for [[_ name _ category type & totals] (csv/read-csv reader)]
{:name (import/format-name name)
:category category
:type type
:amount (apply max (map bigdec (remove str/blank? totals)))})))))
(defn render-transaction [{:keys [date payee desc meta postings]}]
(str/join (concat [(if payee
(format "%s txn \"%s\" \"%s\"\n" date payee desc)
(format "%s txn \"%s\"\n" date desc))]
(for [[k v] meta]
(format " %s: \"%s\"\n" (name k) v))
(for [posting (:postings trans)]
(when (not (zero? (:amount posting)))
(format " %-40s %10.2f %s\n%s" (:account posting) (:amount posting) (:currency posting)
(str/join (for [[k v] (seq (:meta posting))]
(for [{:keys [amount account currency meta]} postings]
(when (not (zero? amount))
(format " %-40s %10.2f %s\n%s" account amount currency
(str/join (for [[k v] meta]
(format " %s: \"%s\"\n" (name k) v)))))))))
(defn cat->acct [cat]
@ -78,196 +61,199 @@
"IL Unemploy" "IL:Unemployment"
"NY Unemploy" "NY:Unemployment"
"OR Unemploy" "OR:Unemployment"
"NY Re-empl Svc" "US:NY:Reempt"
cat))
;; TODO: How do we know we used all the relevant data from the report? Didn't
;; miss anything?
(defn employee-entity-meta [name]
(if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"}
{:entity name}))
(defn import-monthly-payroll [date period receipt-no groups]
{:date date :desc (format "Monthly Payroll - %s - Net Pay" period)
(defn project [name]
(if (= name "Sharp-Sage-A")
"Outreachy"
"Conservancy"))
(defn split-fee
"Share a total fee into n groups allocating the remainder as evenly as possible."
[total n]
(let [total (bigdec total)
fee-share (.setScale (bigdec (/ (double total) n)) 2 java.math.RoundingMode/FLOOR)
extra-cents (* 100M (- total (* fee-share n)))
base-fee-allocation (repeat fee-share)
cents-allocation (take n (concat (repeat extra-cents 0.01M) (repeat 0)))]
(map + base-fee-allocation cents-allocation)))
(defn payroll
"Return a net pay transaction."
[date period receipt-no groups]
(let [postings (for [[name records] groups]
(let [total-net-pay (->> records
(filter #(= (:type %) "Net Pay"))
(map :amount)
(apply +))
total-reimbursement (->> records
(filter #(= (:type %) "Reimbursement"))
(map :amount)
(apply +))
actual-total-net-pay (- total-net-pay total-reimbursement)]
[{:account "Expenses:Payroll:Salary"
:amount actual-total-net-pay
:currency "USD"
:meta (employee-entity-meta name)}
{:account "Assets:FR:Check2721"
:amount (- actual-total-net-pay)
:currency "USD"
:meta {:entity name}}
{:account "Expenses:Hosting"
:amount total-reimbursement
:currency "USD"
:meta (merge (employee-entity-meta name) {:payroll-type "US:Reimbursement"})}
{:account "Assets:FR:Check2721"
:amount (- total-reimbursement)
:currency "USD"
:meta (merge (employee-entity-meta name) {:tax-implication "Reimbursement"})}]))]
[{:date date :desc (format "Monthly Payroll - %s - Net Pay" period)
:meta {:program "Conservancy:Payroll"
:project "Conservancy"
:receipt receipt-no
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
:tax-implication "W2"
:payroll-type "US:General"}
:postings
(apply concat (for [[name records] groups]
(let [name (format-name name)
total-net-pay (->> records
(filter #(= (:type %) "Net Pay"))
(map :t-net-pay)
(apply +)
bigdec)
total-reimbursement (->> records
(filter #(= (:type %) "Reimbursement"))
(map :t-reimbursement)
(apply +)
bigdec)
total-net-pay-less-reimb (- total-net-pay total-reimbursement)
pay-exp-trans [{:account "Expenses:Payroll:Salary"
:amount total-net-pay-less-reimb
:currency "USD"
:meta (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"}
{:entity name})}
{:account "Assets:FR:Check2721"
:amount (- total-net-pay-less-reimb)
:currency "USD"
:meta {:entity name}}]
reimbursement-exp-trans [{:account "Expenses:Hosting"
:amount total-reimbursement
:currency "USD"
:meta (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"
:payroll-type "US:Reimbursement"}
{:entity name
:payroll-type "US:Reimbursement"})}
{:account "Assets:FR:Check2721"
:amount (- total-reimbursement)
:currency "USD"
:meta (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"
:tax-implication "Reimbursement"}
{:entity name
:tax-implication "Reimbursement"})}]]
(concat pay-exp-trans reimbursement-exp-trans))))})
:postings (apply concat postings)}]))
(defn import-individual-taxes [date period receipt-no invoice-no groups]
;; Print the individual taxes blocks
(defn individual-taxes
"Return a transaction of expenses/witholding for each employee."
[date period receipt-no invoice-no groups]
(for [[name records] groups]
{:date date :desc (format "Monthly Payroll - %s - TAXES - %s" period (format-name name))
:meta {:project (if (= (format-name name) "Sharp-Sage-A") "Outreachy" "Conservancy")
:program "Conservancy:Payroll"
:entity (format-name name)
:receipt receipt-no
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
:postings
(let [super-lines (filter #(str/starts-with? (:category %) "403b") records)
;; TODO: Have I got the liability/witholding right? Which is used in which report.
witholding-lines (filter #(= (:type %) "Withholding") records)
;; TODO: We seem to add these extra insurance lines for Karen (NY) only. Confirm with Rosanne.
insurance-lines (filter #(and (= (:type %) "Withholding")
(str/includes? (:category %) "NY Disability")) records)
total-super (->> super-lines
(map :t-retirement)
(apply +)
bigdec)]
(concat
(for [x super-lines]
(if (= (:category x) "403b ER match")
(map :amount)
(apply +))
super-postings (for [{:keys [category amount]} super-lines]
(if (= category "403b ER match")
{:account "Expenses:Payroll:Salary"
:amount (:t-retirement x)
:amount amount
:currency "USD"
:meta {:payroll-type "US:403b:Match"
:invoice invoice-no}}
{:account "Expenses:Payroll:Salary"
:amount (:t-retirement x)
:amount amount
:currency "USD"
:meta {:payroll-type "US:403b:Employee"
:invoice invoice-no}}))
[{:account "Liabilities:Payable:Accounts"
liability-postings [{:account "Liabilities:Payable:Accounts"
:amount (- total-super)
:currency "USD"
:meta {:invoice invoice-no}}]
(conj
(vec (for [x witholding-lines]
withholding-postings (for [{:keys [category amount]} witholding-lines]
{:account "Expenses:Payroll:Salary"
:amount (:t-withholding x)
:amount amount
:currency "USD"
:meta {:payroll-type (cat->acct (:category x))}}))
{:account "Assets:FR:Check2721"
:amount (- (reduce + (map :t-withholding witholding-lines)))
:meta {:payroll-type (cat->acct category)}})
withholding-asset-postings [{:account "Assets:FR:Check2721"
:amount (- (reduce + (map :amount witholding-lines)))
:currency "USD"
:meta {:tax-implication "W2"}})
;; TODO: We seem to add these extra insurance lines for Karen (only). Confirm with Rosanne.
(conj
(vec (for [x insurance-lines]
:meta {:tax-implication "W2"}}]
insurance-postings (for [{:keys [category amount]} insurance-lines]
{:account "Expenses:Insurance"
:amount (- (:t-withholding x))
:amount (- amount)
:currency "USD"
:meta {:payroll-type (cat->acct (:category x))}}))
{:account "Assets:FR:Check2721"
:amount (bigdec (reduce + (map :t-withholding insurance-lines)))
:currency "USD"})))}))
:meta {:payroll-type (cat->acct category)}})
insurance-asset-postings [{:account "Assets:FR:Check2721"
:amount (reduce + (map :amount insurance-lines))
:currency "USD"}]]
{:date date :desc (format "Monthly Payroll - %s - TAXES - %s" period name)
:meta {:project (project name)
:program "Conservancy:Payroll"
:entity name
:receipt receipt-no
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
:postings (concat
super-postings
liability-postings
withholding-postings
withholding-asset-postings
insurance-postings
insurance-asset-postings)})))
(defn import-employer-taxes [date period receipt-no groups]
{:date date :desc (format "Monthly Payroll - %s - TAXES - Employer" period)
(defn employer-taxes
"Return an employer taxes transaction."
[date period receipt-no groups]
(let [liability-postings (apply concat
(for [[name records] groups]
(let [liability-lines (filter #(= (:type %) "Liability") records)]
(for [{:keys [category amount]} liability-lines]
{:account "Expenses:Payroll:Tax"
:amount amount
:currency "USD"
:meta (merge
(employee-entity-meta name)
;; TODO: Check lack of ":Tax:" with Rosanne.
{:payroll-type (str/replace (cat->acct category) "Tax:" "")})}))))
total-liabilities (->> liability-postings (map :amount) (reduce +))
asset-postings [{:account "Assets:FR:Check2721"
:amount (- total-liabilities)
:currency "USD"
:meta {:entity "Paychex"
:tax-implication "Tax-Payment"}}]]
[{:date date :desc (format "Monthly Payroll - %s - TAXES - Employer" period)
:meta {:program "Conservancy:Payroll"
:project "Conservancy"
:receipt receipt-no
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
:postings
(let [total-emp-tax (reduce + (map :t-liability (apply concat (vals groups))))]
(conj (vec (apply concat (for [[name records] groups]
(let [name (format-name name)
liability-lines (filter #(= (:type %) "Liability") records)]
(for [x liability-lines]
{:account "Expenses:Payroll:Tax"
:amount (:t-liability x)
:currency "USD"
:meta (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"
;; TODO: Check lack of ":Tax:" with Rosanne.
:payroll-type (str/replace (cat->acct (:category x)) "Tax:" "")}
{:entity name
:payroll-type (str/replace (cat->acct (:category x)) "Tax:" "")})})))))
{:account "Assets:FR:Check2721"
:amount (- total-emp-tax)
:currency "USD"
:meta {:entity "Paychex"
:tax-implication "Tax-Payment"}}))})
:postings (concat liability-postings asset-postings)}]))
(defn payroll-fees [date period receipt-no invoice-no total-fees groups]
(let [employees (map import/format-name (keys groups))
num-employees (count employees)
fee-share (.setScale (/ total-fees num-employees) 2 java.math.RoundingMode/FLOOR)
extra-cents (* 100 (- total-fees (* fee-share num-employees)))
base-fee-allocation (repeat fee-share)
cents-allocation (concat (repeat extra-cents (bigdec 0.01)) (repeat 0))
exact-fee-allocation (reverse (take num-employees (map + base-fee-allocation cents-allocation)))
expense-postings (for [[name fee] (map vector employees exact-fee-allocation)]
(defn payroll-fees
"Return a payroll fees transaction."
[date period receipt-no invoice-no total-fees groups]
(let [employees (keys groups)
exact-fee-allocation (split-fee total-fees (count employees))
employee-fees (map vector employees exact-fee-allocation)
expense-postings (for [[name fee] employee-fees]
{:account "Expenses:Payroll:Fees"
:amount fee
:currency "USD"
:meta {:entity name}})
asset-posting {:account "Assets:FR:Check2721"
:meta (employee-entity-meta name)})
asset-postings [{:account "Assets:FR:Check2721"
:amount (- total-fees)
:currency "USD"}
all-postings (conj (vec expense-postings) asset-posting)]
{:date date :payee "Paychex" :desc (format "Monthly Payroll - %s - Fee" period)
:currency "USD"}]]
[{:date date :payee "Paychex" :desc (format "Monthly Payroll - %s - Fee" period)
:meta {:program "Conservancy:Payroll"
:project "Conservancy"
:receipt receipt-no
:invoice invoice-no
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
:tax-implication "USA-Corporation"}
:postings all-postings}))
:postings (concat expense-postings asset-postings)}]))
(defn import-retirement [date period receipt-no invoice-no groups]
{:date date :desc (format "ASCENSUS TRUST RET PLAN - ACH DEBIT - Vanguard 403(b) - %s" period)
(defn retirement
"Return a retirement transaction."
[date period receipt-no invoice-no groups]
(let [liability-postings (for [[name records] groups]
(let [total-retirement (->> records
(filter #(= (:type %) "Retirement"))
(map :amount)
(reduce +))]
{:account "Liabilities:Payable:Accounts",
:amount total-retirement,
:currency "USD",
:meta {:entity name}}))
total-liabilities (->> liability-postings (map :amount) (reduce +))
asset-postings [{:account "Assets:FR:Check1345"
:amount (- total-liabilities)
:currency "USD"}]]
[{:date date :desc (format "ASCENSUS TRUST RET PLAN - ACH DEBIT - Vanguard 403(b) - %s" period)
:meta {:program "Conservancy:Payroll"
:project "Conservancy"
:receipt receipt-no
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
:tax-implication "Retirement-Pretax"
:invoice invoice-no}
:postings
(let [total-retirement (reduce + (map :t-retirement (apply concat (vals groups))))]
(conj (vec (for [[name records] groups]
(let [name (format-name name)
total-retirement (->> records
(filter #(= (:type %) "Retirement"))
(map :t-retirement)
(apply +)
bigdec)]
{:account "Liabilities:Payable:Accounts",
:amount total-retirement,
:currency "USD",
:meta {:entity name}})))
{:account "Assets:FR:Check1345"
:amount (- total-retirement)
:currency "USD"}))})
:postings (concat liability-postings asset-postings)}]))