NPO-Accounting/payroll-import
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Restructure the importers to be more consistent

Browse source
This commit is contained in:
Ben Sturmfels 2024-02-22 01:02:38 +11:00
parent 18eb076df0
commit d77967d61b
Signed by: bsturmfels
GPG key ID: 023C05E2C9C068F0
2 changed files with 228 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
src/core.clj
View file

@ -25,7 +25,8 @@
[nil "--pay-invoice-no REFERENCE" "Payroll invoice number, eg. \"rt:111/222\"" [nil "--pay-invoice-no REFERENCE" "Payroll invoice number, eg. \"rt:111/222\""
:default "FIXME"] :default "FIXME"]
[nil "--total-fees NUM" "Total fee charged by Paychex, eg. \"206.50\"" [nil "--total-fees NUM" "Total fee charged by Paychex, eg. \"206.50\""
:default (bigdec 0)] :parse-fn bigdec
:default 0M]
[nil "--fees-receipt-no REFERENCE" "Paychex fees receipt number, eg. \"rt:111/222\"" [nil "--fees-receipt-no REFERENCE" "Paychex fees receipt number, eg. \"rt:111/222\""
:default "FIXME"] :default "FIXME"]
[nil "--fees-invoice-no REFERENCE" "Paychex fees invoice number, eg. \"rt:111/222\"" [nil "--fees-invoice-no REFERENCE" "Paychex fees invoice number, eg. \"rt:111/222\""
@ -46,11 +47,11 @@
(println summary) (println summary)
(System/exit 0)) (System/exit 0))
(let [grouped-data (import/read-grouped-csv (:csv options)) (let [grouped-data (import/read-grouped-csv (:csv options))
imported (concat [(import/import-monthly-payroll pay-date period pay-receipt-no grouped-data)] imported (concat (import/payroll pay-date period pay-receipt-no grouped-data)
(import/import-individual-taxes pay-date period pay-receipt-no pay-invoice-no grouped-data) (import/individual-taxes pay-date period pay-receipt-no pay-invoice-no grouped-data)
[(import/import-employer-taxes pay-date period pay-receipt-no grouped-data)] (import/employer-taxes pay-date period pay-receipt-no grouped-data)
[(import/payroll-fees pay-date period fees-receipt-no fees-invoice-no total-fees grouped-data)] (import/payroll-fees pay-date period fees-receipt-no fees-invoice-no total-fees grouped-data)
[(import/import-retirement retirement-date period retirement-receipt-no retirement-invoice-no grouped-data)])] (import/retirement retirement-date period retirement-receipt-no retirement-invoice-no grouped-data))]
(doseq [i imported] (doseq [i imported]
(println (import/render-transaction i)))))) (println (import/render-transaction i))))))
@ -58,17 +59,30 @@
(require '[examples :as examples]) (require '[examples :as examples])
(def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2023-12-27_Pay-Item-Details_2023-12-2.csv")) (def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2023-12-27_Pay-Item-Details_2023-12-2.csv"))
(def imported (def imported
(concat [(import/import-monthly-payroll "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)] (concat (import/payroll "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)
(import/import-individual-taxes "2023-12-29" "December 2023" "rt:19462/674660" "rt:19403/675431" grouped-data) (import/individual-taxes "2023-12-29" "December 2023" "rt:19462/674660" "rt:19403/675431" grouped-data)
[(import/import-employer-taxes "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)] (import/employer-taxes "2023-12-29" "December 2023" "rt:19462/674660" grouped-data)
[(import/payroll-fees "2023-12-29" "December 2023" "rt:19459/675387" "rt:19459/674887" (bigdec 206.50) grouped-data)] (import/payroll-fees "2023-12-29" "December 2023" "rt:19459/675387" "rt:19459/674887" 206.50M grouped-data)
[(import/import-retirement "2024-01-02" "December 2023" "rt:19403/676724" "rt:19403/675431" grouped-data)])) (import/retirement "2024-01-02" "December 2023" "rt:19403/676724" "rt:19403/675431" grouped-data)))
(dd/pretty-print (dd/pretty-print
(dd/diff (dd/diff
(sort-postings (parse examples/human)) (sort-postings (parse examples/human))
(sort-postings imported))) (sort-postings imported)))
(require '[examples2 :as examples])
(def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2024-01-29_Pay-Item-Details_2024-01.csv"))
(def imported
(concat (import/payroll "2024-01-31" "January 2024" "rt:19462/685751" grouped-data)
(import/individual-taxes "2024-01-31" "January 2024" "rt:19462/685751" "rt:19403/685602" grouped-data)
;; TODO: Needs fixes for the correct :entity and :memo
(import/employer-taxes "2024-01-31" "January 2024" "rt:19462/685751" grouped-data)
#_(import/payroll-fees "2024-01-31" "January 2024" "rt:19459/675387" "rt:19459/674887" 206.50M grouped-data)
(import/retirement "2024-01-31" "January 2024" "rt:19403/685929" "rt:19403/685602" grouped-data)))
(dd/pretty-print
(dd/diff
(sort-postings (parse examples2/human))
(sort-postings imported)))
(doseq [i imported] (doseq [i imported]
(println (import/render-transaction i))) (println (import/render-transaction i)))

418
src/import.clj
View file

@ -9,36 +9,6 @@
(str/replace " " "-") (str/replace " " "-")
keyword)) keyword))
(defn update-vals-subset [m ks f]
(reduce #(update %1 %2 f) m ks))
(defn bigdec-or-zero [s]
(if (str/blank? s)
(bigdec 0)
(bigdec s)))
;; TODO: Consider merging into a single amount column.
(defn prep-csv-data [records]
(let [headers [:org :name :id :category :type :t-earnings :_ :t-earnings-match :_ :t-reimbursement :t-retirement :_ :t-withholding :t-liability :_ :t-net-pay]
map-records (map zipmap (repeat headers) records)
cleaned-records (->> map-records
(map #(-> %
(dissoc :org :_)
(update-vals-subset [:t-earnings
:t-earnings-match
:t-reimbursement
:t-retirement
:t-withholding
:t-liability
:t-net-pay]
bigdec-or-zero))))]
cleaned-records))
(defn read-grouped-csv [filename]
(with-open [reader (io/reader filename)]
(doall
(group-by :name (import/prep-csv-data (csv/read-csv reader))))))
(defn format-name [name] (defn format-name [name]
(case name (case name
"Sharp, Sage A" "Sharp-Sage-A" "Sharp, Sage A" "Sharp-Sage-A"
@ -47,14 +17,27 @@
(str/replace ", " "-") (str/replace ", " "-")
(str/replace #" \w$" "")))) (str/replace #" \w$" ""))))
(defn render-transaction [trans] (defn read-grouped-csv [filename]
(str/join (concat [(format "%s txn \"%s\"\n" (:date trans) (:desc trans))] (with-open [reader (io/reader filename)]
(for [[k v] (seq (:meta trans))] (doall
(group-by
:name
(for [[_ name _ category type & totals] (csv/read-csv reader)]
{:name (import/format-name name)
:category category
:type type
:amount (apply max (map bigdec (remove str/blank? totals)))})))))
(defn render-transaction [{:keys [date payee desc meta postings]}]
(str/join (concat [(if payee
(format "%s txn \"%s\" \"%s\"\n" date payee desc)
(format "%s txn \"%s\"\n" date desc))]
(for [[k v] meta]
(format " %s: \"%s\"\n" (name k) v)) (format " %s: \"%s\"\n" (name k) v))
(for [posting (:postings trans)] (for [{:keys [amount account currency meta]} postings]
(when (not (zero? (:amount posting))) (when (not (zero? amount))
(format " %-40s %10.2f %s\n%s" (:account posting) (:amount posting) (:currency posting) (format " %-40s %10.2f %s\n%s" account amount currency
(str/join (for [[k v] (seq (:meta posting))] (str/join (for [[k v] meta]
(format " %s: \"%s\"\n" (name k) v))))))))) (format " %s: \"%s\"\n" (name k) v)))))))))
(defn cat->acct [cat] (defn cat->acct [cat]
@ -78,196 +61,199 @@
"IL Unemploy" "IL:Unemployment" "IL Unemploy" "IL:Unemployment"
"NY Unemploy" "NY:Unemployment" "NY Unemploy" "NY:Unemployment"
"OR Unemploy" "OR:Unemployment" "OR Unemploy" "OR:Unemployment"
"NY Re-empl Svc" "US:NY:Reempt"
cat)) cat))
;; TODO: How do we know we used all the relevant data from the report? Didn't (defn employee-entity-meta [name]
;; miss anything? (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"}
{:entity name}))
(defn import-monthly-payroll [date period receipt-no groups] (defn project [name]
{:date date :desc (format "Monthly Payroll - %s - Net Pay" period) (if (= name "Sharp-Sage-A")
:meta {:program "Conservancy:Payroll" "Outreachy"
:project "Conservancy" "Conservancy"))
:receipt receipt-no
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt" (defn split-fee
:tax-implication "W2" "Share a total fee into n groups allocating the remainder as evenly as possible."
:payroll-type "US:General"} [total n]
:postings (let [total (bigdec total)
(apply concat (for [[name records] groups] fee-share (.setScale (bigdec (/ (double total) n)) 2 java.math.RoundingMode/FLOOR)
(let [name (format-name name) extra-cents (* 100M (- total (* fee-share n)))
total-net-pay (->> records base-fee-allocation (repeat fee-share)
cents-allocation (take n (concat (repeat extra-cents 0.01M) (repeat 0)))]
(map + base-fee-allocation cents-allocation)))
(defn payroll
"Return a net pay transaction."
[date period receipt-no groups]
(let [postings (for [[name records] groups]
(let [total-net-pay (->> records
(filter #(= (:type %) "Net Pay")) (filter #(= (:type %) "Net Pay"))
(map :t-net-pay) (map :amount)
(apply +) (apply +))
bigdec)
total-reimbursement (->> records total-reimbursement (->> records
(filter #(= (:type %) "Reimbursement")) (filter #(= (:type %) "Reimbursement"))
(map :t-reimbursement) (map :amount)
(apply +) (apply +))
bigdec) actual-total-net-pay (- total-net-pay total-reimbursement)]
total-net-pay-less-reimb (- total-net-pay total-reimbursement) [{:account "Expenses:Payroll:Salary"
pay-exp-trans [{:account "Expenses:Payroll:Salary" :amount actual-total-net-pay
:amount total-net-pay-less-reimb :currency "USD"
:currency "USD" :meta (employee-entity-meta name)}
:meta (if (= name "Sharp-Sage-A") {:account "Assets:FR:Check2721"
{:entity name :amount (- actual-total-net-pay)
:project "Outreachy"} :currency "USD"
{:entity name})} :meta {:entity name}}
{:account "Assets:FR:Check2721" {:account "Expenses:Hosting"
:amount (- total-net-pay-less-reimb) :amount total-reimbursement
:currency "USD" :currency "USD"
:meta {:entity name}}] :meta (merge (employee-entity-meta name) {:payroll-type "US:Reimbursement"})}
reimbursement-exp-trans [{:account "Expenses:Hosting" {:account "Assets:FR:Check2721"
:amount total-reimbursement :amount (- total-reimbursement)
:currency "USD" :currency "USD"
:meta (if (= name "Sharp-Sage-A") :meta (merge (employee-entity-meta name) {:tax-implication "Reimbursement"})}]))]
{:entity name [{:date date :desc (format "Monthly Payroll - %s - Net Pay" period)
:project "Outreachy" :meta {:program "Conservancy:Payroll"
:payroll-type "US:Reimbursement"} :project "Conservancy"
{:entity name :receipt receipt-no
:payroll-type "US:Reimbursement"})} :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
{:account "Assets:FR:Check2721" :tax-implication "W2"
:amount (- total-reimbursement) :payroll-type "US:General"}
:currency "USD" :postings (apply concat postings)}]))
:meta (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"
:tax-implication "Reimbursement"}
{:entity name
:tax-implication "Reimbursement"})}]]
(concat pay-exp-trans reimbursement-exp-trans))))})
(defn import-individual-taxes [date period receipt-no invoice-no groups] (defn individual-taxes
;; Print the individual taxes blocks "Return a transaction of expenses/witholding for each employee."
[date period receipt-no invoice-no groups]
(for [[name records] groups] (for [[name records] groups]
{:date date :desc (format "Monthly Payroll - %s - TAXES - %s" period (format-name name)) (let [super-lines (filter #(str/starts-with? (:category %) "403b") records)
:meta {:project (if (= (format-name name) "Sharp-Sage-A") "Outreachy" "Conservancy") ;; TODO: Have I got the liability/witholding right? Which is used in which report.
:program "Conservancy:Payroll" witholding-lines (filter #(= (:type %) "Withholding") records)
:entity (format-name name) ;; TODO: We seem to add these extra insurance lines for Karen (NY) only. Confirm with Rosanne.
:receipt receipt-no insurance-lines (filter #(and (= (:type %) "Withholding")
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"} (str/includes? (:category %) "NY Disability")) records)
:postings total-super (->> super-lines
(let [super-lines (filter #(str/starts-with? (:category %) "403b") records) (map :amount)
;; TODO: Have I got the liability/witholding right? Which is used in which report. (apply +))
witholding-lines (filter #(= (:type %) "Withholding") records) super-postings (for [{:keys [category amount]} super-lines]
insurance-lines (filter #(and (= (:type %) "Withholding") (if (= category "403b ER match")
(str/includes? (:category %) "NY Disability")) records) {:account "Expenses:Payroll:Salary"
total-super (->> super-lines :amount amount
(map :t-retirement) :currency "USD"
(apply +) :meta {:payroll-type "US:403b:Match"
bigdec)] :invoice invoice-no}}
(concat {:account "Expenses:Payroll:Salary"
(for [x super-lines] :amount amount
(if (= (:category x) "403b ER match") :currency "USD"
{:account "Expenses:Payroll:Salary" :meta {:payroll-type "US:403b:Employee"
:amount (:t-retirement x) :invoice invoice-no}}))
:currency "USD" liability-postings [{:account "Liabilities:Payable:Accounts"
:meta {:payroll-type "US:403b:Match" :amount (- total-super)
:invoice invoice-no}} :currency "USD"
{:account "Expenses:Payroll:Salary" :meta {:invoice invoice-no}}]
:amount (:t-retirement x) withholding-postings (for [{:keys [category amount]} witholding-lines]
:currency "USD" {:account "Expenses:Payroll:Salary"
:meta {:payroll-type "US:403b:Employee" :amount amount
:invoice invoice-no}})) :currency "USD"
[{:account "Liabilities:Payable:Accounts" :meta {:payroll-type (cat->acct category)}})
:amount (- total-super) withholding-asset-postings [{:account "Assets:FR:Check2721"
:currency "USD" :amount (- (reduce + (map :amount witholding-lines)))
:meta {:invoice invoice-no}}] :currency "USD"
(conj :meta {:tax-implication "W2"}}]
(vec (for [x witholding-lines] insurance-postings (for [{:keys [category amount]} insurance-lines]
{:account "Expenses:Payroll:Salary" {:account "Expenses:Insurance"
:amount (:t-withholding x) :amount (- amount)
:currency "USD" :currency "USD"
:meta {:payroll-type (cat->acct (:category x))}})) :meta {:payroll-type (cat->acct category)}})
{:account "Assets:FR:Check2721" insurance-asset-postings [{:account "Assets:FR:Check2721"
:amount (- (reduce + (map :t-withholding witholding-lines))) :amount (reduce + (map :amount insurance-lines))
:currency "USD" :currency "USD"}]]
:meta {:tax-implication "W2"}}) {:date date :desc (format "Monthly Payroll - %s - TAXES - %s" period name)
;; TODO: We seem to add these extra insurance lines for Karen (only). Confirm with Rosanne. :meta {:project (project name)
(conj :program "Conservancy:Payroll"
(vec (for [x insurance-lines] :entity name
{:account "Expenses:Insurance" :receipt receipt-no
:amount (- (:t-withholding x)) :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
:currency "USD" :postings (concat
:meta {:payroll-type (cat->acct (:category x))}})) super-postings
{:account "Assets:FR:Check2721" liability-postings
:amount (bigdec (reduce + (map :t-withholding insurance-lines))) withholding-postings
:currency "USD"})))})) withholding-asset-postings
insurance-postings
insurance-asset-postings)})))
(defn import-employer-taxes [date period receipt-no groups] (defn employer-taxes
{:date date :desc (format "Monthly Payroll - %s - TAXES - Employer" period) "Return an employer taxes transaction."
:meta {:program "Conservancy:Payroll" [date period receipt-no groups]
:project "Conservancy" (let [liability-postings (apply concat
:receipt receipt-no (for [[name records] groups]
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"} (let [liability-lines (filter #(= (:type %) "Liability") records)]
:postings (for [{:keys [category amount]} liability-lines]
(let [total-emp-tax (reduce + (map :t-liability (apply concat (vals groups))))] {:account "Expenses:Payroll:Tax"
(conj (vec (apply concat (for [[name records] groups] :amount amount
(let [name (format-name name) :currency "USD"
liability-lines (filter #(= (:type %) "Liability") records)] :meta (merge
(for [x liability-lines] (employee-entity-meta name)
{:account "Expenses:Payroll:Tax" ;; TODO: Check lack of ":Tax:" with Rosanne.
:amount (:t-liability x) {:payroll-type (str/replace (cat->acct category) "Tax:" "")})}))))
:currency "USD" total-liabilities (->> liability-postings (map :amount) (reduce +))
:meta (if (= name "Sharp-Sage-A") asset-postings [{:account "Assets:FR:Check2721"
{:entity name :amount (- total-liabilities)
:project "Outreachy" :currency "USD"
;; TODO: Check lack of ":Tax:" with Rosanne. :meta {:entity "Paychex"
:payroll-type (str/replace (cat->acct (:category x)) "Tax:" "")} :tax-implication "Tax-Payment"}}]]
{:entity name [{:date date :desc (format "Monthly Payroll - %s - TAXES - Employer" period)
:payroll-type (str/replace (cat->acct (:category x)) "Tax:" "")})}))))) :meta {:program "Conservancy:Payroll"
{:account "Assets:FR:Check2721" :project "Conservancy"
:amount (- total-emp-tax) :receipt receipt-no
:currency "USD" :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
:meta {:entity "Paychex" :postings (concat liability-postings asset-postings)}]))
:tax-implication "Tax-Payment"}}))})
(defn payroll-fees [date period receipt-no invoice-no total-fees groups] (defn payroll-fees
(let [employees (map import/format-name (keys groups)) "Return a payroll fees transaction."
num-employees (count employees) [date period receipt-no invoice-no total-fees groups]
fee-share (.setScale (/ total-fees num-employees) 2 java.math.RoundingMode/FLOOR) (let [employees (keys groups)
extra-cents (* 100 (- total-fees (* fee-share num-employees))) exact-fee-allocation (split-fee total-fees (count employees))
base-fee-allocation (repeat fee-share) employee-fees (map vector employees exact-fee-allocation)
cents-allocation (concat (repeat extra-cents (bigdec 0.01)) (repeat 0)) expense-postings (for [[name fee] employee-fees]
exact-fee-allocation (reverse (take num-employees (map + base-fee-allocation cents-allocation)))
expense-postings (for [[name fee] (map vector employees exact-fee-allocation)]
{:account "Expenses:Payroll:Fees" {:account "Expenses:Payroll:Fees"
:amount fee :amount fee
:currency "USD" :currency "USD"
:meta {:entity name}}) :meta (employee-entity-meta name)})
asset-posting {:account "Assets:FR:Check2721" asset-postings [{:account "Assets:FR:Check2721"
:amount (- total-fees) :amount (- total-fees)
:currency "USD"} :currency "USD"}]]
all-postings (conj (vec expense-postings) asset-posting)] [{:date date :payee "Paychex" :desc (format "Monthly Payroll - %s - Fee" period)
{:date date :payee "Paychex" :desc (format "Monthly Payroll - %s - Fee" period) :meta {:program "Conservancy:Payroll"
:meta {:program "Conservancy:Payroll" :project "Conservancy"
:project "Conservancy" :receipt receipt-no
:receipt receipt-no :invoice invoice-no
:invoice invoice-no :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt" :tax-implication "USA-Corporation"}
:tax-implication "USA-Corporation"} :postings (concat expense-postings asset-postings)}]))
:postings all-postings}))
(defn import-retirement [date period receipt-no invoice-no groups] (defn retirement
{:date date :desc (format "ASCENSUS TRUST RET PLAN - ACH DEBIT - Vanguard 403(b) - %s" period) "Return a retirement transaction."
:meta {:program "Conservancy:Payroll" [date period receipt-no invoice-no groups]
:project "Conservancy" (let [liability-postings (for [[name records] groups]
:receipt receipt-no (let [total-retirement (->> records
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt" (filter #(= (:type %) "Retirement"))
:tax-implication "Retirement-Pretax" (map :amount)
:invoice invoice-no} (reduce +))]
:postings {:account "Liabilities:Payable:Accounts",
(let [total-retirement (reduce + (map :t-retirement (apply concat (vals groups))))] :amount total-retirement,
(conj (vec (for [[name records] groups] :currency "USD",
(let [name (format-name name) :meta {:entity name}}))
total-retirement (->> records total-liabilities (->> liability-postings (map :amount) (reduce +))
(filter #(= (:type %) "Retirement")) asset-postings [{:account "Assets:FR:Check1345"
(map :t-retirement) :amount (- total-liabilities)
(apply +) :currency "USD"}]]
bigdec)] [{:date date :desc (format "ASCENSUS TRUST RET PLAN - ACH DEBIT - Vanguard 403(b) - %s" period)
{:account "Liabilities:Payable:Accounts", :meta {:program "Conservancy:Payroll"
:amount total-retirement, :project "Conservancy"
:currency "USD", :receipt receipt-no
:meta {:entity name}}))) :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
{:account "Assets:FR:Check1345" :tax-implication "Retirement-Pretax"
:amount (- total-retirement) :invoice invoice-no}
:currency "USD"}))}) :postings (concat liability-postings asset-postings)}]))