NPO-Accounting/payroll-import
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial importer/parser to handle monthly net pay and individual taxes transactions

Browse source 
Does not yet handle employer taxes, fees or retirement transactions.
This commit is contained in:
Ben Sturmfels 2024-02-20 14:19:23 +11:00
commit 783d98c4da
Signed by: bsturmfels
GPG key ID: 023C05E2C9C068F0
4 changed files with 344 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
deps.edn Normal file
View file

@ -0,0 +1,4 @@
{:deps {
org.clojure/clojure {:mvn/version "1.11.1"}
org.clojure/data.csv {:mvn/version "1.0.1"}
lambdaisland/deep-diff2 {:mvn/version "2.10.211"}}}

32
src/core.clj Normal file
View file

@ -0,0 +1,32 @@
(ns core
(:require [clojure.data.csv :as csv]
[clojure.java.io :as io]
[import :as import]
[parse :refer [parse]]
[lambdaisland.deep-diff2 :as dd]))
(defn sort-postings [transactions]
(for [t transactions]
(update t :postings
(fn [ps] (sort-by (juxt #(get-in % [:meta :entity]) :account :amount) (filter #(not (zero? (:amount %))) ps))))))
(comment
(def data
(with-open [reader (io/reader "/home/ben/Downloads/2023-12-27_Pay-Item-Details_2023-12-2.csv")]
(doall
(import/prep-csv-data (csv/read-csv reader)))))
(def groups (group-by :name data))
(def imported
(concat [(import/import-monthly-payroll groups)]
(import/import-individual-taxes groups)))
(dd/pretty-print (dd/diff
(sort-postings human)
(sort-postings imported)
))
(doseq [i imported]
(println (import/render-transaction i)))
)

219
src/import.clj Normal file
View file

@ -0,0 +1,219 @@
(ns import
(:require [clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.string :as str]))
(defn kebab-kw [x]
(-> x
str/lower-case
(str/replace " " "-")
keyword))
(defn update-vals-subset [m ks f]
(reduce #(update %1 %2 f) m ks))
(defn bigdec-or-zero [s]
(if (str/blank? s) (bigdec 0) (bigdec s)))
(defn prep-csv-data [records]
(let [headers [:org :name :id :category :type :t-earnings :_ :t-earnings-match :_ :t-reimbursement :t-retirement :_ :t-withholding :t-liability :_ :t-net-pay]
map-records (map zipmap (repeat headers) records)
cleaned-records (->> map-records
(map #(-> %
(dissoc :org :_)
(update-vals-subset [:t-earnings
:t-earnings-match
:t-reimbursement
:t-retirement
:t-withholding
:t-liability
:t-net-pay]
bigdec-or-zero))))]
cleaned-records))
(defn monthly-payroll-trans [date desc receipt postings]
{:date date :desc desc
:meta {:program "Conservancy:Payroll"
:project "Conservancy"
:receipt receipt
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
:tax-implication "W2"
:payroll-type "US:General"}
:postings postings})
(defn format-name [name]
(case name
"Sharp, Sage A" "Sharp-Sage-A"
(-> name
(str/replace " Jr." "")
(str/replace ", " "-")
(str/replace #" \w$" ""))))
(defn render-transaction [trans]
(str/join (concat [(format "%s txn \"%s\"\n" (:date trans) (:desc trans))]
(for [[k v] (seq (:meta trans))]
(format " %s: \"%s\"\n" (name k) v))
(for [posting (:postings trans)]
(when (not (zero? (:amount posting)))
(format " %-40s %10.2f %s\n%s" (:account posting) (:amount posting) (:currency posting)
(str/join (for [[k v] (seq (:meta posting))]
(format " %s: \"%s\"\n" (name k) v)))))))))
(defn cat->acct [cat]
;; TODO: Can we make this more general for other states?
(case cat
"Fed Income Tax" "US:Tax:Income"
"Medicare" "US:Tax:Medicare"
"IL Income Tax" "US:IL:Tax:Income"
"NY Income Tax" "US:NY:Tax:Income"
"NYC Income Tax" "US:NY:Tax:NYC"
"OR Income Tax" "US:OR:Tax:Income"
"OH Income Tax" "US:OH:Tax:Income"
"PNTSD Income Tax" "US:OH:Tax:PNTSD"
"COLMB Income Tax" "US:OH:Tax:COLUMB"
"Social Security" "US:Tax:SocialSecurity"
"NY Disability" "US:NY:Disability"
"OR Disability PFL" "US:OR:Disability:PFL"
"NY Disability PFL" "US:NY:Disability:PFL"
"OR TRANS STT" "US:OR:Tax:STT"
cat))
(defn indiv-payroll-trans [date desc name receipt postings]
{:date date :desc (format desc (format-name name))
:meta {:project (if (= (format-name name) "Sharp-Sage-A") "Outreachy" "Conservancy")
:program "Conservancy:Payroll"
:entity (format-name name)
:receipt receipt
:approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
:postings postings})
;; TODO: How do we know we used all the relevant data from the report? Didn't
;; miss anything?
;; TODO: Split out the match amount into a separate transaction entry?
(defn import-monthly-payroll [groups]
(monthly-payroll-trans
"2023-12-29"
"Monthly Payroll - December 2023 - Net Pay"
"rt:19462/674660"
(apply concat (for [[name records] groups]
(let [name (format-name name)
total-net-pay (->> records
(filter #(= (:type %) "Net Pay"))
(map :t-net-pay)
(apply +)
bigdec)
total-reimbursement (->> records
(filter #(= (:type %) "Reimbursement"))
(map :t-reimbursement)
(apply +)
bigdec)
total-net-pay-less-reimb (- total-net-pay total-reimbursement)
pay-exp-trans [{:account "Expenses:Payroll:Salary"
:amount total-net-pay-less-reimb
:currency "USD"
:meta (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"}
{:entity name})}
{:account "Assets:FR:Check2721"
:amount (- total-net-pay-less-reimb)
:currency "USD"
:meta {:entity name}}]
reimbursement-exp-trans [{:account "Expenses:Hosting"
:amount total-reimbursement
:currency "USD"
:meta (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"
:payroll-type "US:Reimbursement"}
{:entity name
:payroll-type "US:Reimbursement"})}
{:account "Assets:FR:Check2721"
:amount (- total-reimbursement)
:currency "USD"
:meta (if (= name "Sharp-Sage-A")
{:entity name
:project "Outreachy"
:tax-implication "Reimbursement"}
{:entity name
:tax-implication "Reimbursement"})}]]
(concat pay-exp-trans reimbursement-exp-trans))))
#_(apply concat (for [x data :when (or (and (= (:type x) "Net Pay") (> (:t-net-pay x) 0))
(and (= (:type x) "Reimbursement") (> (:t-reimbursement x) 0)))]
(let [name (format-name (:name x))]
(case (:type x)
"Net Pay"
"Reimbursement"
[{:account "Expenses:Hosting"
:amount (:t-reimbursement x)
:currency "USD"
:meta {:entity name
:payroll-type "US:Reimbursement"}}
{:account "Assets:FR:Check2721"
:amount (- (:t-reimbursement x))
:currency "USD"
:meta {:entity name
:tax-implication "Reimbursement"}}]))))))
(defn import-individual-taxes [groups]
;; Print the individual taxes blocks
(for [[name records] groups]
(indiv-payroll-trans
"2023-12-29"
"Monthly Payroll - December 2023 - TAXES - %s"
name
"rt:19462/674660"
(let [r-super-lines (filter #(str/starts-with? (:category %) "403b") records)
r-witholding-lines (filter #(= (:type %) "Withholding") records)
r-insurance-lines (filter #(and (= (:type %) "Withholding")
(str/includes? (:category %) "NY Disability")) records)
total-super (->> r-super-lines
(map :t-retirement)
(apply +)
bigdec)]
(concat
(for [x r-super-lines]
(if (= (:category x) "403b ER match")
{:account "Expenses:Payroll:Salary"
:amount (:t-retirement x)
:currency "USD"
:meta {:payroll-type "US:403b:Match"
:invoice "rt:19403/675431"}}
{:account "Expenses:Payroll:Salary"
:amount (:t-retirement x)
:currency "USD"
:meta {:payroll-type "US:403b:Employee"
:invoice "rt:19403/675431"}}))
[{:account "Liabilities:Payable:Accounts"
:amount (- total-super)
:currency "USD"
:meta {:invoice "rt:19403/675431"}}]
(conj
(vec (for [x r-witholding-lines]
{:account "Expenses:Payroll:Salary"
:amount (:t-withholding x)
:currency "USD"
:meta {:payroll-type (cat->acct (:category x))}}))
{:account "Assets:FR:Check2721"
:amount (- (reduce + (map :t-withholding r-witholding-lines)))
:currency "USD"
:meta {:tax-implication "W2"}})
;; We seem to add these extra insurance lines for Karen (only). Confirm with Rosanne.
(conj
(vec (for [x r-insurance-lines]
{:account "Expenses:Insurance"
:amount (- (:t-withholding x))
:currency "USD"
:meta {:payroll-type (cat->acct (:category x))}}))
{:account "Assets:FR:Check2721"
:amount (bigdec (reduce + (map :t-withholding r-insurance-lines)))
:currency "USD"
:meta {}}))))))
;; TODO: TAXES - Employer
;; TODO: Fee
;; TODO: Vanguard 403(b)

89
src/parse.clj Normal file
View file

@ -0,0 +1,89 @@
(ns parse
(:require [clojure.spec.alpha :as s]
[clojure.walk :as walk]))
(s/def ::token (s/+ (set "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890-:./")))
(s/def ::quoted-token (s/cat
:_ #{\"}
:token (s/+ (set "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890-:./ "))
:_ #{\"}))
(s/def ::whitespace (s/+ #{\space}))
(s/def ::meta
(s/cat :_ ::whitespace
:key ::token
:_ #{\:}
:_ ::whitespace
:_ #{\"}
:value ::token
:_ #{\"}
:_ #{\newline}))
(s/def ::posting
(s/cat :_ ::whitespace
:account ::token
:_ ::whitespace
:amount ::token
:_ ::whitespace
:currency ::token
:_ #{\newline}
:meta (s/* ::meta)))
(s/def ::transaction
(s/cat
:date ::token
:_ ::whitespace
:_ ::token
:_ ::whitespace
:desc ::quoted-token
:_ #{\newline}
:meta (s/* ::meta)
:postings (s/* ::posting)
:_ (s/* #{\newline})))
(s/def ::transactions
(s/+ ::transaction))
(defn kv->map
"Convert map {:key \"x\" :value \"y\"} to {:x y}"
[vec]
(into {} (for [{:keys [key value]} vec]
[(keyword key) value])))
(defn convert-parse-tree
"Reformat the parse tree into a similar data structure as used during import."
[tree]
(walk/postwalk #(cond
;; vector of chars to string
(and (vector? %)
(= (type (first %)) java.lang.Character)) (apply str %)
;; posting amount to bigdec
(and (= (type %) clojure.lang.PersistentArrayMap)
(contains? % :amount)) (dissoc (update % :amount bigdec) :_)
;; flatten description
(and (= (type %) clojure.lang.PersistentArrayMap)
(contains? % :desc)) (-> %
(update :desc :token)
(dissoc :_))
;; drop :_ keys
(= (type %) clojure.lang.PersistentArrayMap) (dissoc % :_)
;; convert {:key a :value b} to {:a b}
(and (vector? %)
(= (type (first %)) clojure.lang.PersistentArrayMap)
(contains? (first %) :key)) (kv->map %)
:else %)
tree))
(defn parse
"Parse a Beancount transaction.
Use so we can convert the data generated during import with the hand-generated
Beancount transactions."
[text]
(let [tree (s/conform ::transactions (conj (vec text) \newline))]
(convert-parse-tree tree)))
(convert-parse-tree
[{:desc {:token "foo"}}])