payroll-import/src/import.clj

;; Copyright 2024 Ben Sturmfels
;; License: GPLv3-or-later

(ns import
  (:require [clojure.data.csv :as csv]
            [clojure.java.io :as io]
            [clojure.string :as str]))

(defn- employee-name->entity-tag
  "Convert the name from the Pay Item Details report into an entity slug for Beancount."
  [name]
  ;; Should potentially be a lookup table in config.
  (case name
    "Sharp, Sage A" "Sharp-Sage-A"
    (-> name
        (str/replace " Jr." "")
        (str/replace ", " "-")
        (str/replace #" \w$" ""))))

(defn read-csv
  "Read in CSV and return a vector of maps with only the fields we want.
  Merges the various number fields into a single \"amount\" field."
  [filename]
  (with-open [reader (io/reader filename)]
    (doall
     (for [[_ name _ category type & totals] (csv/read-csv reader)]
       {:name (employee-name->entity-tag name)
        :category category
        :type type
        :amount (apply max (map bigdec (remove str/blank? totals)))}))))

(defn- cat->payroll-type
  "Map the CSV withholding categories to Beancount payroll-type tags."
  [cat]
  ;; May need to become config, similar to names. Should prompt you if no
  ;; mapping exists to avoid mistakes.
  (case cat
    "Fed Income Tax" "US:Tax:Income"
    "Medicare" "US:Tax:Medicare"
    "IL Income Tax" "US:IL:Tax:Income"
    "NY Income Tax" "US:NY:Tax:Income"
    "NYC Income Tax" "US:NY:Tax:NYC"
    "OR Income Tax" "US:OR:Tax:Income"
    "OH Income Tax" "US:OH:Tax:Income"
    "PNTSD Income Tax" "US:OH:Tax:PNTSD"
    "COLMB Income Tax" "US:OH:Tax:COLUMB"
    "Social Security" "US:Tax:SocialSecurity"
    "NY Disability" "US:NY:Disability"
    "OR Disability PFL" "US:OR:Disability:PFL"
    "NY Disability PFL" "US:NY:Disability:PFL"
    "OR TRANS STT" "US:OR:Tax:STT"
    "Fed Unemploy" "US:Unemployment"
    "IL Unemploy" "IL:Unemployment"
    "NY Unemploy" "NY:Unemployment"
    "OR Unemploy" "OR:Unemployment"
    "NY Re-empl Svc" "US:NY:Reempt"
    cat))

(defn- assoc-project
  "Conditionally adds a specific project tag to metadata.
  Some employees need to be tagged with a particular project to override the
  default \"Conservancy\" project."
  [projects name m]
  (if (contains? projects name)
    (assoc m :project (get projects name))
    m))

(defn total
  "Sum the provided records."
  [records]
  (->> records (map :amount) (reduce + 0M)))

(defn total-by-type
  "Sum up the provided records that match this type."
  [type records]
  (->> records
       (filter #(= (:type %) type))
       (map :amount)
       (reduce + 0M)))

(defn- split-fee
  "Share a total fee into n groups allocating the remainder as evenly as possible.
  (split-fee 36.02 4) => (9.01M 9.01M 9.00M 9.00M)"
  [total n]
  (let [total (bigdec total)
        fee-share (.setScale (bigdec (/ (double total) n)) 2 java.math.RoundingMode/FLOOR)
        extra-cents (* 100M (- total (* fee-share n)))
        base-fee-allocation (repeat fee-share)
        cents-allocation (take n (concat (repeat extra-cents 0.01M) (repeat 0)))]
    (map + base-fee-allocation cents-allocation)))

(defn render-transaction
  "Turn a Beancount transaction data structure into text."
  [{:keys [date payee desc meta postings]}]
  (str/join (concat
             ;; Transaction date/description
             [(if payee
                (format "%s txn \"%s\" \"%s\"\n" date payee desc)
                (format "%s txn \"%s\"\n" date desc))]
             ;; Transaction metadata
             (for [[k v] meta]
               (format "  %s: \"%s\"\n" (name k) v))
             ;; Postings and posting metadata
             (for [{:keys [amount account currency meta]} postings]
               (when-not (zero? amount)
                 (format "  %-40s  %10.2f %s\n%s" account amount currency
                         (str/join (for [[k v] meta]
                                     (format "     %s: \"%s\"\n" (name k) v)))))))))

;; Each of the below functions returns one of the five payroll
;; transaction "sections" (one or more Beancount transaction entry):
;;
;; * net pay (single transaction)
;; * individual taxes (one transaction for each employee)
;; * employer taxes (single transaction)
;; * fees (single transaction)
;; * retirement (single trasaction)
;;
;; These transaction sections are described in detail in
;; beancount/doc/Payroll.md (though in truth the initial importer was written
;; without realising that file existed and instead by precisely matching the
;; recent manually created payroll transactions).
;;
;; These functions take the input CSV records, pre-formatted and grouped by
;; employee.
;;
;; The output is an intermediate data structure that can then be run through
;; `render-transaction` to produce Beancount transaction text. The data
;; structures can also be programatically compared to a parsed version of the
;; hand-written Beancount transactions for development and testing (see
;; parse.clj).
;;
;; (All functions return a sequence of transactions so we can concatenate them.)

(defn net-pay
  "Return a net pay transaction."
  [date period invoice-no projects records]
  (let [template {:date date :desc (format "Monthly Payroll - %s - Net Pay" period)
                  :meta {:program "Conservancy:Payroll"
                         :project "Conservancy"
                         :invoice invoice-no
                         :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
                         :tax-implication "W2"
                         :payroll-type "US:General"}
                  :postings []}
        postings (flatten
                  (for [[name employee-records] (group-by :name records)]
                    (let [total-net-pay (total-by-type "Net Pay" employee-records)
                          total-reimbursement (total-by-type "Reimbursement" employee-records)
                          actual-total-net-pay (- total-net-pay total-reimbursement)]
                      [{:account "Expenses:Payroll:Salary"
                        :amount actual-total-net-pay :currency "USD"
                        :meta (assoc-project projects name {:entity name})}
                       {:account "Liabilities:Payable:Accounts"
                        :amount (- actual-total-net-pay) :currency "USD"
                        :meta {:entity name}}
                       {:account "Expenses:Hosting"
                        :amount total-reimbursement :currency "USD"
                        :meta (assoc-project projects name {:entity name :payroll-type "US:Reimbursement"})}
                       {:account "Liabilities:Payable:Accounts"
                        :amount (- total-reimbursement) :currency "USD"
                        :meta (assoc-project projects name {:entity name :tax-implication "Reimbursement"})}])))]
    [(assoc template :postings postings)]))

(defn individual-taxes
  "Return a transaction of expenses/witholding for each employee."
  [date period pay-invoice-no retirement-invoice-no projects records]
  (for [[name employee-records] (group-by :name records)]
    (let [template {:date date :desc (format "Monthly Payroll - %s - TAXES - %s" period name)
                    :meta (assoc-project
                           projects name
                           {:project "Conservancy"
                            :program "Conservancy:Payroll"
                            :entity name
                            :invoice pay-invoice-no
                            :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"})
                    :postings []}
          retirement-records (filter #(= (:type %) "Retirement") employee-records)
          witholding-records (filter #(= (:type %) "Withholding") employee-records)
          ;; We add these extra disability insurance/asset postings for NY only
          ;; as discussed in beancount/doc/Payroll.md.
          insurance-records (filter (fn [{:keys [category type]}]
                                    (and (= type "Withholding")
                                         (str/starts-with? category "NY Disability"))) employee-records)
          total-retirement (total retirement-records)
          retirement-postings (for [{:keys [category amount]} retirement-records]
                                (if (= category "403b ER match")
                                  {:account "Expenses:Payroll:Salary"
                                   :amount amount :currency "USD"
                                   :meta {:payroll-type "US:403b:Match"
                                          :invoice retirement-invoice-no}}
                                  {:account "Expenses:Payroll:Salary"
                                   :amount amount :currency "USD"
                                   :meta {:payroll-type "US:403b:Employee"
                                          :invoice retirement-invoice-no}}))
          liability-postings [{:account "Liabilities:Payable:Accounts"
                               :amount (- total-retirement) :currency "USD"
                               :meta {:invoice retirement-invoice-no}}]
          withholding-postings (for [{:keys [category amount]} witholding-records]
                                 {:account "Expenses:Payroll:Salary"
                                  :amount amount :currency "USD"
                                  :meta {:payroll-type (cat->payroll-type category)}})
          withholding-asset-postings [{:account "Liabilities:Payable:Accounts"
                                       :amount (- (total witholding-records)) :currency "USD"
                                       :meta {:tax-implication "W2"}}]
          insurance-postings (for [{:keys [category amount]} insurance-records]
                               {:account "Expenses:Insurance"
                                :amount (- amount) :currency "USD"
                                :meta {:payroll-type (cat->payroll-type category)}})
          insurance-asset-postings [{:account "Liabilities:Payable:Accounts"
                                     :amount (total insurance-records) :currency "USD"}]
          all-postings (concat
                        retirement-postings
                        liability-postings
                        withholding-postings
                        withholding-asset-postings
                        insurance-postings
                        insurance-asset-postings)]
      (assoc template :postings all-postings))))

(defn employer-taxes
  "Return an employer taxes transaction."
  [date period invoice-no projects records]
  (let [template {:date date :desc (format "Monthly Payroll - %s - TAXES - Employer" period)
                  :meta {:program "Conservancy:Payroll"
                         :project "Conservancy"
                         :invoice invoice-no
                         :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
                  :postings []}
        liability-records (filter (fn [{:keys [category type]}]
                                  (and (= type "Liability")
                                       (not (str/includes? category "Unemploy")))) records)
        liability-postings (for [{:keys [amount name category]} liability-records]
                             {:account "Expenses:Payroll:Tax"
                              :amount amount :currency "USD"
                              ;; Use eg. "US:Medicare", not "US:Tax:Medicare" as
                              ;; in individual-taxes as confirmed by Rosanne.
                              :meta (assoc-project
                                     projects
                                     name
                                     {:entity name
                                      :payroll-type (str/replace (cat->payroll-type category) "Tax:" "")})})
        total-liabilities (total liability-records)
        unemploy-records (filter (fn [{:keys [category type]}]
                                 (and (= type "Liability")
                                      (str/includes? category "Unemploy"))) records)
        unemploy-postings (for [{:keys [amount name category]} unemploy-records]
                            {:account "Expenses:Payroll:Tax"
                             :amount amount :currency "USD"
                             :meta (assoc-project projects
                                                  name
                                                  {:entity (first (str/split category #" "))
                                                   :memo name  ; distinguishes multiple employees in one state
                                                   :payroll-type (str "US:" (cat->payroll-type category))})})
        total-unemploy (total unemploy-records)
        asset-postings [{:account "Liabilities:Payable:Accounts"
                         :amount (- (+ total-liabilities total-unemploy)) :currency "USD"
                         :meta {:entity "Paychex"
                                :tax-implication "Tax-Payment"}}]
        all-postings (concat liability-postings unemploy-postings asset-postings)]
    [(assoc template :postings all-postings)]))

(defn fees
  "Return a transaction paying payroll fees to Paychex payroll fees transaction."
  [date period receipt-no invoice-no total-fees projects records]
  (let [template {:date date :payee "Paychex" :desc (format "Monthly Payroll - %s - Fee" period)
                  :meta {:program "Conservancy:Payroll"
                         :project "Conservancy"
                         :receipt receipt-no
                         :invoice invoice-no
                         :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
                         :tax-implication "USA-Corporation"}
                  :postings []}
        employees (distinct (map :name records))
        exact-fee-allocation (split-fee total-fees (count employees))
        employee-fees (map vector employees exact-fee-allocation)
        expense-postings (for [[name fee] employee-fees]
                           {:account "Expenses:Payroll:Fees"
                            :amount fee :currency "USD"
                            :meta (assoc-project projects name {:entity name})})
        asset-postings [{:account "Assets:FR:Check2721"
                         :amount (- total-fees) :currency "USD"}]
        all-postings (concat expense-postings asset-postings)]
    [(assoc template :postings all-postings)]))

(defn retirement
  "Return a transaction paying off retirement liabilities."
  [date period receipt-no invoice-no records]
  (let [template {:date date :desc (format "ASCENSUS TRUST RET PLAN - ACH DEBIT - Vanguard 403(b) - %s" period)
                  :meta {:program "Conservancy:Payroll"
                         :project "Conservancy"
                         :receipt receipt-no
                         :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
                         :tax-implication "Retirement-Pretax"
                         :invoice invoice-no}
                  :postings []}
        liability-postings (for [[name employee-records] (group-by :name records)]
                             (let [total-retirement (total-by-type "Retirement" employee-records)]
                               {:account "Liabilities:Payable:Accounts"
                                :amount total-retirement :currency "USD"
                                :meta {:entity name}}))
        total-liabilities (total liability-postings)
        asset-postings [{:account "Assets:FR:Check1345"
                         :amount (- total-liabilities) :currency "USD"}]
        all-postings (concat liability-postings asset-postings)]
    [(assoc template :postings all-postings)]))


(defn net-pay-ach-debit
  "Return a transaction paying off net pay liabilities"
  [date period receipt-no invoice-no projects records]
  (let [template {:date date :desc (format "Monthly Payroll - %s - Net Pay - ACH debit" period)
                  :meta {:program "Conservancy:Payroll"
                         :project "Conservancy"
                         :receipt receipt-no
                         :invoice invoice-no
                         :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"
                         :tax-implication "W2"
                         :payroll-type "US:General"}
                  :postings []}
        employee-postings (flatten
                           (for [[name employee-records] (group-by :name records)]
                             (let [net-pay (total-by-type "Net Pay" employee-records)
                                   reimbursements (total-by-type "Reimbursement" employee-records)]
                               [{:account "Liabilities:Payable:Accounts"
                                 :amount (- net-pay reimbursements) :currency "USD"
                                 :meta (assoc-project projects name {:entity name})}
                                {:account "Liabilities:Payable:Accounts"
                                 :amount reimbursements :currency "USD"
                                 :meta (assoc-project projects name {:entity name})}])))
        total-net-pay (total-by-type "Net Pay" records)
        total-reimbursements (total-by-type "Reimbursement" records)
        total-net-pay-posting {:account "Assets:FR:Check2721"
                               :amount (- (- total-net-pay total-reimbursements)) :currency "USD"
                               :meta {:entity "Paychex" :tax-implication "W2"}}
        total-reimbursements-posting {:account "Assets:FR:Check2721"
                                      :amount (- total-reimbursements) :currency "USD"
                                      :meta {:entity "Paychex" :tax-implication "Reimbursement"}}
        all-postings (concat employee-postings [total-net-pay-posting total-reimbursements-posting])]
    [(assoc template :postings all-postings)]))

(defn taxes-ach-debit
  "Return a transaction paying off tax liabilities."
  [date period receipt-no invoice-no projects records]
  (let [template {:date date :desc (format "Monthly Payroll - %s - TAXES - ACH debit" period)
                  :meta {:program "Conservancy:Payroll"
                         :project "Conservancy"
                         :receipt receipt-no
                         :invoice invoice-no
                         :approval "Financial/Employment-Records/memo-re-board-approval-of-payroll.txt"}
                  :postings []}
        liability-records (filter (fn [{:keys [category type]}]
                                  (and (= type "Liability")
                                       (not (str/includes? category "Unemploy")))) records)
        total-liabilities (total liability-records)
        unemploy-records (filter (fn [{:keys [category type]}]
                                 (and (= type "Liability")
                                      (str/includes? category "Unemploy"))) records)
        total-unemploy (total unemploy-records)
        liability-postings [{:account "Liabilities:Payable:Accounts"
                             :amount (+ total-liabilities total-unemploy) :currency "USD"
                             :meta {:entity "Paychex"}}]
        withholding-liability-postings (flatten
                                        (for [[name employee-records] (group-by :name records)]
                                          (let [witholding-records (filter #(= (:type %) "Withholding") employee-records)
                                                insurance-records (filter (fn [{:keys [category type]}]
                                                                          (and (= type "Withholding")
                                                                               (str/starts-with? category "NY Disability")))
                                                                        employee-records)]
                                            [{:account "Liabilities:Payable:Accounts"
                                              :amount (total witholding-records) :currency "USD"
                                              :meta (assoc-project projects name {:entity name})}
                                             {:account "Liabilities:Payable:Accounts"
                                              :amount (- (total insurance-records)) :currency "USD"
                                              :meta (assoc-project projects name {:entity name})}])))
        asset-postings [{:account "Assets:FR:Check2721"
                         :amount (- (+ total-liabilities total-unemploy (total withholding-liability-postings))) :currency "USD"
                         :meta {:entity "Paychex" :tax-implication "Tax-Payment"}}]
        all-postings (concat withholding-liability-postings liability-postings asset-postings)]
    [(assoc template :postings all-postings)]))