From f91302fbd47db92b5db51d1031e312f2976d06b1 Mon Sep 17 00:00:00 2001 From: Ben Sturmfels Date: Fri, 23 Feb 2024 10:09:49 +1100 Subject: [PATCH] Move grouping into the separate report functions A little simpler this way because the data each report function receives looks more like the CSV. Also avoids the need to ungroup for `employer-taxes`. --- src/core.clj | 36 ++++++++++++++++++------------------ src/import.clj | 40 +++++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/src/core.clj b/src/core.clj index 01fdf60..853e3c7 100644 --- a/src/core.clj +++ b/src/core.clj @@ -37,12 +37,12 @@ (when (:help options) (println summary) (System/exit 0)) - (let [grouped-data (import/read-grouped-csv (:csv options)) - imported (concat (import/payroll date period pay-receipt-no grouped-data) - (import/individual-taxes date period pay-receipt-no pay-invoice-no grouped-data) - (import/employer-taxes date period pay-receipt-no grouped-data) - (import/payroll-fees date period fees-receipt-no fees-invoice-no total-fees grouped-data) - (import/retirement date period retirement-receipt-no retirement-invoice-no grouped-data))] + (let [data (import/read-csv (:csv options)) + imported (concat (import/payroll date period pay-receipt-no data) + (import/individual-taxes date period pay-receipt-no pay-invoice-no data) + (import/employer-taxes date period pay-receipt-no data) + (import/payroll-fees date period fees-receipt-no fees-invoice-no total-fees data) + (import/retirement date period retirement-receipt-no retirement-invoice-no data))] (doseq [i imported] (println (import/render-transaction i)))))) @@ -55,25 +55,25 @@ ;; These examples are not included with the code for privacy reasons. (require '[examples]) - (def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2023-12-27_Pay-Item-Details_2023-12-2.csv")) + (def data (import/read-csv "/home/ben/Downloads/2023-12-27_Pay-Item-Details_2023-12-2.csv")) (def imported - (concat (import/payroll "2023-12-29" "December 2023" "rt:19462/674660" grouped-data) - (import/individual-taxes "2023-12-29" "December 2023" "rt:19462/674660" "rt:19403/675431" grouped-data) - (import/employer-taxes "2023-12-29" "December 2023" "rt:19462/674660" grouped-data) - (import/payroll-fees "2023-12-29" "December 2023" "rt:19459/675387" "rt:19459/674887" 206.50M grouped-data) - (import/retirement "2024-01-02" "December 2023" "rt:19403/676724" "rt:19403/675431" grouped-data))) + (concat (import/payroll "2023-12-29" "December 2023" "rt:19462/674660" data) + (import/individual-taxes "2023-12-29" "December 2023" "rt:19462/674660" "rt:19403/675431" data) + (import/employer-taxes "2023-12-29" "December 2023" "rt:19462/674660" data) + (import/payroll-fees "2023-12-29" "December 2023" "rt:19459/675387" "rt:19459/674887" 206.50M data) + (import/retirement "2024-01-02" "December 2023" "rt:19403/676724" "rt:19403/675431" data))) (dd/pretty-print (dd/diff (sort-postings (parse examples/dec-2023)) (sort-postings imported))) - (def grouped-data (import/read-grouped-csv "/home/ben/Downloads/2024-01-29_Pay-Item-Details_2024-01.csv")) + (def data (import/read-csv "/home/ben/Downloads/2024-01-29_Pay-Item-Details_2024-01.csv")) (def imported - (concat (import/payroll "2024-01-31" "January 2024" "rt:19462/685751" grouped-data) - (import/individual-taxes "2024-01-31" "January 2024" "rt:19462/685751" "rt:19403/685602" grouped-data) - (import/employer-taxes "2024-01-31" "January 2024" "rt:19462/685751" grouped-data) - (import/payroll-fees "2024-01-31" "January 2024" "rt:19459/675387" "rt:19459/674887" 206.50M grouped-data) - (import/retirement "2024-01-31" "January 2024" "rt:19403/685929" "rt:19403/685602" grouped-data))) + (concat (import/payroll "2024-01-31" "January 2024" "rt:19462/685751" data) + (import/individual-taxes "2024-01-31" "January 2024" "rt:19462/685751" "rt:19403/685602" data) + (import/employer-taxes "2024-01-31" "January 2024" "rt:19462/685751" data) + (import/payroll-fees "2024-01-31" "January 2024" "rt:19459/675387" "rt:19459/674887" 206.50M data) + (import/retirement "2024-01-31" "January 2024" "rt:19403/685929" "rt:19403/685602" data))) ;; Compare hand-written transactions with imported (ignoring ordering). (dd/pretty-print diff --git a/src/import.clj b/src/import.clj index fca4a55..d72f1de 100644 --- a/src/import.clj +++ b/src/import.clj @@ -17,19 +17,17 @@ (str/replace ", " "-") (str/replace #" \w$" "")))) -(defn read-grouped-csv +(defn read-csv "Read in CSV and return a vector of maps with only the fields we want. Merges the various number fields into a single \"amount\" field." [filename] (with-open [reader (io/reader filename)] (doall - (group-by - :name - (for [[_ name _ category type & totals] (csv/read-csv reader)] - {:name (employee-name->entity-tag name) - :category category - :type type - :amount (apply max (map bigdec (remove str/blank? totals)))}))))) + (for [[_ name _ category type & totals] (csv/read-csv reader)] + {:name (employee-name->entity-tag name) + :category category + :type type + :amount (apply max (map bigdec (remove str/blank? totals)))})))) (defn- cat->payroll-type "Map the CSV withholding categories to Beancount payroll-type tags." @@ -118,8 +116,8 @@ (defn payroll "Return a net pay transaction." - [date period receipt-no groups] - (let [postings (for [[name records] groups] + [date period receipt-no data] + (let [postings (for [[name records] (group-by :name data)] (let [total-net-pay (->> records (filter #(= (:type %) "Net Pay")) (map :amount) @@ -156,8 +154,8 @@ (defn individual-taxes "Return a transaction of expenses/witholding for each employee." - [date period receipt-no invoice-no groups] - (for [[name records] groups] + [date period receipt-no invoice-no data] + (for [[name records] (group-by :name data)] (let [retirement-lines (filter #(= (:type %) "Retirement") records) witholding-lines (filter #(= (:type %) "Withholding") records) ;; TODO: We seem to add these extra insurance/asset postings for @@ -217,10 +215,9 @@ (defn employer-taxes "Return an employer taxes transaction." - [date period receipt-no groups] - (let [ungrouped (apply concat (vals groups)) ; Grouping by employee not useful here - liability-lines (filter #(and (= (:type %) "Liability") - (not (str/includes? (:category %) "Unemploy"))) ungrouped) + [date period receipt-no data] + (let [liability-lines (filter #(and (= (:type %) "Liability") + (not (str/includes? (:category %) "Unemploy"))) data) liability-postings (for [{:keys [amount name category]} liability-lines] {:account "Expenses:Payroll:Tax" :amount amount @@ -232,7 +229,7 @@ :payroll-type (str/replace (cat->payroll-type category) "Tax:" "")})}) total-liabilities (->> liability-postings (map :amount) (reduce +)) unemploy-lines (filter #(and (= (:type %) "Liability") - (str/includes? (:category %) "Unemploy")) ungrouped) + (str/includes? (:category %) "Unemploy")) data) unemploy-postings (for [{:keys [amount name category]} unemploy-lines] {:account "Expenses:Payroll:Tax" :amount amount @@ -257,8 +254,9 @@ (defn payroll-fees "Return a payroll fees transaction." - [date period receipt-no invoice-no total-fees groups] - (let [employees (keys groups) + [date period receipt-no invoice-no total-fees data] + (let [employees (distinct (map :name data)) + _ (println employees) exact-fee-allocation (split-fee total-fees (count employees)) employee-fees (map vector employees exact-fee-allocation) expense-postings (for [[name fee] employee-fees] @@ -280,8 +278,8 @@ (defn retirement "Return a retirement transaction." - [date period receipt-no invoice-no groups] - (let [liability-postings (for [[name records] groups] + [date period receipt-no invoice-no data] + (let [liability-postings (for [[name records] (group-by :name data)] (let [total-retirement (->> records (filter #(= (:type %) "Retirement")) (map :amount)