NPO-Accounting/houdini
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Support for generated suppoters.csv in the background

Browse source
This commit is contained in:
Eric Schultz 2018-05-07 16:05:27 -05:00 committed by Eric Schultz
parent 45f8923382
commit ec6f06f387
17 changed files with 491 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
app/controllers/nonprofits/supporters_controller.rb
View file

@ -24,6 +24,22 @@ class SupportersController < ApplicationController
end
end
def export
begin
@nonprofit = current_nonprofit
@user = current_user_id
ExportSupporters::initiate_export(@nonprofit.id, params, @user)
rescue => e
e
end
if e.nil?
flash[:notice] = "Your export was successfully initiated and you'll be emailed at #{current_user.email} as soon as it's available. Feel free to use the site in the meantime."
render json: {}, status: :ok
else
render json: e, status: :ok
end
end
def index_metrics
render_json do
QuerySupporters.full_search_metrics(params[:nonprofit_id], params)

12
app/mailers/export_mailer.rb
View file

@ -30,4 +30,16 @@ class ExportMailer < BaseMailer
mail(to: @export.user.email, subject: 'Your recurring donations export has failed')
end
def export_supporters_completed_notification(export)
@export = export
mail(to: @export.user.email, subject: 'Your supporters export is available!')
end
def export_supporters_failed_notification(export)
@export = export
mail(to: @export.user.email, subject: 'Your supporters export has failed')
end
end

9
app/views/export_mailer/export_supporters_completed_notification.html.erb Normal file
View file

@ -0,0 +1,9 @@
<p>Your export from <%= Format::Date.simple @export.created_at %> was completed successfully.</p>
<p>To view your exported CSV, visit: <a href='<%= @export.url %>'>your export.</a></p>
<p>Note: your generated CSV file will be automatically deleted from our servers after seven days. Don't worry; you can always visit your recurring donations panel and export a new CSV file.</p>
<p>If you have any questions about this export, please contact <a href="mailto:support@commitchange.com">support@commitchange.com</a>.</p>
<%= render 'emails/sig' %>

12
app/views/export_mailer/export_supporters_failed_notification.html.erb Normal file
View file

@ -0,0 +1,12 @@
<p>Your export from <%= Format::Date.simple @export.created_at %> did not succeed due to a technical error.</p>
<p>While CommitChange engineers have been notified, don't hesitate to contact
<a href="mailto:support@commitchange.com">support@commitchange.com</a> with the following information:</p>
<ul>
<li>Export ID: <%= @export.id %></li>
<li>User ID: <%= @export.user_id %></li>
<li>Nonprofit ID: <%= @export.nonprofit_id %></li>
<li>Exception: <%= @export.exception %></li>
</ul>
<%= render 'emails/sig' %>

46
app/views/nonprofits/supporters/_export_supporters_s3_modal.html.erb Normal file
View file

@ -0,0 +1,46 @@
<div class='modal skinny' id='exportSupportersS3Modal'>
<%= render 'common/modal_header', title: 'Export Supporters' %>
<div class='modal-body'>
<div class='u-marginBottom--10'>
<!--= show_if supporters.filter_count -->
<p class="filteringBy">
Filtering by:
<!--= put (readable_filter_names supporters.query) -->
</p>
<hr>
</div>
<div class='u-overflow--hidden'>
<!--= if show_fields_in_export (remove_class 'u-hide') (add_class 'u-hide') -->
<p>
Select which custom fields you want to export:
</p>
<p>
<!--= repeat custom_fields.masters.data -->
<input type='checkbox' class='checkbox'>
<!--= set_attr 'value' this.id -->
<!--= set_attr 'id' (cat 'checkbox-export-custom-fields-' this.i -->
<!--= on 'change' set_export_custom_fields -->
<label>
<!--= set_attr 'for' (cat 'checkbox-export-custom-fields-' this.i) -->
<!--= put this.name -->
</label>
</p>
</div>
<p>
<a> Export custom fields </a>
<!--= on 'click' (toggle 'show_fields_in_export' true) -->
</p>
<form autosubmit method="POST" data-success-message="Your export was successfully initiated and you'll be emailed as soon as it's available. Feel free to use the site in the meantime." data-reload>
<!--= set_attr 'action' (cat '/nonprofits/<%= @nonprofit.id %>/supporters/export.json' (obj_to_url_params supporters.query)) -->
<div class='u-centered'>
<p class='error'></p>
<button id='csv-export-button' class='button--small' type='submit' data-loading='Initiating Download'>
<i class='fa fa-cloud-download'></i> Download as CSV
</button>
</div>
</form>
</div>
</div>

1
app/views/nonprofits/supporters/index.html.erb
View file

@ -42,6 +42,7 @@
<%= render 'manage_field_master_modal' %>
<%= render 'export_supporters_modal' %>
<%= render 'export_supporters_s3_modal' %>
<%= render 'edit_custom_fields_modal' %>
<%= render 'edit_bulk_custom_fields_modal' %>

1
config/routes.rb
View file

@ -89,6 +89,7 @@ Commitchange::Application.routes.draw do
resources(:custom_field_joins, {only: [:index, :destroy]})
resources(:supporter_notes, {only: [:create, :update, :destroy]})
resources(:activities, {only: [:index]})
post(:export, {on: :collection})
put :bulk_delete, on: :collection
post :merge, on: :collection
get :merge_data, on: :collection

73
lib/export/export_supporters.rb Normal file
View file

@ -0,0 +1,73 @@
module ExportSupporters
def self.initiate_export(npo_id, params, user_id)
ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id },
npo_id: { required: true, is_integer: true },
params: { required: true, is_hash: true },
user_id: { required: true, is_integer: true })
npo = Nonprofit.where('id = ?', npo_id).first
unless npo
raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id)
end
user = User.where('id = ?', user_id).first
unless user
raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id)
end
e = Export.create(nonprofit: npo, user: user, status: :queued, export_type: 'ExportSupporters', parameters: params.to_json)
DelayedJobHelper.enqueue_job(ExportSupporters, :run_export, [npo_id, params.to_json, user_id, e.id])
end
def self.run_export(npo_id, params, user_id, export_id)
# need to check that
ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id, export_id: export_id },
npo_id: { required: true, is_integer: true },
params: { required: true, is_json: true },
user_id: { required: true, is_integer: true },
export_id: { required: true, is_integer: true })
params = JSON.parse(params, :object_class=> HashWithIndifferentAccess)
# verify that it's also a hash since we can't do that at once
ParamValidation.new({ params: params },
params: { is_hash: true })
begin
export = Export.find(export_id)
rescue ActiveRecord::RecordNotFound
raise ParamValidation::ValidationError.new("Export #{export_id} doesn't exist!", key: :export_id)
end
export.status = :started
export.save!
unless Nonprofit.exists?(npo_id)
raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id)
end
user = User.where('id = ?', user_id).first
unless user
raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id)
end
file_date = Time.now.getutc().strftime('%m-%d-%Y--%H-%M-%S')
filename = "tmp/csv-exports/supporters-#{file_date}.csv"
url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.for_export_enumerable(npo_id, params, 30000).map{|i| i.to_csv}, content_type: 'text/csv', content_disposition: 'attachment')
export.url = url
export.status = :completed
export.ended = Time.now
export.save!
EmailJobQueue.queue(JobTypes::ExportSupportersCompletedJob, export)
rescue => e
if export
export.status = :failed
export.exception = e.to_s
export.ended = Time.now
export.save!
if user
EmailJobQueue.queue(JobTypes::ExportSupportersFailedJob, export)
end
raise e
end
raise e
end
end

13
lib/job_types/export_supporters_completed_job.rb Normal file
View file

@ -0,0 +1,13 @@
module JobTypes
class ExportSupportersCompletedJob < EmailJob
attr_reader :export
def initialize(export)
@export = export
end
def perform
ExportMailer.export_supporters_completed_notification(@export).deliver
end
end
end

13
lib/job_types/export_supporters_failed_job.rb Normal file
View file

@ -0,0 +1,13 @@
module JobTypes
class ExportSupportersFailedJob < EmailJob
attr_reader :export
def initialize(export)
@export = export
end
def perform
ExportMailer.export_supporters_failed_notification(@export).deliver
end
end
end

70
lib/query/query_supporters.rb
View file

@ -268,6 +268,74 @@ module QuerySupporters
return expr
end
def self.for_export_enumerable(npo_id, query, chunk_limit=35000)
ParamValidation.new({npo_id: npo_id, query:query}, {npo_id: {required: true, is_int: true},
query: {required:true, is_hash: true}})
return QxQueryChunker.for_export_enumerable(chunk_limit) do |offset, limit, skip_header|
get_chunk_of_export(npo_id, query, offset, limit, skip_header)
end
end
def self.get_chunk_of_export(np_id, query, offset=nil, limit=nil, skip_header=false)
return QxQueryChunker.get_chunk_of_query(offset, limit, skip_header) do
expr = full_filter_expr(np_id, query)
selects = supporter_export_selections.concat([
'(payments.sum / 100)::money::text AS total_contributed',
'supporters.id AS id'
])
if query[:export_custom_fields]
# Add a select/csv-column for every custom field master for this nonprofit
# and add a left join for every custom field master
# eg if the npo has a custom field like Employer with id 99, then the query will be
# SELECT export_cfj_Employer.value AS Employer, ...
# FROM supporters
# LEFT JOIN custom_field_joins AS export_cfj_Employer ON export_cfj_Employer.supporter_id=supporters.id AND export_cfj_Employer.custom_field_master_id=99
# ...
ids = query[:export_custom_fields].split(',').map(&:to_i)
if ids.any?
cfms = Qx.select("name", "id").from(:custom_field_masters).where(nonprofit_id: np_id).and_where("id IN ($ids)", ids: ids).ex
cfms.compact.map do |cfm|
table_alias = "cfjs_#{cfm['name'].gsub(/\$/, "")}"
table_alias_quot = "\"#{table_alias}\""
field_join_subq = Qx.select("STRING_AGG(value, ',') as value", "supporter_id")
.from("custom_field_joins")
.join("custom_field_masters" , "custom_field_masters.id=custom_field_joins.custom_field_master_id")
.where("custom_field_masters.id=$id", id: cfm['id'])
.group_by(:supporter_id)
.as(table_alias)
expr.add_left_join(field_join_subq, "#{table_alias_quot}.supporter_id=supporters.id")
selects = selects.concat(["#{table_alias_quot}.value AS \"#{cfm['name']}\""])
end
end
end
get_last_payment_query = Qx.select('supporter_id', "MAX(date) AS date")
.from(:payments)
.group_by("supporter_id")
.as("last_payment")
expr.add_left_join(get_last_payment_query, 'last_payment.supporter_id = supporters.id')
selects = selects.concat(['last_payment.date as "Last Payment Received"'])
supporter_note_query = Qx.select("STRING_AGG(supporter_notes.created_at || ': ' || supporter_notes.content, '\r\n' ORDER BY supporter_notes.created_at DESC) as notes", "supporter_notes.supporter_id")
.from(:supporter_notes)
.group_by('supporter_notes.supporter_id')
.as("supporter_note_query")
expr.add_left_join(supporter_note_query, 'supporter_note_query.supporter_id=supporters.id')
selects = selects.concat(["supporter_note_query.notes AS notes"])
expr.select(selects)
end
end
# Give supp data for csv
def self.for_export(np_id, query)
expr = full_filter_expr(np_id, query)
@ -520,7 +588,7 @@ module QuerySupporters
.execute(format: 'csv')
end
#
def self.get_min_or_max_dates_for_range(time_range_params)
begin
if (time_range_params[:year])

1
lib/qx_query_chunker.rb Normal file
View file

@ -0,0 +1 @@
QxQueryChunker = QexprQueryChunker

0
spec/lib/exports/export_payments_spec.rb → spec/lib/export/export_payments_spec.rb
View file

0
spec/lib/exports/export_recurring_donations_spec.rb → spec/lib/export/export_recurring_donations_spec.rb
View file

198
spec/lib/export/export_supporters_spec.rb Normal file
View file

@ -0,0 +1,198 @@
require 'rails_helper'
require 'support/test_chunked_uploader'
describe ExportSupporters do
before(:each) do
stub_const('CHUNKED_UPLOADER',TestChunkedUploader)
@nonprofit = force_create(:nonprofit)
@email = 'example@example.com'
@user = force_create(:user, email: @email)
@supporters = 2.times { force_create(:supporter, nonprofit: @nonprofit)}
CHUNKED_UPLOADER.clear
end
let(:export_header) { "Last Name,First Name,Full Name,Organization,Email,Phone,Address,City,State,Postal Code,Country,Anonymous?,Supporter Id,Total Contributed,Id,Last Payment Received,Notes".split(',')}
context '.initiate_export' do
context 'param verification' do
it 'performs initial verification' do
expect { ExportSupporters.initiate_export(nil, nil, nil) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect(error.data.length).to eq(6)
expect_validation_errors(error.data, [{ key: 'npo_id', name: :required },
{ key: 'npo_id', name: :is_integer },
{ key: 'user_id', name: :required },
{ key: 'user_id', name: :is_integer },
{ key: 'params', name: :required },
{ key: 'params', name: :is_hash }])
end)
end
it 'nonprofit doesnt exist' do
fake_npo = 8_888_881
expect { ExportSupporters.initiate_export(fake_npo, {}, 8_888_883) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect(error.message).to eq "Nonprofit #{fake_npo} doesn't exist!"
end)
end
it 'user doesnt exist' do
fake_user = 8_888_883
expect { ExportSupporters.initiate_export(@nonprofit.id, {}, fake_user) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect(error.message).to eq "User #{fake_user} doesn't exist!"
end)
end
end
it 'creates an export object and schedules job' do
Timecop.freeze(2020, 4, 5) do
DelayedJobHelper = double('delayed')
params = { param1: 'pp', root_url: 'https://localhost:8080' }.with_indifferent_access
expect(Export).to receive(:create).and_wrap_original {|m, *args|
e = m.call(*args) # get original create
expect(DelayedJobHelper).to receive(:enqueue_job).with(ExportSupporters, :run_export, [@nonprofit.id, params.to_json, @user.id, e.id]) #add the enqueue
e
}
ExportSupporters.initiate_export(@nonprofit.id, params, @user.id)
export = Export.first
expected_export = { id: export.id,
user_id: @user.id,
nonprofit_id: @nonprofit.id,
status: 'queued',
export_type: 'ExportSupporters',
parameters: params.to_json,
updated_at: Time.now,
created_at: Time.now,
url: nil,
ended: nil,
exception: nil }.with_indifferent_access
expect(export.attributes).to eq(expected_export)
end
end
end
context '.run_export' do
context 'param validation' do
it 'rejects basic invalid data' do
expect { ExportSupporters.run_export(nil, nil, nil, nil) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect_validation_errors(error, [{ key: 'npo_id', name: :required },
{ key: 'npo_id', name: :is_integer },
{ key: 'user_id', name: :required },
{ key: 'user_id', name: :is_integer },
{ key: 'params', name: :required },
{ key: 'params', name: :is_json },
{ key: 'export_id', name: :required },
{ key: 'export_id', name: :is_integer }])
end)
end
it 'rejects json which isnt a hash' do
expect { ExportSupporters.run_export(1, [{ item: '' }, { item: '' }].to_json, 1, 1) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect_validation_errors(error, [
{ key: :params, name: :is_hash }
])
end)
end
it 'no export throw an exception' do
expect { ExportSupporters.run_export(0, { x: 1 }.to_json, 0, 11_111) }.to(raise_error do |error|
expect(error).to be_a ParamValidation::ValidationError
expect(error.data[:key]).to eq :export_id
expect(error.message).to start_with('Export')
end)
end
it 'no nonprofit' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: @user)
Timecop.freeze(2020, 4, 6) do
expect { ExportSupporters.run_export(0, { x: 1 }.to_json, @user.id, @export.id) }.to(raise_error do |error|
expect(error).to be_a ParamValidation::ValidationError
expect(error.data[:key]).to eq :npo_id
expect(error.message).to start_with('Nonprofit')
@export.reload
expect(@export.status).to eq 'failed'
expect(@export.exception).to eq error.to_s
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
end)
end
end
end
it 'no user' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: @user)
Timecop.freeze(2020, 4, 6) do
expect { ExportSupporters.run_export(@nonprofit.id, { x: 1 }.to_json, 0, @export.id) }.to(raise_error do |error|
expect(error).to be_a ParamValidation::ValidationError
expect(error.data[:key]).to eq :user_id
expect(error.message).to start_with('User')
@export.reload
expect(@export.status).to eq 'failed'
expect(@export.exception).to eq error.to_s
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
end)
end
end
end
end
it 'handles exception in upload properly' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: @user)
expect_email_queued.with(JobTypes::ExportSupportersFailedJob, @export)
CHUNKED_UPLOADER.raise_error
Timecop.freeze(2020, 4, 6) do
expect { ExportSupporters.run_export(@nonprofit.id, {}.to_json, @user.id, @export.id) }.to(raise_error do |error|
expect(error).to be_a StandardError
expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE
@export.reload
expect(@export.status).to eq 'failed'
expect(@export.exception).to eq error.to_s
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
end)
end
end
end
it 'uploads as expected' do
Timecop.freeze(2020, 4, 5) do
@export = create(:export, user: @user, created_at: Time.now, updated_at: Time.now)
expect_email_queued.with(JobTypes::ExportSupportersCompletedJob, @export)
Timecop.freeze(2020, 4, 6, 1, 2, 3) do
ExportSupporters.run_export(@nonprofit.id, {:root_url => "https://localhost:8080/"}.to_json, @user.id, @export.id)
@export.reload
expect(@export.url).to eq 'http://fake.url/tmp/csv-exports/supporters-04-06-2020--01-02-03.csv'
expect(@export.status).to eq 'completed'
expect(@export.exception).to be_nil
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
csv = CSV.parse(TestChunkedUploader.output)
expect(csv.length).to eq (3)
expect(csv[0]).to eq export_header
expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv'
expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment'
end
end
end
end
end

13
spec/lib/job_types/export_supporters_completed_job_spec.rb Normal file
View file

@ -0,0 +1,13 @@
require 'rails_helper.rb'
describe JobTypes::ExportSupportersCompletedJob do
describe '.perform' do
it 'calls the correct active mailer' do
input = 1
expect(ExportMailer).to receive(:export_supporters_completed_notification).with(input).and_wrap_original{|m, *args| mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer}
job = JobTypes::ExportSupportersCompletedJob.new(input)
job.perform
end
end
end

13
spec/lib/job_types/export_supporters_failed_job_spec.rb Normal file
View file

@ -0,0 +1,13 @@
require 'rails_helper.rb'
describe JobTypes::ExportSupportersFailedJob do
describe '.perform' do
it 'calls the correct active mailer' do
input = 1
expect(ExportMailer).to receive(:export_supporters_failed_notification).with(input).and_wrap_original{|m, *args| mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer}
job = JobTypes::ExportSupportersFailedJob.new(input)
job.perform
end
end
end