diff --git a/app/controllers/nonprofits/supporters_controller.rb b/app/controllers/nonprofits/supporters_controller.rb
index c9c43cef..970e7a27 100644
--- a/app/controllers/nonprofits/supporters_controller.rb
+++ b/app/controllers/nonprofits/supporters_controller.rb
@@ -22,7 +22,23 @@ class SupportersController < ApplicationController
send_data(Format::Csv.from_vectors(supporters), filename: "supporters-#{file_date}.csv")
end
end
- end
+ end
+
+ def export
+ begin
+ @nonprofit = current_nonprofit
+ @user = current_user_id
+ ExportSupporters::initiate_export(@nonprofit.id, params, @user)
+ rescue => e
+ e
+ end
+ if e.nil?
+ flash[:notice] = "Your export was successfully initiated and you'll be emailed at #{current_user.email} as soon as it's available. Feel free to use the site in the meantime."
+ render json: {}, status: :ok
+ else
+ render json: e, status: :ok
+ end
+ end
def index_metrics
render_json do
diff --git a/app/mailers/export_mailer.rb b/app/mailers/export_mailer.rb
index 0237721d..7f49c462 100644
--- a/app/mailers/export_mailer.rb
+++ b/app/mailers/export_mailer.rb
@@ -30,4 +30,16 @@ class ExportMailer < BaseMailer
mail(to: @export.user.email, subject: 'Your recurring donations export has failed')
end
+
+ def export_supporters_completed_notification(export)
+ @export = export
+
+ mail(to: @export.user.email, subject: 'Your supporters export is available!')
+ end
+
+ def export_supporters_failed_notification(export)
+ @export = export
+
+ mail(to: @export.user.email, subject: 'Your supporters export has failed')
+ end
end
diff --git a/app/views/export_mailer/export_supporters_completed_notification.html.erb b/app/views/export_mailer/export_supporters_completed_notification.html.erb
new file mode 100644
index 00000000..a47203a8
--- /dev/null
+++ b/app/views/export_mailer/export_supporters_completed_notification.html.erb
@@ -0,0 +1,9 @@
+
Your export from <%= Format::Date.simple @export.created_at %> was completed successfully.
+
+To view your exported CSV, visit: your export.
+
+Note: your generated CSV file will be automatically deleted from our servers after seven days. Don't worry; you can always visit your recurring donations panel and export a new CSV file.
+
+If you have any questions about this export, please contact support@commitchange.com.
+
+<%= render 'emails/sig' %>
\ No newline at end of file
diff --git a/app/views/export_mailer/export_supporters_failed_notification.html.erb b/app/views/export_mailer/export_supporters_failed_notification.html.erb
new file mode 100644
index 00000000..bb8e13f7
--- /dev/null
+++ b/app/views/export_mailer/export_supporters_failed_notification.html.erb
@@ -0,0 +1,12 @@
+Your export from <%= Format::Date.simple @export.created_at %> did not succeed due to a technical error.
+While CommitChange engineers have been notified, don't hesitate to contact
+ support@commitchange.com with the following information:
+
+
+ - Export ID: <%= @export.id %>
+ - User ID: <%= @export.user_id %>
+ - Nonprofit ID: <%= @export.nonprofit_id %>
+ - Exception: <%= @export.exception %>
+
+
+<%= render 'emails/sig' %>
\ No newline at end of file
diff --git a/app/views/nonprofits/supporters/_export_supporters_s3_modal.html.erb b/app/views/nonprofits/supporters/_export_supporters_s3_modal.html.erb
new file mode 100644
index 00000000..0914d930
--- /dev/null
+++ b/app/views/nonprofits/supporters/_export_supporters_s3_modal.html.erb
@@ -0,0 +1,46 @@
+
\ No newline at end of file
diff --git a/app/views/nonprofits/supporters/index.html.erb b/app/views/nonprofits/supporters/index.html.erb
index c2904053..97e50379 100644
--- a/app/views/nonprofits/supporters/index.html.erb
+++ b/app/views/nonprofits/supporters/index.html.erb
@@ -42,6 +42,7 @@
<%= render 'manage_field_master_modal' %>
<%= render 'export_supporters_modal' %>
+<%= render 'export_supporters_s3_modal' %>
<%= render 'edit_custom_fields_modal' %>
<%= render 'edit_bulk_custom_fields_modal' %>
diff --git a/config/routes.rb b/config/routes.rb
index 26214066..cde1d099 100755
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -89,6 +89,7 @@ Commitchange::Application.routes.draw do
resources(:custom_field_joins, {only: [:index, :destroy]})
resources(:supporter_notes, {only: [:create, :update, :destroy]})
resources(:activities, {only: [:index]})
+ post(:export, {on: :collection})
put :bulk_delete, on: :collection
post :merge, on: :collection
get :merge_data, on: :collection
diff --git a/lib/export/export_supporters.rb b/lib/export/export_supporters.rb
new file mode 100644
index 00000000..c657ffb0
--- /dev/null
+++ b/lib/export/export_supporters.rb
@@ -0,0 +1,73 @@
+module ExportSupporters
+ def self.initiate_export(npo_id, params, user_id)
+
+ ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id },
+ npo_id: { required: true, is_integer: true },
+ params: { required: true, is_hash: true },
+ user_id: { required: true, is_integer: true })
+ npo = Nonprofit.where('id = ?', npo_id).first
+ unless npo
+ raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id)
+ end
+ user = User.where('id = ?', user_id).first
+ unless user
+ raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id)
+ end
+
+ e = Export.create(nonprofit: npo, user: user, status: :queued, export_type: 'ExportSupporters', parameters: params.to_json)
+
+ DelayedJobHelper.enqueue_job(ExportSupporters, :run_export, [npo_id, params.to_json, user_id, e.id])
+ end
+
+ def self.run_export(npo_id, params, user_id, export_id)
+ # need to check that
+ ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id, export_id: export_id },
+ npo_id: { required: true, is_integer: true },
+ params: { required: true, is_json: true },
+ user_id: { required: true, is_integer: true },
+ export_id: { required: true, is_integer: true })
+
+ params = JSON.parse(params, :object_class=> HashWithIndifferentAccess)
+ # verify that it's also a hash since we can't do that at once
+ ParamValidation.new({ params: params },
+ params: { is_hash: true })
+ begin
+ export = Export.find(export_id)
+ rescue ActiveRecord::RecordNotFound
+ raise ParamValidation::ValidationError.new("Export #{export_id} doesn't exist!", key: :export_id)
+ end
+ export.status = :started
+ export.save!
+
+ unless Nonprofit.exists?(npo_id)
+ raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id)
+ end
+ user = User.where('id = ?', user_id).first
+ unless user
+ raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id)
+ end
+
+ file_date = Time.now.getutc().strftime('%m-%d-%Y--%H-%M-%S')
+ filename = "tmp/csv-exports/supporters-#{file_date}.csv"
+
+ url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.for_export_enumerable(npo_id, params, 30000).map{|i| i.to_csv}, content_type: 'text/csv', content_disposition: 'attachment')
+ export.url = url
+ export.status = :completed
+ export.ended = Time.now
+ export.save!
+
+ EmailJobQueue.queue(JobTypes::ExportSupportersCompletedJob, export)
+ rescue => e
+ if export
+ export.status = :failed
+ export.exception = e.to_s
+ export.ended = Time.now
+ export.save!
+ if user
+ EmailJobQueue.queue(JobTypes::ExportSupportersFailedJob, export)
+ end
+ raise e
+ end
+ raise e
+ end
+end
\ No newline at end of file
diff --git a/lib/job_types/export_supporters_completed_job.rb b/lib/job_types/export_supporters_completed_job.rb
new file mode 100644
index 00000000..7c9b09da
--- /dev/null
+++ b/lib/job_types/export_supporters_completed_job.rb
@@ -0,0 +1,13 @@
+module JobTypes
+ class ExportSupportersCompletedJob < EmailJob
+ attr_reader :export
+
+ def initialize(export)
+ @export = export
+ end
+
+ def perform
+ ExportMailer.export_supporters_completed_notification(@export).deliver
+ end
+ end
+end
\ No newline at end of file
diff --git a/lib/job_types/export_supporters_failed_job.rb b/lib/job_types/export_supporters_failed_job.rb
new file mode 100644
index 00000000..e7555f9e
--- /dev/null
+++ b/lib/job_types/export_supporters_failed_job.rb
@@ -0,0 +1,13 @@
+module JobTypes
+ class ExportSupportersFailedJob < EmailJob
+ attr_reader :export
+
+ def initialize(export)
+ @export = export
+ end
+
+ def perform
+ ExportMailer.export_supporters_failed_notification(@export).deliver
+ end
+ end
+end
\ No newline at end of file
diff --git a/lib/query/query_supporters.rb b/lib/query/query_supporters.rb
index 5250f4f2..9b89bf6a 100644
--- a/lib/query/query_supporters.rb
+++ b/lib/query/query_supporters.rb
@@ -268,6 +268,74 @@ module QuerySupporters
return expr
end
+ def self.for_export_enumerable(npo_id, query, chunk_limit=35000)
+ ParamValidation.new({npo_id: npo_id, query:query}, {npo_id: {required: true, is_int: true},
+ query: {required:true, is_hash: true}})
+
+ return QxQueryChunker.for_export_enumerable(chunk_limit) do |offset, limit, skip_header|
+ get_chunk_of_export(npo_id, query, offset, limit, skip_header)
+ end
+
+ end
+
+ def self.get_chunk_of_export(np_id, query, offset=nil, limit=nil, skip_header=false)
+ return QxQueryChunker.get_chunk_of_query(offset, limit, skip_header) do
+ expr = full_filter_expr(np_id, query)
+ selects = supporter_export_selections.concat([
+ '(payments.sum / 100)::money::text AS total_contributed',
+ 'supporters.id AS id'
+ ])
+ if query[:export_custom_fields]
+ # Add a select/csv-column for every custom field master for this nonprofit
+ # and add a left join for every custom field master
+ # eg if the npo has a custom field like Employer with id 99, then the query will be
+ # SELECT export_cfj_Employer.value AS Employer, ...
+ # FROM supporters
+ # LEFT JOIN custom_field_joins AS export_cfj_Employer ON export_cfj_Employer.supporter_id=supporters.id AND export_cfj_Employer.custom_field_master_id=99
+ # ...
+ ids = query[:export_custom_fields].split(',').map(&:to_i)
+ if ids.any?
+ cfms = Qx.select("name", "id").from(:custom_field_masters).where(nonprofit_id: np_id).and_where("id IN ($ids)", ids: ids).ex
+ cfms.compact.map do |cfm|
+ table_alias = "cfjs_#{cfm['name'].gsub(/\$/, "")}"
+ table_alias_quot = "\"#{table_alias}\""
+ field_join_subq = Qx.select("STRING_AGG(value, ',') as value", "supporter_id")
+ .from("custom_field_joins")
+ .join("custom_field_masters" , "custom_field_masters.id=custom_field_joins.custom_field_master_id")
+ .where("custom_field_masters.id=$id", id: cfm['id'])
+ .group_by(:supporter_id)
+ .as(table_alias)
+ expr.add_left_join(field_join_subq, "#{table_alias_quot}.supporter_id=supporters.id")
+ selects = selects.concat(["#{table_alias_quot}.value AS \"#{cfm['name']}\""])
+ end
+ end
+ end
+
+
+ get_last_payment_query = Qx.select('supporter_id', "MAX(date) AS date")
+ .from(:payments)
+ .group_by("supporter_id")
+ .as("last_payment")
+
+ expr.add_left_join(get_last_payment_query, 'last_payment.supporter_id = supporters.id')
+ selects = selects.concat(['last_payment.date as "Last Payment Received"'])
+
+
+ supporter_note_query = Qx.select("STRING_AGG(supporter_notes.created_at || ': ' || supporter_notes.content, '\r\n' ORDER BY supporter_notes.created_at DESC) as notes", "supporter_notes.supporter_id")
+ .from(:supporter_notes)
+ .group_by('supporter_notes.supporter_id')
+ .as("supporter_note_query")
+
+ expr.add_left_join(supporter_note_query, 'supporter_note_query.supporter_id=supporters.id')
+ selects = selects.concat(["supporter_note_query.notes AS notes"])
+
+
+ expr.select(selects)
+ end
+ end
+
+
+
# Give supp data for csv
def self.for_export(np_id, query)
expr = full_filter_expr(np_id, query)
@@ -520,7 +588,7 @@ module QuerySupporters
.execute(format: 'csv')
end
- #
+
def self.get_min_or_max_dates_for_range(time_range_params)
begin
if (time_range_params[:year])
diff --git a/lib/qx_query_chunker.rb b/lib/qx_query_chunker.rb
new file mode 100644
index 00000000..cee3ca0b
--- /dev/null
+++ b/lib/qx_query_chunker.rb
@@ -0,0 +1 @@
+QxQueryChunker = QexprQueryChunker
\ No newline at end of file
diff --git a/spec/lib/exports/export_payments_spec.rb b/spec/lib/export/export_payments_spec.rb
similarity index 100%
rename from spec/lib/exports/export_payments_spec.rb
rename to spec/lib/export/export_payments_spec.rb
diff --git a/spec/lib/exports/export_recurring_donations_spec.rb b/spec/lib/export/export_recurring_donations_spec.rb
similarity index 100%
rename from spec/lib/exports/export_recurring_donations_spec.rb
rename to spec/lib/export/export_recurring_donations_spec.rb
diff --git a/spec/lib/export/export_supporters_spec.rb b/spec/lib/export/export_supporters_spec.rb
new file mode 100644
index 00000000..df55ac32
--- /dev/null
+++ b/spec/lib/export/export_supporters_spec.rb
@@ -0,0 +1,198 @@
+require 'rails_helper'
+require 'support/test_chunked_uploader'
+
+describe ExportSupporters do
+ before(:each) do
+ stub_const('CHUNKED_UPLOADER',TestChunkedUploader)
+ @nonprofit = force_create(:nonprofit)
+ @email = 'example@example.com'
+ @user = force_create(:user, email: @email)
+ @supporters = 2.times { force_create(:supporter, nonprofit: @nonprofit)}
+ CHUNKED_UPLOADER.clear
+ end
+ let(:export_header) { "Last Name,First Name,Full Name,Organization,Email,Phone,Address,City,State,Postal Code,Country,Anonymous?,Supporter Id,Total Contributed,Id,Last Payment Received,Notes".split(',')}
+
+ context '.initiate_export' do
+ context 'param verification' do
+ it 'performs initial verification' do
+ expect { ExportSupporters.initiate_export(nil, nil, nil) }.to(raise_error do |error|
+ expect(error).to be_a(ParamValidation::ValidationError)
+ expect(error.data.length).to eq(6)
+ expect_validation_errors(error.data, [{ key: 'npo_id', name: :required },
+ { key: 'npo_id', name: :is_integer },
+ { key: 'user_id', name: :required },
+ { key: 'user_id', name: :is_integer },
+ { key: 'params', name: :required },
+ { key: 'params', name: :is_hash }])
+ end)
+ end
+
+ it 'nonprofit doesnt exist' do
+ fake_npo = 8_888_881
+ expect { ExportSupporters.initiate_export(fake_npo, {}, 8_888_883) }.to(raise_error do |error|
+ expect(error).to be_a(ParamValidation::ValidationError)
+ expect(error.message).to eq "Nonprofit #{fake_npo} doesn't exist!"
+ end)
+ end
+
+ it 'user doesnt exist' do
+ fake_user = 8_888_883
+ expect { ExportSupporters.initiate_export(@nonprofit.id, {}, fake_user) }.to(raise_error do |error|
+ expect(error).to be_a(ParamValidation::ValidationError)
+ expect(error.message).to eq "User #{fake_user} doesn't exist!"
+ end)
+ end
+ end
+
+ it 'creates an export object and schedules job' do
+ Timecop.freeze(2020, 4, 5) do
+ DelayedJobHelper = double('delayed')
+ params = { param1: 'pp', root_url: 'https://localhost:8080' }.with_indifferent_access
+
+ expect(Export).to receive(:create).and_wrap_original {|m, *args|
+ e = m.call(*args) # get original create
+ expect(DelayedJobHelper).to receive(:enqueue_job).with(ExportSupporters, :run_export, [@nonprofit.id, params.to_json, @user.id, e.id]) #add the enqueue
+ e
+ }
+
+
+ ExportSupporters.initiate_export(@nonprofit.id, params, @user.id)
+ export = Export.first
+ expected_export = { id: export.id,
+ user_id: @user.id,
+ nonprofit_id: @nonprofit.id,
+ status: 'queued',
+ export_type: 'ExportSupporters',
+ parameters: params.to_json,
+ updated_at: Time.now,
+ created_at: Time.now,
+ url: nil,
+ ended: nil,
+ exception: nil }.with_indifferent_access
+ expect(export.attributes).to eq(expected_export)
+ end
+ end
+ end
+ context '.run_export' do
+ context 'param validation' do
+ it 'rejects basic invalid data' do
+ expect { ExportSupporters.run_export(nil, nil, nil, nil) }.to(raise_error do |error|
+ expect(error).to be_a(ParamValidation::ValidationError)
+ expect_validation_errors(error, [{ key: 'npo_id', name: :required },
+ { key: 'npo_id', name: :is_integer },
+ { key: 'user_id', name: :required },
+ { key: 'user_id', name: :is_integer },
+ { key: 'params', name: :required },
+ { key: 'params', name: :is_json },
+ { key: 'export_id', name: :required },
+ { key: 'export_id', name: :is_integer }])
+ end)
+ end
+
+ it 'rejects json which isnt a hash' do
+ expect { ExportSupporters.run_export(1, [{ item: '' }, { item: '' }].to_json, 1, 1) }.to(raise_error do |error|
+ expect(error).to be_a(ParamValidation::ValidationError)
+ expect_validation_errors(error, [
+ { key: :params, name: :is_hash }
+ ])
+ end)
+ end
+
+ it 'no export throw an exception' do
+ expect { ExportSupporters.run_export(0, { x: 1 }.to_json, 0, 11_111) }.to(raise_error do |error|
+ expect(error).to be_a ParamValidation::ValidationError
+ expect(error.data[:key]).to eq :export_id
+ expect(error.message).to start_with('Export')
+ end)
+ end
+
+ it 'no nonprofit' do
+ Timecop.freeze(2020, 4, 5) do
+ @export = force_create(:export, user: @user)
+ Timecop.freeze(2020, 4, 6) do
+ expect { ExportSupporters.run_export(0, { x: 1 }.to_json, @user.id, @export.id) }.to(raise_error do |error|
+ expect(error).to be_a ParamValidation::ValidationError
+ expect(error.data[:key]).to eq :npo_id
+ expect(error.message).to start_with('Nonprofit')
+
+ @export.reload
+ expect(@export.status).to eq 'failed'
+ expect(@export.exception).to eq error.to_s
+ expect(@export.ended).to eq Time.now
+ expect(@export.updated_at).to eq Time.now
+
+ end)
+ end
+ end
+ end
+
+ it 'no user' do
+ Timecop.freeze(2020, 4, 5) do
+ @export = force_create(:export, user: @user)
+ Timecop.freeze(2020, 4, 6) do
+ expect { ExportSupporters.run_export(@nonprofit.id, { x: 1 }.to_json, 0, @export.id) }.to(raise_error do |error|
+ expect(error).to be_a ParamValidation::ValidationError
+ expect(error.data[:key]).to eq :user_id
+ expect(error.message).to start_with('User')
+
+ @export.reload
+ expect(@export.status).to eq 'failed'
+ expect(@export.exception).to eq error.to_s
+ expect(@export.ended).to eq Time.now
+ expect(@export.updated_at).to eq Time.now
+ end)
+ end
+ end
+ end
+ end
+
+ it 'handles exception in upload properly' do
+ Timecop.freeze(2020, 4, 5) do
+ @export = force_create(:export, user: @user)
+ expect_email_queued.with(JobTypes::ExportSupportersFailedJob, @export)
+ CHUNKED_UPLOADER.raise_error
+ Timecop.freeze(2020, 4, 6) do
+ expect { ExportSupporters.run_export(@nonprofit.id, {}.to_json, @user.id, @export.id) }.to(raise_error do |error|
+ expect(error).to be_a StandardError
+ expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE
+
+ @export.reload
+ expect(@export.status).to eq 'failed'
+ expect(@export.exception).to eq error.to_s
+ expect(@export.ended).to eq Time.now
+ expect(@export.updated_at).to eq Time.now
+
+
+ end)
+ end
+ end
+ end
+
+ it 'uploads as expected' do
+ Timecop.freeze(2020, 4, 5) do
+ @export = create(:export, user: @user, created_at: Time.now, updated_at: Time.now)
+ expect_email_queued.with(JobTypes::ExportSupportersCompletedJob, @export)
+ Timecop.freeze(2020, 4, 6, 1, 2, 3) do
+ ExportSupporters.run_export(@nonprofit.id, {:root_url => "https://localhost:8080/"}.to_json, @user.id, @export.id)
+
+ @export.reload
+
+ expect(@export.url).to eq 'http://fake.url/tmp/csv-exports/supporters-04-06-2020--01-02-03.csv'
+ expect(@export.status).to eq 'completed'
+ expect(@export.exception).to be_nil
+ expect(@export.ended).to eq Time.now
+ expect(@export.updated_at).to eq Time.now
+ csv = CSV.parse(TestChunkedUploader.output)
+ expect(csv.length).to eq (3)
+
+ expect(csv[0]).to eq export_header
+
+ expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv'
+ expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment'
+
+ end
+ end
+ end
+ end
+
+end
\ No newline at end of file
diff --git a/spec/lib/job_types/export_supporters_completed_job_spec.rb b/spec/lib/job_types/export_supporters_completed_job_spec.rb
new file mode 100644
index 00000000..b8f79aeb
--- /dev/null
+++ b/spec/lib/job_types/export_supporters_completed_job_spec.rb
@@ -0,0 +1,13 @@
+require 'rails_helper.rb'
+
+describe JobTypes::ExportSupportersCompletedJob do
+ describe '.perform' do
+ it 'calls the correct active mailer' do
+ input = 1
+ expect(ExportMailer).to receive(:export_supporters_completed_notification).with(input).and_wrap_original{|m, *args| mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer}
+
+ job = JobTypes::ExportSupportersCompletedJob.new(input)
+ job.perform
+ end
+ end
+end
\ No newline at end of file
diff --git a/spec/lib/job_types/export_supporters_failed_job_spec.rb b/spec/lib/job_types/export_supporters_failed_job_spec.rb
new file mode 100644
index 00000000..b1b57d11
--- /dev/null
+++ b/spec/lib/job_types/export_supporters_failed_job_spec.rb
@@ -0,0 +1,13 @@
+require 'rails_helper.rb'
+
+describe JobTypes::ExportSupportersFailedJob do
+ describe '.perform' do
+ it 'calls the correct active mailer' do
+ input = 1
+ expect(ExportMailer).to receive(:export_supporters_failed_notification).with(input).and_wrap_original{|m, *args| mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer}
+
+ job = JobTypes::ExportSupportersFailedJob.new(input)
+ job.perform
+ end
+ end
+end
\ No newline at end of file