diff --git a/app/controllers/nonprofits/supporters_controller.rb b/app/controllers/nonprofits/supporters_controller.rb index c9c43cef..970e7a27 100644 --- a/app/controllers/nonprofits/supporters_controller.rb +++ b/app/controllers/nonprofits/supporters_controller.rb @@ -22,7 +22,23 @@ class SupportersController < ApplicationController send_data(Format::Csv.from_vectors(supporters), filename: "supporters-#{file_date}.csv") end end - end + end + + def export + begin + @nonprofit = current_nonprofit + @user = current_user_id + ExportSupporters::initiate_export(@nonprofit.id, params, @user) + rescue => e + e + end + if e.nil? + flash[:notice] = "Your export was successfully initiated and you'll be emailed at #{current_user.email} as soon as it's available. Feel free to use the site in the meantime." + render json: {}, status: :ok + else + render json: e, status: :ok + end + end def index_metrics render_json do diff --git a/app/mailers/export_mailer.rb b/app/mailers/export_mailer.rb index 0237721d..7f49c462 100644 --- a/app/mailers/export_mailer.rb +++ b/app/mailers/export_mailer.rb @@ -30,4 +30,16 @@ class ExportMailer < BaseMailer mail(to: @export.user.email, subject: 'Your recurring donations export has failed') end + + def export_supporters_completed_notification(export) + @export = export + + mail(to: @export.user.email, subject: 'Your supporters export is available!') + end + + def export_supporters_failed_notification(export) + @export = export + + mail(to: @export.user.email, subject: 'Your supporters export has failed') + end end diff --git a/app/views/export_mailer/export_supporters_completed_notification.html.erb b/app/views/export_mailer/export_supporters_completed_notification.html.erb new file mode 100644 index 00000000..a47203a8 --- /dev/null +++ b/app/views/export_mailer/export_supporters_completed_notification.html.erb @@ -0,0 +1,9 @@ +

Your export from <%= Format::Date.simple @export.created_at %> was completed successfully.

+ +

To view your exported CSV, visit: your export.

+ +

Note: your generated CSV file will be automatically deleted from our servers after seven days. Don't worry; you can always visit your recurring donations panel and export a new CSV file.

+ +

If you have any questions about this export, please contact support@commitchange.com.

+ +<%= render 'emails/sig' %> \ No newline at end of file diff --git a/app/views/export_mailer/export_supporters_failed_notification.html.erb b/app/views/export_mailer/export_supporters_failed_notification.html.erb new file mode 100644 index 00000000..bb8e13f7 --- /dev/null +++ b/app/views/export_mailer/export_supporters_failed_notification.html.erb @@ -0,0 +1,12 @@ +

Your export from <%= Format::Date.simple @export.created_at %> did not succeed due to a technical error.

+

While CommitChange engineers have been notified, don't hesitate to contact + support@commitchange.com with the following information:

+ + + +<%= render 'emails/sig' %> \ No newline at end of file diff --git a/app/views/nonprofits/supporters/_export_supporters_s3_modal.html.erb b/app/views/nonprofits/supporters/_export_supporters_s3_modal.html.erb new file mode 100644 index 00000000..0914d930 --- /dev/null +++ b/app/views/nonprofits/supporters/_export_supporters_s3_modal.html.erb @@ -0,0 +1,46 @@ + \ No newline at end of file diff --git a/app/views/nonprofits/supporters/index.html.erb b/app/views/nonprofits/supporters/index.html.erb index c2904053..97e50379 100644 --- a/app/views/nonprofits/supporters/index.html.erb +++ b/app/views/nonprofits/supporters/index.html.erb @@ -42,6 +42,7 @@ <%= render 'manage_field_master_modal' %> <%= render 'export_supporters_modal' %> +<%= render 'export_supporters_s3_modal' %> <%= render 'edit_custom_fields_modal' %> <%= render 'edit_bulk_custom_fields_modal' %> diff --git a/config/routes.rb b/config/routes.rb index 26214066..cde1d099 100755 --- a/config/routes.rb +++ b/config/routes.rb @@ -89,6 +89,7 @@ Commitchange::Application.routes.draw do resources(:custom_field_joins, {only: [:index, :destroy]}) resources(:supporter_notes, {only: [:create, :update, :destroy]}) resources(:activities, {only: [:index]}) + post(:export, {on: :collection}) put :bulk_delete, on: :collection post :merge, on: :collection get :merge_data, on: :collection diff --git a/lib/export/export_supporters.rb b/lib/export/export_supporters.rb new file mode 100644 index 00000000..c657ffb0 --- /dev/null +++ b/lib/export/export_supporters.rb @@ -0,0 +1,73 @@ +module ExportSupporters + def self.initiate_export(npo_id, params, user_id) + + ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id }, + npo_id: { required: true, is_integer: true }, + params: { required: true, is_hash: true }, + user_id: { required: true, is_integer: true }) + npo = Nonprofit.where('id = ?', npo_id).first + unless npo + raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id) + end + user = User.where('id = ?', user_id).first + unless user + raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id) + end + + e = Export.create(nonprofit: npo, user: user, status: :queued, export_type: 'ExportSupporters', parameters: params.to_json) + + DelayedJobHelper.enqueue_job(ExportSupporters, :run_export, [npo_id, params.to_json, user_id, e.id]) + end + + def self.run_export(npo_id, params, user_id, export_id) + # need to check that + ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id, export_id: export_id }, + npo_id: { required: true, is_integer: true }, + params: { required: true, is_json: true }, + user_id: { required: true, is_integer: true }, + export_id: { required: true, is_integer: true }) + + params = JSON.parse(params, :object_class=> HashWithIndifferentAccess) + # verify that it's also a hash since we can't do that at once + ParamValidation.new({ params: params }, + params: { is_hash: true }) + begin + export = Export.find(export_id) + rescue ActiveRecord::RecordNotFound + raise ParamValidation::ValidationError.new("Export #{export_id} doesn't exist!", key: :export_id) + end + export.status = :started + export.save! + + unless Nonprofit.exists?(npo_id) + raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id) + end + user = User.where('id = ?', user_id).first + unless user + raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id) + end + + file_date = Time.now.getutc().strftime('%m-%d-%Y--%H-%M-%S') + filename = "tmp/csv-exports/supporters-#{file_date}.csv" + + url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.for_export_enumerable(npo_id, params, 30000).map{|i| i.to_csv}, content_type: 'text/csv', content_disposition: 'attachment') + export.url = url + export.status = :completed + export.ended = Time.now + export.save! + + EmailJobQueue.queue(JobTypes::ExportSupportersCompletedJob, export) + rescue => e + if export + export.status = :failed + export.exception = e.to_s + export.ended = Time.now + export.save! + if user + EmailJobQueue.queue(JobTypes::ExportSupportersFailedJob, export) + end + raise e + end + raise e + end +end \ No newline at end of file diff --git a/lib/job_types/export_supporters_completed_job.rb b/lib/job_types/export_supporters_completed_job.rb new file mode 100644 index 00000000..7c9b09da --- /dev/null +++ b/lib/job_types/export_supporters_completed_job.rb @@ -0,0 +1,13 @@ +module JobTypes + class ExportSupportersCompletedJob < EmailJob + attr_reader :export + + def initialize(export) + @export = export + end + + def perform + ExportMailer.export_supporters_completed_notification(@export).deliver + end + end +end \ No newline at end of file diff --git a/lib/job_types/export_supporters_failed_job.rb b/lib/job_types/export_supporters_failed_job.rb new file mode 100644 index 00000000..e7555f9e --- /dev/null +++ b/lib/job_types/export_supporters_failed_job.rb @@ -0,0 +1,13 @@ +module JobTypes + class ExportSupportersFailedJob < EmailJob + attr_reader :export + + def initialize(export) + @export = export + end + + def perform + ExportMailer.export_supporters_failed_notification(@export).deliver + end + end +end \ No newline at end of file diff --git a/lib/query/query_supporters.rb b/lib/query/query_supporters.rb index 5250f4f2..9b89bf6a 100644 --- a/lib/query/query_supporters.rb +++ b/lib/query/query_supporters.rb @@ -268,6 +268,74 @@ module QuerySupporters return expr end + def self.for_export_enumerable(npo_id, query, chunk_limit=35000) + ParamValidation.new({npo_id: npo_id, query:query}, {npo_id: {required: true, is_int: true}, + query: {required:true, is_hash: true}}) + + return QxQueryChunker.for_export_enumerable(chunk_limit) do |offset, limit, skip_header| + get_chunk_of_export(npo_id, query, offset, limit, skip_header) + end + + end + + def self.get_chunk_of_export(np_id, query, offset=nil, limit=nil, skip_header=false) + return QxQueryChunker.get_chunk_of_query(offset, limit, skip_header) do + expr = full_filter_expr(np_id, query) + selects = supporter_export_selections.concat([ + '(payments.sum / 100)::money::text AS total_contributed', + 'supporters.id AS id' + ]) + if query[:export_custom_fields] + # Add a select/csv-column for every custom field master for this nonprofit + # and add a left join for every custom field master + # eg if the npo has a custom field like Employer with id 99, then the query will be + # SELECT export_cfj_Employer.value AS Employer, ... + # FROM supporters + # LEFT JOIN custom_field_joins AS export_cfj_Employer ON export_cfj_Employer.supporter_id=supporters.id AND export_cfj_Employer.custom_field_master_id=99 + # ... + ids = query[:export_custom_fields].split(',').map(&:to_i) + if ids.any? + cfms = Qx.select("name", "id").from(:custom_field_masters).where(nonprofit_id: np_id).and_where("id IN ($ids)", ids: ids).ex + cfms.compact.map do |cfm| + table_alias = "cfjs_#{cfm['name'].gsub(/\$/, "")}" + table_alias_quot = "\"#{table_alias}\"" + field_join_subq = Qx.select("STRING_AGG(value, ',') as value", "supporter_id") + .from("custom_field_joins") + .join("custom_field_masters" , "custom_field_masters.id=custom_field_joins.custom_field_master_id") + .where("custom_field_masters.id=$id", id: cfm['id']) + .group_by(:supporter_id) + .as(table_alias) + expr.add_left_join(field_join_subq, "#{table_alias_quot}.supporter_id=supporters.id") + selects = selects.concat(["#{table_alias_quot}.value AS \"#{cfm['name']}\""]) + end + end + end + + + get_last_payment_query = Qx.select('supporter_id', "MAX(date) AS date") + .from(:payments) + .group_by("supporter_id") + .as("last_payment") + + expr.add_left_join(get_last_payment_query, 'last_payment.supporter_id = supporters.id') + selects = selects.concat(['last_payment.date as "Last Payment Received"']) + + + supporter_note_query = Qx.select("STRING_AGG(supporter_notes.created_at || ': ' || supporter_notes.content, '\r\n' ORDER BY supporter_notes.created_at DESC) as notes", "supporter_notes.supporter_id") + .from(:supporter_notes) + .group_by('supporter_notes.supporter_id') + .as("supporter_note_query") + + expr.add_left_join(supporter_note_query, 'supporter_note_query.supporter_id=supporters.id') + selects = selects.concat(["supporter_note_query.notes AS notes"]) + + + expr.select(selects) + end + end + + + # Give supp data for csv def self.for_export(np_id, query) expr = full_filter_expr(np_id, query) @@ -520,7 +588,7 @@ module QuerySupporters .execute(format: 'csv') end - # + def self.get_min_or_max_dates_for_range(time_range_params) begin if (time_range_params[:year]) diff --git a/lib/qx_query_chunker.rb b/lib/qx_query_chunker.rb new file mode 100644 index 00000000..cee3ca0b --- /dev/null +++ b/lib/qx_query_chunker.rb @@ -0,0 +1 @@ +QxQueryChunker = QexprQueryChunker \ No newline at end of file diff --git a/spec/lib/exports/export_payments_spec.rb b/spec/lib/export/export_payments_spec.rb similarity index 100% rename from spec/lib/exports/export_payments_spec.rb rename to spec/lib/export/export_payments_spec.rb diff --git a/spec/lib/exports/export_recurring_donations_spec.rb b/spec/lib/export/export_recurring_donations_spec.rb similarity index 100% rename from spec/lib/exports/export_recurring_donations_spec.rb rename to spec/lib/export/export_recurring_donations_spec.rb diff --git a/spec/lib/export/export_supporters_spec.rb b/spec/lib/export/export_supporters_spec.rb new file mode 100644 index 00000000..df55ac32 --- /dev/null +++ b/spec/lib/export/export_supporters_spec.rb @@ -0,0 +1,198 @@ +require 'rails_helper' +require 'support/test_chunked_uploader' + +describe ExportSupporters do + before(:each) do + stub_const('CHUNKED_UPLOADER',TestChunkedUploader) + @nonprofit = force_create(:nonprofit) + @email = 'example@example.com' + @user = force_create(:user, email: @email) + @supporters = 2.times { force_create(:supporter, nonprofit: @nonprofit)} + CHUNKED_UPLOADER.clear + end + let(:export_header) { "Last Name,First Name,Full Name,Organization,Email,Phone,Address,City,State,Postal Code,Country,Anonymous?,Supporter Id,Total Contributed,Id,Last Payment Received,Notes".split(',')} + + context '.initiate_export' do + context 'param verification' do + it 'performs initial verification' do + expect { ExportSupporters.initiate_export(nil, nil, nil) }.to(raise_error do |error| + expect(error).to be_a(ParamValidation::ValidationError) + expect(error.data.length).to eq(6) + expect_validation_errors(error.data, [{ key: 'npo_id', name: :required }, + { key: 'npo_id', name: :is_integer }, + { key: 'user_id', name: :required }, + { key: 'user_id', name: :is_integer }, + { key: 'params', name: :required }, + { key: 'params', name: :is_hash }]) + end) + end + + it 'nonprofit doesnt exist' do + fake_npo = 8_888_881 + expect { ExportSupporters.initiate_export(fake_npo, {}, 8_888_883) }.to(raise_error do |error| + expect(error).to be_a(ParamValidation::ValidationError) + expect(error.message).to eq "Nonprofit #{fake_npo} doesn't exist!" + end) + end + + it 'user doesnt exist' do + fake_user = 8_888_883 + expect { ExportSupporters.initiate_export(@nonprofit.id, {}, fake_user) }.to(raise_error do |error| + expect(error).to be_a(ParamValidation::ValidationError) + expect(error.message).to eq "User #{fake_user} doesn't exist!" + end) + end + end + + it 'creates an export object and schedules job' do + Timecop.freeze(2020, 4, 5) do + DelayedJobHelper = double('delayed') + params = { param1: 'pp', root_url: 'https://localhost:8080' }.with_indifferent_access + + expect(Export).to receive(:create).and_wrap_original {|m, *args| + e = m.call(*args) # get original create + expect(DelayedJobHelper).to receive(:enqueue_job).with(ExportSupporters, :run_export, [@nonprofit.id, params.to_json, @user.id, e.id]) #add the enqueue + e + } + + + ExportSupporters.initiate_export(@nonprofit.id, params, @user.id) + export = Export.first + expected_export = { id: export.id, + user_id: @user.id, + nonprofit_id: @nonprofit.id, + status: 'queued', + export_type: 'ExportSupporters', + parameters: params.to_json, + updated_at: Time.now, + created_at: Time.now, + url: nil, + ended: nil, + exception: nil }.with_indifferent_access + expect(export.attributes).to eq(expected_export) + end + end + end + context '.run_export' do + context 'param validation' do + it 'rejects basic invalid data' do + expect { ExportSupporters.run_export(nil, nil, nil, nil) }.to(raise_error do |error| + expect(error).to be_a(ParamValidation::ValidationError) + expect_validation_errors(error, [{ key: 'npo_id', name: :required }, + { key: 'npo_id', name: :is_integer }, + { key: 'user_id', name: :required }, + { key: 'user_id', name: :is_integer }, + { key: 'params', name: :required }, + { key: 'params', name: :is_json }, + { key: 'export_id', name: :required }, + { key: 'export_id', name: :is_integer }]) + end) + end + + it 'rejects json which isnt a hash' do + expect { ExportSupporters.run_export(1, [{ item: '' }, { item: '' }].to_json, 1, 1) }.to(raise_error do |error| + expect(error).to be_a(ParamValidation::ValidationError) + expect_validation_errors(error, [ + { key: :params, name: :is_hash } + ]) + end) + end + + it 'no export throw an exception' do + expect { ExportSupporters.run_export(0, { x: 1 }.to_json, 0, 11_111) }.to(raise_error do |error| + expect(error).to be_a ParamValidation::ValidationError + expect(error.data[:key]).to eq :export_id + expect(error.message).to start_with('Export') + end) + end + + it 'no nonprofit' do + Timecop.freeze(2020, 4, 5) do + @export = force_create(:export, user: @user) + Timecop.freeze(2020, 4, 6) do + expect { ExportSupporters.run_export(0, { x: 1 }.to_json, @user.id, @export.id) }.to(raise_error do |error| + expect(error).to be_a ParamValidation::ValidationError + expect(error.data[:key]).to eq :npo_id + expect(error.message).to start_with('Nonprofit') + + @export.reload + expect(@export.status).to eq 'failed' + expect(@export.exception).to eq error.to_s + expect(@export.ended).to eq Time.now + expect(@export.updated_at).to eq Time.now + + end) + end + end + end + + it 'no user' do + Timecop.freeze(2020, 4, 5) do + @export = force_create(:export, user: @user) + Timecop.freeze(2020, 4, 6) do + expect { ExportSupporters.run_export(@nonprofit.id, { x: 1 }.to_json, 0, @export.id) }.to(raise_error do |error| + expect(error).to be_a ParamValidation::ValidationError + expect(error.data[:key]).to eq :user_id + expect(error.message).to start_with('User') + + @export.reload + expect(@export.status).to eq 'failed' + expect(@export.exception).to eq error.to_s + expect(@export.ended).to eq Time.now + expect(@export.updated_at).to eq Time.now + end) + end + end + end + end + + it 'handles exception in upload properly' do + Timecop.freeze(2020, 4, 5) do + @export = force_create(:export, user: @user) + expect_email_queued.with(JobTypes::ExportSupportersFailedJob, @export) + CHUNKED_UPLOADER.raise_error + Timecop.freeze(2020, 4, 6) do + expect { ExportSupporters.run_export(@nonprofit.id, {}.to_json, @user.id, @export.id) }.to(raise_error do |error| + expect(error).to be_a StandardError + expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE + + @export.reload + expect(@export.status).to eq 'failed' + expect(@export.exception).to eq error.to_s + expect(@export.ended).to eq Time.now + expect(@export.updated_at).to eq Time.now + + + end) + end + end + end + + it 'uploads as expected' do + Timecop.freeze(2020, 4, 5) do + @export = create(:export, user: @user, created_at: Time.now, updated_at: Time.now) + expect_email_queued.with(JobTypes::ExportSupportersCompletedJob, @export) + Timecop.freeze(2020, 4, 6, 1, 2, 3) do + ExportSupporters.run_export(@nonprofit.id, {:root_url => "https://localhost:8080/"}.to_json, @user.id, @export.id) + + @export.reload + + expect(@export.url).to eq 'http://fake.url/tmp/csv-exports/supporters-04-06-2020--01-02-03.csv' + expect(@export.status).to eq 'completed' + expect(@export.exception).to be_nil + expect(@export.ended).to eq Time.now + expect(@export.updated_at).to eq Time.now + csv = CSV.parse(TestChunkedUploader.output) + expect(csv.length).to eq (3) + + expect(csv[0]).to eq export_header + + expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv' + expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment' + + end + end + end + end + +end \ No newline at end of file diff --git a/spec/lib/job_types/export_supporters_completed_job_spec.rb b/spec/lib/job_types/export_supporters_completed_job_spec.rb new file mode 100644 index 00000000..b8f79aeb --- /dev/null +++ b/spec/lib/job_types/export_supporters_completed_job_spec.rb @@ -0,0 +1,13 @@ +require 'rails_helper.rb' + +describe JobTypes::ExportSupportersCompletedJob do + describe '.perform' do + it 'calls the correct active mailer' do + input = 1 + expect(ExportMailer).to receive(:export_supporters_completed_notification).with(input).and_wrap_original{|m, *args| mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer} + + job = JobTypes::ExportSupportersCompletedJob.new(input) + job.perform + end + end +end \ No newline at end of file diff --git a/spec/lib/job_types/export_supporters_failed_job_spec.rb b/spec/lib/job_types/export_supporters_failed_job_spec.rb new file mode 100644 index 00000000..b1b57d11 --- /dev/null +++ b/spec/lib/job_types/export_supporters_failed_job_spec.rb @@ -0,0 +1,13 @@ +require 'rails_helper.rb' + +describe JobTypes::ExportSupportersFailedJob do + describe '.perform' do + it 'calls the correct active mailer' do + input = 1 + expect(ExportMailer).to receive(:export_supporters_failed_notification).with(input).and_wrap_original{|m, *args| mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer} + + job = JobTypes::ExportSupportersFailedJob.new(input) + job.perform + end + end +end \ No newline at end of file