Support for generated suppoters.csv in the background

app/controllers/nonprofits/supporters_controller.rb

@@ -24,6 +24,22 @@ class SupportersController < ApplicationController
 		end
   end
 
+  def export
+    begin
+      @nonprofit = current_nonprofit
+      @user = current_user_id
+      ExportSupporters::initiate_export(@nonprofit.id, params, @user)
+    rescue => e
+      e
+    end
+    if e.nil?
+      flash[:notice] = "Your export was successfully initiated and you'll be emailed at #{current_user.email} as soon as it's available. Feel free to use the site in the meantime."
+      render json: {}, status: :ok
+    else
+      render json: e, status: :ok
+    end
+  end
+
   def index_metrics
     render_json do
       QuerySupporters.full_search_metrics(params[:nonprofit_id], params)

app/mailers/export_mailer.rb

@@ -30,4 +30,16 @@ class ExportMailer < BaseMailer
 
     mail(to: @export.user.email, subject: 'Your recurring donations export has failed')
   end
+
+  def export_supporters_completed_notification(export)
+    @export = export
+
+    mail(to: @export.user.email, subject: 'Your supporters export is available!')
+  end
+
+  def export_supporters_failed_notification(export)
+    @export = export
+
+    mail(to: @export.user.email, subject: 'Your supporters export has failed')
+  end
 end

app/views/export_mailer/export_supporters_completed_notification.html.erb

@@ -0,0 +1,9 @@
+<p>Your export from <%= Format::Date.simple @export.created_at %> was completed successfully.</p>
+
+<p>To view your exported CSV, visit: <a href='<%= @export.url %>'>your export.</a></p>
+
+<p>Note: your generated CSV file will be automatically deleted from our servers after seven days. Don't worry; you can always visit your recurring donations panel and export a new CSV file.</p>
+
+<p>If you have any questions about this export, please contact <a href="mailto:support@commitchange.com">support@commitchange.com</a>.</p>
+
+<%= render 'emails/sig' %>

app/views/export_mailer/export_supporters_failed_notification.html.erb

@@ -0,0 +1,12 @@
+<p>Your export from <%= Format::Date.simple @export.created_at %> did not succeed due to a technical error.</p>
+<p>While CommitChange engineers have been notified, don't hesitate to contact
+  <a href="mailto:support@commitchange.com">support@commitchange.com</a> with the following information:</p>
+
+<ul>
+  <li>Export ID: <%= @export.id %></li>
+  <li>User ID: <%= @export.user_id %></li>
+  <li>Nonprofit ID: <%= @export.nonprofit_id %></li>
+  <li>Exception: <%= @export.exception %></li>
+</ul>
+
+<%= render 'emails/sig' %>

app/views/nonprofits/supporters/_export_supporters_s3_modal.html.erb

@@ -0,0 +1,46 @@
+<div class='modal skinny' id='exportSupportersS3Modal'>
+  <%= render 'common/modal_header', title: 'Export Supporters' %>
+  <div class='modal-body'>
+    <div class='u-marginBottom--10'>
+      <!--= show_if supporters.filter_count -->
+      <p class="filteringBy">
+        Filtering by:
+        <!--= put (readable_filter_names supporters.query) -->
+      </p>
+      <hr>
+    </div>
+    <div class='u-overflow--hidden'>
+      <!--= if show_fields_in_export (remove_class 'u-hide') (add_class 'u-hide') -->
+      <p>
+        Select which custom fields you want to export:
+      </p>
+      <p>
+        <!--= repeat custom_fields.masters.data -->
+        <input type='checkbox' class='checkbox'>
+        <!--= set_attr 'value' this.id -->
+        <!--= set_attr 'id' (cat 'checkbox-export-custom-fields-' this.i -->
+        <!--= on 'change' set_export_custom_fields -->
+        <label>
+          <!--= set_attr 'for' (cat 'checkbox-export-custom-fields-' this.i) -->
+          <!--= put this.name -->
+        </label>
+      </p>
+
+    </div>
+
+    <p>
+      <a> Export custom fields </a>
+      <!--= on 'click' (toggle 'show_fields_in_export' true) -->
+    </p>
+
+    <form autosubmit method="POST" data-success-message="Your export was successfully initiated and you'll be emailed as soon as it's available. Feel free to use the site in the meantime." data-reload>
+      <!--= set_attr 'action' (cat '/nonprofits/<%= @nonprofit.id %>/supporters/export.json' (obj_to_url_params supporters.query)) -->
+      <div class='u-centered'>
+        <p class='error'></p>
+        <button id='csv-export-button' class='button--small' type='submit' data-loading='Initiating Download'>
+          <i class='fa fa-cloud-download'></i> Download as CSV
+        </button>
+      </div>
+    </form>
+  </div>
+</div>

app/views/nonprofits/supporters/index.html.erb

@@ -42,6 +42,7 @@
 <%= render 'manage_field_master_modal' %>
 
 <%= render 'export_supporters_modal' %>
+<%= render 'export_supporters_s3_modal' %>
 
 <%= render 'edit_custom_fields_modal' %>
 <%= render 'edit_bulk_custom_fields_modal' %>

config/routes.rb

@@ -89,6 +89,7 @@ Commitchange::Application.routes.draw do
 			resources(:custom_field_joins, {only: [:index, :destroy]})
 			resources(:supporter_notes, {only: [:create, :update, :destroy]})
       resources(:activities, {only: [:index]})
+			post(:export, {on: :collection})
 			put :bulk_delete, on: :collection
 			post :merge, on: :collection
 			get :merge_data, on: :collection

lib/export/export_supporters.rb

@@ -0,0 +1,73 @@
+module ExportSupporters
+  def self.initiate_export(npo_id, params, user_id)
+
+    ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id },
+                        npo_id: { required: true, is_integer: true },
+                        params: { required: true, is_hash: true },
+                        user_id: { required: true, is_integer: true })
+    npo = Nonprofit.where('id = ?', npo_id).first
+    unless npo
+      raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id)
+    end
+    user = User.where('id = ?', user_id).first
+    unless user
+      raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id)
+    end
+
+    e = Export.create(nonprofit: npo, user: user, status: :queued, export_type: 'ExportSupporters', parameters: params.to_json)
+
+    DelayedJobHelper.enqueue_job(ExportSupporters, :run_export, [npo_id, params.to_json, user_id, e.id])
+  end
+
+  def self.run_export(npo_id, params, user_id, export_id)
+    # need to check that
+    ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id, export_id: export_id },
+                        npo_id: { required: true, is_integer: true },
+                        params: { required: true, is_json: true },
+                        user_id: { required: true, is_integer: true },
+                        export_id: { required: true, is_integer: true })
+
+    params = JSON.parse(params, :object_class=> HashWithIndifferentAccess)
+    # verify that it's also a hash since we can't do that at once
+    ParamValidation.new({ params: params },
+                        params: { is_hash: true })
+    begin
+      export = Export.find(export_id)
+    rescue ActiveRecord::RecordNotFound
+      raise ParamValidation::ValidationError.new("Export #{export_id} doesn't exist!", key: :export_id)
+    end
+    export.status = :started
+    export.save!
+
+    unless Nonprofit.exists?(npo_id)
+      raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id)
+    end
+    user = User.where('id = ?', user_id).first
+    unless user
+      raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id)
+    end
+
+    file_date = Time.now.getutc().strftime('%m-%d-%Y--%H-%M-%S')
+    filename = "tmp/csv-exports/supporters-#{file_date}.csv"
+
+    url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.for_export_enumerable(npo_id, params, 30000).map{|i| i.to_csv}, content_type: 'text/csv', content_disposition: 'attachment')
+    export.url = url
+    export.status = :completed
+    export.ended = Time.now
+    export.save!
+
+     EmailJobQueue.queue(JobTypes::ExportSupportersCompletedJob, export)
+  rescue => e
+    if export
+      export.status = :failed
+      export.exception = e.to_s
+      export.ended = Time.now
+      export.save!
+      if user
+        EmailJobQueue.queue(JobTypes::ExportSupportersFailedJob, export)
+      end
+      raise e
+    end
+    raise e
+  end
+end

lib/job_types/export_supporters_completed_job.rb

@@ -0,0 +1,13 @@
+module JobTypes
+  class ExportSupportersCompletedJob < EmailJob
+    attr_reader :export
+
+    def initialize(export)
+      @export = export
+    end
+
+    def perform
+      ExportMailer.export_supporters_completed_notification(@export).deliver
+    end
+  end
+end

lib/job_types/export_supporters_failed_job.rb

@@ -0,0 +1,13 @@
+module JobTypes
+  class ExportSupportersFailedJob < EmailJob
+    attr_reader :export
+
+    def initialize(export)
+      @export = export
+    end
+
+    def perform
+      ExportMailer.export_supporters_failed_notification(@export).deliver
+    end
+  end
+end

lib/query/query_supporters.rb

@@ -268,6 +268,74 @@ module QuerySupporters
     return expr
   end
 
+  def self.for_export_enumerable(npo_id, query, chunk_limit=35000)
+    ParamValidation.new({npo_id: npo_id, query:query}, {npo_id: {required: true, is_int: true},
+                                                        query: {required:true, is_hash: true}})
+
+    return QxQueryChunker.for_export_enumerable(chunk_limit) do |offset, limit, skip_header|
+      get_chunk_of_export(npo_id, query, offset, limit, skip_header)
+    end
+
+  end
+
+  def self.get_chunk_of_export(np_id, query, offset=nil, limit=nil, skip_header=false)
+    return QxQueryChunker.get_chunk_of_query(offset, limit, skip_header)  do
+      expr = full_filter_expr(np_id, query)
+      selects = supporter_export_selections.concat([
+                                                       '(payments.sum / 100)::money::text AS total_contributed',
+                                                       'supporters.id AS id'
+                                                   ])
+      if query[:export_custom_fields]
+        # Add a select/csv-column for every custom field master for this nonprofit
+        # and add a left join for every custom field master
+        # eg if the npo has a custom field like Employer with id 99, then the query will be
+        #   SELECT export_cfj_Employer.value AS Employer, ...
+        #   FROM supporters
+        #   LEFT JOIN custom_field_joins AS export_cfj_Employer ON export_cfj_Employer.supporter_id=supporters.id AND export_cfj_Employer.custom_field_master_id=99
+        #   ...
+        ids = query[:export_custom_fields].split(',').map(&:to_i)
+        if ids.any?
+          cfms = Qx.select("name", "id").from(:custom_field_masters).where(nonprofit_id: np_id).and_where("id IN ($ids)", ids: ids).ex
+          cfms.compact.map do |cfm|
+            table_alias = "cfjs_#{cfm['name'].gsub(/\$/, "")}"
+            table_alias_quot = "\"#{table_alias}\""
+            field_join_subq = Qx.select("STRING_AGG(value, ',') as value", "supporter_id")
+                                  .from("custom_field_joins")
+                                  .join("custom_field_masters" , "custom_field_masters.id=custom_field_joins.custom_field_master_id")
+                                  .where("custom_field_masters.id=$id", id: cfm['id'])
+                                  .group_by(:supporter_id)
+                                  .as(table_alias)
+            expr.add_left_join(field_join_subq, "#{table_alias_quot}.supporter_id=supporters.id")
+            selects = selects.concat(["#{table_alias_quot}.value AS \"#{cfm['name']}\""])
+          end
+        end
+      end
+
+
+      get_last_payment_query = Qx.select('supporter_id', "MAX(date) AS date")
+                                   .from(:payments)
+                                   .group_by("supporter_id")
+                                   .as("last_payment")
+
+      expr.add_left_join(get_last_payment_query, 'last_payment.supporter_id = supporters.id')
+      selects = selects.concat(['last_payment.date as "Last Payment Received"'])
+
+
+      supporter_note_query = Qx.select("STRING_AGG(supporter_notes.created_at || ': ' || supporter_notes.content, '\r\n' ORDER BY supporter_notes.created_at DESC) as notes", "supporter_notes.supporter_id")
+                                 .from(:supporter_notes)
+                                 .group_by('supporter_notes.supporter_id')
+                                 .as("supporter_note_query")
+
+      expr.add_left_join(supporter_note_query, 'supporter_note_query.supporter_id=supporters.id')
+      selects = selects.concat(["supporter_note_query.notes AS notes"])
+
+
+      expr.select(selects)
+    end
+  end
+
+
+
   # Give supp data for csv
   def self.for_export(np_id, query)
     expr = full_filter_expr(np_id, query)
@@ -520,7 +588,7 @@ module QuerySupporters
       .execute(format: 'csv')
   end
 
-  #
+
   def self.get_min_or_max_dates_for_range(time_range_params)
     begin
       if (time_range_params[:year])

lib/qx_query_chunker.rb

@@ -0,0 +1 @@
+QxQueryChunker = QexprQueryChunker

spec/lib/export/export_supporters_spec.rb

@@ -0,0 +1,198 @@
+require 'rails_helper'
+require 'support/test_chunked_uploader'
+
+describe ExportSupporters do
+  before(:each) do
+    stub_const('CHUNKED_UPLOADER',TestChunkedUploader)
+    @nonprofit = force_create(:nonprofit)
+    @email = 'example@example.com'
+    @user = force_create(:user, email: @email)
+    @supporters = 2.times { force_create(:supporter, nonprofit: @nonprofit)}
+    CHUNKED_UPLOADER.clear
+  end
+  let(:export_header) { "Last Name,First Name,Full Name,Organization,Email,Phone,Address,City,State,Postal Code,Country,Anonymous?,Supporter Id,Total Contributed,Id,Last Payment Received,Notes".split(',')}
+
+  context '.initiate_export' do
+    context 'param verification' do
+      it 'performs initial verification' do
+        expect { ExportSupporters.initiate_export(nil, nil, nil) }.to(raise_error do |error|
+          expect(error).to be_a(ParamValidation::ValidationError)
+          expect(error.data.length).to eq(6)
+          expect_validation_errors(error.data, [{ key: 'npo_id', name: :required },
+                                                { key: 'npo_id', name: :is_integer },
+                                                { key: 'user_id', name: :required },
+                                                { key: 'user_id', name: :is_integer },
+                                                { key: 'params', name: :required },
+                                                { key: 'params', name: :is_hash }])
+        end)
+      end
+
+      it 'nonprofit doesnt exist' do
+        fake_npo = 8_888_881
+        expect { ExportSupporters.initiate_export(fake_npo, {}, 8_888_883) }.to(raise_error do |error|
+          expect(error).to be_a(ParamValidation::ValidationError)
+          expect(error.message).to eq "Nonprofit #{fake_npo} doesn't exist!"
+        end)
+      end
+
+      it 'user doesnt exist' do
+        fake_user = 8_888_883
+        expect { ExportSupporters.initiate_export(@nonprofit.id, {}, fake_user) }.to(raise_error do |error|
+          expect(error).to be_a(ParamValidation::ValidationError)
+          expect(error.message).to eq "User #{fake_user} doesn't exist!"
+        end)
+      end
+    end
+
+    it 'creates an export object and schedules job' do
+      Timecop.freeze(2020, 4, 5) do
+        DelayedJobHelper = double('delayed')
+        params =  { param1: 'pp', root_url: 'https://localhost:8080' }.with_indifferent_access
+
+        expect(Export).to receive(:create).and_wrap_original {|m, *args|
+          e = m.call(*args) # get original create
+          expect(DelayedJobHelper).to receive(:enqueue_job).with(ExportSupporters, :run_export, [@nonprofit.id, params.to_json, @user.id, e.id])  #add the enqueue
+          e
+        }
+
+
+        ExportSupporters.initiate_export(@nonprofit.id, params, @user.id)
+        export = Export.first
+        expected_export = { id: export.id,
+                            user_id: @user.id,
+                            nonprofit_id: @nonprofit.id,
+                            status: 'queued',
+                            export_type: 'ExportSupporters',
+                            parameters: params.to_json,
+                            updated_at: Time.now,
+                            created_at: Time.now,
+                            url: nil,
+                            ended: nil,
+                            exception: nil }.with_indifferent_access
+        expect(export.attributes).to eq(expected_export)
+      end
+    end
+  end
+  context '.run_export' do
+    context 'param validation' do
+      it 'rejects basic invalid data' do
+        expect { ExportSupporters.run_export(nil, nil, nil, nil) }.to(raise_error do |error|
+          expect(error).to be_a(ParamValidation::ValidationError)
+          expect_validation_errors(error, [{ key: 'npo_id', name: :required },
+                                           { key: 'npo_id', name: :is_integer },
+                                           { key: 'user_id', name: :required },
+                                           { key: 'user_id', name: :is_integer },
+                                           { key: 'params', name: :required },
+                                           { key: 'params', name: :is_json },
+                                           { key: 'export_id', name: :required },
+                                           { key: 'export_id', name: :is_integer }])
+        end)
+      end
+
+      it 'rejects json which isnt a hash' do
+        expect { ExportSupporters.run_export(1, [{ item: '' }, { item: '' }].to_json, 1, 1) }.to(raise_error do |error|
+          expect(error).to be_a(ParamValidation::ValidationError)
+          expect_validation_errors(error, [
+              { key: :params, name: :is_hash }
+          ])
+        end)
+      end
+
+      it 'no export throw an exception' do
+        expect { ExportSupporters.run_export(0, { x: 1 }.to_json, 0, 11_111) }.to(raise_error do |error|
+          expect(error).to be_a ParamValidation::ValidationError
+          expect(error.data[:key]).to eq :export_id
+          expect(error.message).to start_with('Export')
+        end)
+      end
+
+      it 'no nonprofit' do
+        Timecop.freeze(2020, 4, 5) do
+          @export = force_create(:export, user: @user)
+          Timecop.freeze(2020, 4, 6) do
+            expect { ExportSupporters.run_export(0, { x: 1 }.to_json, @user.id, @export.id) }.to(raise_error do |error|
+              expect(error).to be_a ParamValidation::ValidationError
+              expect(error.data[:key]).to eq :npo_id
+              expect(error.message).to start_with('Nonprofit')
+
+              @export.reload
+              expect(@export.status).to eq 'failed'
+              expect(@export.exception).to eq error.to_s
+              expect(@export.ended).to eq Time.now
+              expect(@export.updated_at).to eq Time.now
+
+            end)
+          end
+        end
+      end
+
+      it 'no user' do
+        Timecop.freeze(2020, 4, 5) do
+          @export = force_create(:export, user: @user)
+          Timecop.freeze(2020, 4, 6) do
+            expect { ExportSupporters.run_export(@nonprofit.id, { x: 1 }.to_json, 0, @export.id) }.to(raise_error do |error|
+              expect(error).to be_a ParamValidation::ValidationError
+              expect(error.data[:key]).to eq :user_id
+              expect(error.message).to start_with('User')
+
+              @export.reload
+              expect(@export.status).to eq 'failed'
+              expect(@export.exception).to eq error.to_s
+              expect(@export.ended).to eq Time.now
+              expect(@export.updated_at).to eq Time.now
+            end)
+          end
+        end
+      end
+    end
+
+    it 'handles exception in upload properly' do
+      Timecop.freeze(2020, 4, 5) do
+        @export = force_create(:export, user: @user)
+        expect_email_queued.with(JobTypes::ExportSupportersFailedJob, @export)
+        CHUNKED_UPLOADER.raise_error
+        Timecop.freeze(2020, 4, 6) do
+          expect { ExportSupporters.run_export(@nonprofit.id, {}.to_json, @user.id, @export.id) }.to(raise_error do |error|
+            expect(error).to be_a StandardError
+            expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE
+
+            @export.reload
+            expect(@export.status).to eq 'failed'
+            expect(@export.exception).to eq error.to_s
+            expect(@export.ended).to eq Time.now
+            expect(@export.updated_at).to eq Time.now
+
+
+          end)
+        end
+      end
+    end
+
+    it 'uploads as expected' do
+      Timecop.freeze(2020, 4, 5) do
+        @export = create(:export, user: @user, created_at: Time.now, updated_at: Time.now)
+        expect_email_queued.with(JobTypes::ExportSupportersCompletedJob, @export)
+        Timecop.freeze(2020, 4, 6, 1, 2, 3) do
+          ExportSupporters.run_export(@nonprofit.id, {:root_url => "https://localhost:8080/"}.to_json, @user.id, @export.id)
+
+          @export.reload
+
+          expect(@export.url).to eq 'http://fake.url/tmp/csv-exports/supporters-04-06-2020--01-02-03.csv'
+          expect(@export.status).to eq 'completed'
+          expect(@export.exception).to be_nil
+          expect(@export.ended).to eq Time.now
+          expect(@export.updated_at).to eq Time.now
+          csv = CSV.parse(TestChunkedUploader.output)
+          expect(csv.length).to eq (3)
+
+          expect(csv[0]).to eq export_header
+
+          expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv'
+          expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment'
+
+        end
+      end
+    end
+  end
+
+end

spec/lib/job_types/export_supporters_completed_job_spec.rb

@@ -0,0 +1,13 @@
+require 'rails_helper.rb'
+
+describe JobTypes::ExportSupportersCompletedJob do
+  describe '.perform' do
+    it 'calls the correct active mailer' do
+      input = 1
+      expect(ExportMailer).to receive(:export_supporters_completed_notification).with(input).and_wrap_original{|m, *args|  mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer}
+
+      job = JobTypes::ExportSupportersCompletedJob.new(input)
+      job.perform
+    end
+  end
+end

spec/lib/job_types/export_supporters_failed_job_spec.rb

@@ -0,0 +1,13 @@
+require 'rails_helper.rb'
+
+describe JobTypes::ExportSupportersFailedJob do
+  describe '.perform' do
+    it 'calls the correct active mailer' do
+      input = 1
+      expect(ExportMailer).to receive(:export_supporters_failed_notification).with(input).and_wrap_original{|m, *args|  mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer}
+
+      job = JobTypes::ExportSupportersFailedJob.new(input)
+      job.perform
+    end
+  end
+end