Merge branch 'master' into fix_license

This commit is contained in:
Eric Schultz 2019-03-21 14:53:24 -05:00
commit bdc6331e6a
15 changed files with 479 additions and 11 deletions

View file

@ -42,4 +42,14 @@ class ExportMailer < BaseMailer
mail(to: @export.user.email, subject: 'Your supporters export has failed')
end
def export_supporter_notes_completed_notification(export)
@export = export
mail(to: @export.user.email, subject: 'Your supporter notes export is available!')
end
def export_supporter_notes_failed_notification(export)
@export = export
mail(to: @export.user.email, subject: 'Your supporter notes export has failed.')
end
end

View file

@ -0,0 +1,9 @@
<p>Your export from <%= Format::Date.simple @export.created_at %> was completed successfully.</p>
<p>To view your exported CSV, visit: <a href='<%= @export.url %>'>your export.</a></p>
<p>Note: your generated CSV file will be automatically deleted from our servers after seven days. Don't worry; you can always visit your recurring donations panel and export a new CSV file.</p>
<p>If you have any questions about this export, please contact <a href="mailto:<%= Settings.mailer.email %>"><%= Settings.mailer.email %></a>.</p>
<%= render 'emails/sig' %>

View file

@ -0,0 +1,12 @@
<p>Your export from <%= Format::Date.simple @export.created_at %> did not succeed due to a technical error.</p>
<p>While CommitChange engineers have been notified, don't hesitate to contact
<a href="mailto:<%= Settings.mailer.email %>"><%= Settings.mailer.email %></a> with the following information:</p>
<ul>
<li>Export ID: <%= @export.id %></li>
<li>User ID: <%= @export.user_id %></li>
<li>Nonprofit ID: <%= @export.nonprofit_id %></li>
<li>Exception: <%= @export.exception %></li>
</ul>
<%= render 'emails/sig' %>

View file

@ -0,0 +1,74 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module ExportSupporterNotes
def self.initiate_export(npo_id, params, user_id)
ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id },
npo_id: { required: true, is_integer: true },
params: { required: true, is_hash: true },
user_id: { required: true, is_integer: true })
npo = Nonprofit.where('id = ?', npo_id).first
unless npo
raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id)
end
user = User.where('id = ?', user_id).first
unless user
raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id)
end
e = Export.create(nonprofit: npo, user: user, status: :queued, export_type: 'ExportSupporterNotes', parameters: params.to_json)
DelayedJobHelper.enqueue_job(ExportSupporterNotes, :run_export, [npo_id, params.to_json, user_id, e.id])
end
def self.run_export(npo_id, params, user_id, export_id)
# need to check that
ParamValidation.new({ npo_id: npo_id, params: params, user_id: user_id, export_id: export_id },
npo_id: { required: true, is_integer: true },
params: { required: true, is_json: true },
user_id: { required: true, is_integer: true },
export_id: { required: true, is_integer: true })
params = JSON.parse(params, :object_class=> HashWithIndifferentAccess)
# verify that it's also a hash since we can't do that at once
ParamValidation.new({ params: params },
params: { is_hash: true })
begin
export = Export.find(export_id)
rescue ActiveRecord::RecordNotFound
raise ParamValidation::ValidationError.new("Export #{export_id} doesn't exist!", key: :export_id)
end
export.status = :started
export.save!
unless Nonprofit.exists?(npo_id)
raise ParamValidation::ValidationError.new("Nonprofit #{npo_id} doesn't exist!", key: :npo_id)
end
user = User.where('id = ?', user_id).first
unless user
raise ParamValidation::ValidationError.new("User #{user_id} doesn't exist!", key: :user_id)
end
file_date = Time.now.getutc().strftime('%m-%d-%Y--%H-%M-%S')
filename = "tmp/csv-exports/supporters-notes-#{file_date}.csv"
url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.supporter_note_export_enumerable(npo_id, params, 30000).map{|i| i.to_csv}, content_type: 'text/csv', content_disposition: 'attachment')
export.url = url
export.status = :completed
export.ended = Time.now
export.save!
EmailJobQueue.queue(JobTypes::ExportSupporterNotesCompletedJob, export)
rescue => e
if export
export.status = :failed
export.exception = e.to_s
export.ended = Time.now
export.save!
if user
EmailJobQueue.queue(JobTypes::ExportSupporterNotesFailedJob, export)
end
raise e
end
raise e
end
end

View file

@ -0,0 +1,14 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module JobTypes
class ExportSupporterNotesCompletedJob < EmailJob
attr_reader :export
def initialize(export)
@export = export
end
def perform
ExportMailer.export_supporter_notes_completed_notification(@export).deliver
end
end
end

View file

@ -0,0 +1,14 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module JobTypes
class ExportSupporterNotesFailedJob < EmailJob
attr_reader :export
def initialize(export)
@export = export
end
def perform
ExportMailer.export_supporter_notes_failed_notification(@export).deliver
end
end
end

View file

@ -313,7 +313,7 @@ module QueryPayments
.select(*export_selects)
.left_outer_join('campaign_gifts', 'campaign_gifts.donation_id=donations.id')
.left_outer_join('campaign_gift_options', 'campaign_gifts.campaign_gift_option_id=campaign_gift_options.id')
.left_outer_join('campaigns campaigns_for_export', 'donations.campaign_id=campaigns_for_export.id')
.left_outer_join("(#{campaigns_with_creator_email}) AS campaigns_for_export", 'donations.campaign_id=campaigns_for_export.id')
.left_outer_join(tickets_subquery, 'tickets.payment_id=payments.id')
.left_outer_join('events events_for_export', 'events_for_export.id=tickets.event_id OR donations.event_id=events_for_export.id')
.left_outer_join('offsite_payments', 'offsite_payments.payment_id=payments.id')
@ -332,7 +332,7 @@ module QueryPayments
.select(*export_selects)
.left_outer_join('campaign_gifts', 'campaign_gifts.donation_id=donations.id')
.left_outer_join('campaign_gift_options', 'campaign_gifts.campaign_gift_option_id=campaign_gift_options.id')
.left_outer_join('campaigns campaigns_for_export', 'donations.campaign_id=campaigns_for_export.id')
.left_outer_join("(#{campaigns_with_creator_email}) AS campaigns_for_export", 'donations.campaign_id=campaigns_for_export.id')
.left_outer_join(tickets_subquery, 'tickets.payment_id=payments.id')
.left_outer_join('events events_for_export', 'events_for_export.id=tickets.event_id OR donations.event_id=events_for_export.id')
.left_outer_join('offsite_payments', 'offsite_payments.payment_id=payments.id')
@ -363,6 +363,8 @@ module QueryPayments
'donations.anonymous',
'donations.comment',
"coalesce(nullif(campaigns_for_export.name, ''), 'None') AS campaign",
"campaigns_for_export.id AS \"Campaign Id\"",
"coalesce(nullif(campaigns_for_export.creator_email, ''), '') AS campaign_creator_email",
"coalesce(nullif(campaign_gift_options.name, ''), 'None') AS campaign_gift_level",
'events_for_export.name AS event_name',
'payments.id AS payment_id',
@ -454,4 +456,8 @@ module QueryPayments
def self.campaign_and_child_query_as_raw_string
"SELECT c_temp.id from campaigns c_temp where c_temp.id=$id OR c_temp.parent_campaign_id=$id"
end
def self.campaigns_with_creator_email
Qexpr.new.select('campaigns.*, users.email AS creator_email').from(:campaigns).left_outer_join(:profiles, "profiles.id = campaigns.profile_id").left_outer_join(:users, 'users.id = profiles.user_id')
end
end

View file

@ -372,14 +372,37 @@ UNION DISTINCT
.as("supporter_note_query")
expr.add_left_join(supporter_note_query, 'supporter_note_query.supporter_id=supporters.id')
selects = selects.concat(["supporter_note_query.notes AS notes"])
selects = selects.concat(["supporter_note_query.notes AS notes"]).concat(["ARRAY_TO_STRING(tags.names, ',') as tags"])
expr.select(selects)
end
end
def self.supporter_note_export_enumerable(npo_id, query, chunk_limit=35000)
ParamValidation.new({npo_id: npo_id, query:query}, {npo_id: {required: true, is_int: true},
query: {required:true, is_hash: true}})
return QxQueryChunker.for_export_enumerable(chunk_limit) do |offset, limit, skip_header|
get_chunk_of_supporter_note_export(npo_id, query, offset, limit, skip_header)
end
end
def self.get_chunk_of_supporter_note_export(np_id, query, offset=nil, limit=nil, skip_header=false)
return QxQueryChunker.get_chunk_of_query(offset, limit, skip_header) do
expr = full_filter_expr(np_id, query)
supporter_note_select = [
'supporters.id',
'supporters.email',
'supporter_notes.created_at as "Note Created At"',
'supporter_notes.content "Note Contents"'
]
expr.add_join(:supporter_notes, 'supporter_notes.supporter_id = supporters.id')
expr.select(supporter_note_select)
end
end
# Give supp data for csv
def self.for_export(np_id, query)

View file

@ -57,11 +57,15 @@ describe ExportPayments do
stub_const("DelayedJobHelper", double('delayed'))
params = { param1: 'pp' }.with_indifferent_access
expect(DelayedJobHelper).to receive(:enqueue_job).with(ExportPayments, :run_export, [nonprofit.id, params.to_json, user.id, 1])
expect(Export).to receive(:create).and_wrap_original {|m, *args|
e = m.call(*args) # get original create
expect(DelayedJobHelper).to receive(:enqueue_job).with(ExportPayments, :run_export, [nonprofit.id, params.to_json, user.id, e.id]) #add the enqueue
e
}
ExportPayments.initiate_export(nonprofit.id, params, user.id)
export = Export.first
expected_export = { id: 1,
expected_export = { id: export.id,
user_id: user.id,
nonprofit_id: nonprofit.id,
status: 'queued',

View file

@ -0,0 +1,232 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'rails_helper'
require 'support/test_chunked_uploader'
describe ExportSupporterNotes do
before(:each) do
stub_const('CHUNKED_UPLOADER',TestChunkedUploader)
supporter_note_for_s1
supporter_note_1_for_s2
supporter_note_2_for_s2
CHUNKED_UPLOADER.clear
end
let(:nonprofit) { force_create(:nonprofit)}
let(:supporter1) { force_create(:supporter, nonprofit: nonprofit)}
let(:supporter2) { force_create(:supporter, nonprofit: nonprofit)}
let(:user) { force_create(:user, email: email) }
let(:email) {'example@example.com'}
let(:export_header) { ['Id', 'Email', 'Note Created At', 'Note Contents']}
let(:note_content_1) do
"CONTENT1"
end
let(:note_content_2) do
"CONTENT2"
end
let(:note_content_3) do
"CONTENT3"
end
let(:supporter_note_for_s1) do
force_create(:supporter_note, supporter: supporter1, created_at: DateTime.new(2018,1,5), content: note_content_1)
end
let(:supporter_note_1_for_s2) do
force_create(:supporter_note, supporter: supporter2, created_at: DateTime.new(2018,2,5), content: note_content_2)
end
let(:supporter_note_2_for_s2) do
force_create(:supporter_note, supporter: supporter2, created_at: DateTime.new(2020,4, 5), content: note_content_3)
end
context '.initiate_export' do
context 'param verification' do
it 'performs initial verification' do
expect { ExportSupporterNotes.initiate_export(nil, nil, nil) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect(error.data.length).to eq(6)
expect_validation_errors(error.data, [{ key: 'npo_id', name: :required },
{ key: 'npo_id', name: :is_integer },
{ key: 'user_id', name: :required },
{ key: 'user_id', name: :is_integer },
{ key: 'params', name: :required },
{ key: 'params', name: :is_hash }])
end)
end
it 'nonprofit doesnt exist' do
fake_npo = 8_888_881
expect { ExportSupporterNotes.initiate_export(fake_npo, {}, 8_888_883) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect(error.message).to eq "Nonprofit #{fake_npo} doesn't exist!"
end)
end
it 'user doesnt exist' do
fake_user = 8_888_883
expect { ExportSupporterNotes.initiate_export(nonprofit.id, {}, fake_user) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect(error.message).to eq "User #{fake_user} doesn't exist!"
end)
end
end
it 'creates an export object and schedules job' do
Timecop.freeze(2020, 4, 5) do
DelayedJobHelper = double('delayed')
params = { param1: 'pp', root_url: 'https://localhost:8080' }.with_indifferent_access
expect(Export).to receive(:create).and_wrap_original {|m, *args|
e = m.call(*args) # get original create
expect(DelayedJobHelper).to receive(:enqueue_job).with(ExportSupporterNotes, :run_export, [nonprofit.id, params.to_json, user.id, e.id]) #add the enqueue
e
}
ExportSupporterNotes.initiate_export(nonprofit.id, params, user.id)
export = Export.first
expected_export = { id: export.id,
user_id: user.id,
nonprofit_id: nonprofit.id,
status: 'queued',
export_type: 'ExportSupporterNotes',
parameters: params.to_json,
updated_at: Time.now,
created_at: Time.now,
url: nil,
ended: nil,
exception: nil }.with_indifferent_access
expect(export.attributes).to eq(expected_export)
end
end
end
context '.run_export' do
context 'param validation' do
it 'rejects basic invalid data' do
expect { ExportSupporterNotes.run_export(nil, nil, nil, nil) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect_validation_errors(error, [{ key: 'npo_id', name: :required },
{ key: 'npo_id', name: :is_integer },
{ key: 'user_id', name: :required },
{ key: 'user_id', name: :is_integer },
{ key: 'params', name: :required },
{ key: 'params', name: :is_json },
{ key: 'export_id', name: :required },
{ key: 'export_id', name: :is_integer }])
end)
end
it 'rejects json which isnt a hash' do
expect { ExportSupporterNotes.run_export(1, [{ item: '' }, { item: '' }].to_json, 1, 1) }.to(raise_error do |error|
expect(error).to be_a(ParamValidation::ValidationError)
expect_validation_errors(error, [
{ key: :params, name: :is_hash }
])
end)
end
it 'no export throw an exception' do
expect { ExportSupporterNotes.run_export(0, { x: 1 }.to_json, 0, 11_111) }.to(raise_error do |error|
expect(error).to be_a ParamValidation::ValidationError
expect(error.data[:key]).to eq :export_id
expect(error.message).to start_with('Export')
end)
end
it 'no nonprofit' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: user)
Timecop.freeze(2020, 4, 6) do
expect { ExportSupporterNotes.run_export(0, { x: 1 }.to_json, user.id, @export.id) }.to(raise_error do |error|
expect(error).to be_a ParamValidation::ValidationError
expect(error.data[:key]).to eq :npo_id
expect(error.message).to start_with('Nonprofit')
@export.reload
expect(@export.status).to eq 'failed'
expect(@export.exception).to eq error.to_s
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
end)
end
end
end
it 'no user' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: user)
Timecop.freeze(2020, 4, 6) do
expect { ExportSupporterNotes.run_export(nonprofit.id, { x: 1 }.to_json, 0, @export.id) }.to(raise_error do |error|
expect(error).to be_a ParamValidation::ValidationError
expect(error.data[:key]).to eq :user_id
expect(error.message).to start_with('User')
@export.reload
expect(@export.status).to eq 'failed'
expect(@export.exception).to eq error.to_s
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
end)
end
end
end
end
it 'handles exception in upload properly' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: user)
expect_email_queued.with(JobTypes::ExportSupporterNotesFailedJob, @export)
CHUNKED_UPLOADER.raise_error
Timecop.freeze(2020, 4, 6) do
expect { ExportSupporterNotes.run_export(nonprofit.id, {}.to_json, user.id, @export.id) }.to(raise_error do |error|
expect(error).to be_a StandardError
expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE
@export.reload
expect(@export.status).to eq 'failed'
expect(@export.exception).to eq error.to_s
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
end)
end
end
end
it 'uploads as expected' do
Timecop.freeze(2020, 4, 5) do
@export = create(:export, user: user, created_at: Time.now, updated_at: Time.now)
expect_email_queued.with(JobTypes::ExportSupporterNotesCompletedJob, @export)
Timecop.freeze(2020, 4, 6, 1, 2, 3) do
ExportSupporterNotes.run_export(nonprofit.id, {:root_url => "https://localhost:8080/"}.to_json, user.id, @export.id)
@export.reload
expect(@export.url).to eq 'http://fake.url/tmp/csv-exports/supporters-notes-04-06-2020--01-02-03.csv'
expect(@export.status).to eq 'completed'
expect(@export.exception).to be_nil
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
csv = CSV.parse(TestChunkedUploader.output)
expect(csv.length).to eq (4)
expect(csv[0]).to eq export_header
expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv'
expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment'
end
end
end
end
end

View file

@ -11,7 +11,7 @@ describe ExportSupporters do
@supporters = 2.times { force_create(:supporter, nonprofit: @nonprofit)}
CHUNKED_UPLOADER.clear
end
let(:export_header) { "Last Name,First Name,Full Name,Organization,Email,Phone,Address,City,State,Postal Code,Country,Anonymous?,Supporter Id,Total Contributed,Id,Last Payment Received,Notes".split(',')}
let(:export_header) { "Last Name,First Name,Full Name,Organization,Email,Phone,Address,City,State,Postal Code,Country,Anonymous?,Supporter Id,Total Contributed,Id,Last Payment Received,Notes,Tags".split(',')}
context '.initiate_export' do
context 'param verification' do

View file

@ -0,0 +1,14 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'rails_helper.rb'
describe JobTypes::ExportSupporterNotesCompletedJob do
describe '.perform' do
it 'calls the correct active mailer' do
input = 1
expect(ExportMailer).to receive(:export_supporter_notes_completed_notification).with(input).and_wrap_original{|m, *args| mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer}
job = JobTypes::ExportSupporterNotesCompletedJob.new(1)
job.perform
end
end
end

View file

@ -0,0 +1,14 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'rails_helper.rb'
describe JobTypes::ExportSupporterNotesFailedJob do
describe '.perform' do
it 'calls the correct active mailer' do
input = 1
expect(ExportMailer).to receive(:export_supporter_notes_failed_notification).with(input).and_wrap_original{|m, *args| mailer = double('object'); expect(mailer).to receive(:deliver).and_return(nil); mailer}
job = JobTypes::ExportSupporterNotesFailedJob.new(1)
job.perform
end
end
end

View file

@ -22,6 +22,30 @@ describe QuerySupporters do
let(:campaign_gift2) { force_create(:campaign_gift, campaign_gift_option: campaign_gift_option, donation: donation2)}
let(:recurring) {force_create(:recurring_donation, donation: donation2, amount: GIFT_LEVEL_CHANGED_RECURRING)}
let(:note_content_1) do
"CONTENT1"
end
let(:note_content_2) do
"CONTENT2"
end
let(:note_content_3) do
"CONTENT3"
end
let(:supporter_note_for_s1) do
force_create(:supporter_note, supporter: supporter1, created_at: DateTime.new(2018,1,5), content: note_content_1)
end
let(:supporter_note_1_for_s2) do
force_create(:supporter_note, supporter: supporter2, created_at: DateTime.new(2018,2,5), content: note_content_2)
end
let(:supporter_note_2_for_s2) do
force_create(:supporter_note, supporter: supporter2, created_at: DateTime.new(2020,4, 5), content: note_content_3)
end
let(:init_all) {
np
@ -40,11 +64,8 @@ describe QuerySupporters do
QuerySupporters.campaign_list(np.id, campaign.id, {page: 0})
}
before(:each) {
init_all
}
it 'counts gift donations properly' do
init_all
glm = campaign_list
data = glm[:data]
@ -52,4 +73,25 @@ describe QuerySupporters do
expect(data.map{|i| i['total_raised']}).to match_array([GIFT_LEVEL_ONE_TIME, GIFT_LEVEL_RECURRING])
end
describe '.supporter_note_export_enumerable' do
let(:lazy_enumerable) do
supporter_note_for_s1
supporter_note_1_for_s2
supporter_note_2_for_s2
QuerySupporters.supporter_note_export_enumerable(np.id, {})
end
it 'is a lazy enumerable' do
expect(lazy_enumerable).to be_a Enumerator::Lazy
end
it 'is three items long' do
expect(lazy_enumerable.to_a.count).to eq 4
end
it 'has correct headers' do
expect(lazy_enumerable.to_a.first).to eq ['Id', 'Email', 'Note Created At', 'Note Contents']
end
end
end

View file

@ -7,7 +7,7 @@ module MockHelpers
"Dedicated To: Email",
"Dedicated To: Phone",
"Dedicated To: Address",
"Dedicated To: Note", 'Anonymous','Comment','Campaign', 'Campaign Gift Level', 'Event Name', 'Payment', 'Check Number', 'Donation Note']
"Dedicated To: Note", 'Anonymous','Comment','Campaign', 'Campaign Id', 'Campaign Creator Email', 'Campaign Gift Level', 'Event Name', 'Payment', 'Check Number', 'Donation Note']
end
def self.recurring_donation_export_headers()