NPO-Accounting/houdini
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Migrate all chunked uploading to the new ChunkedUploader.upload method

Browse source
This commit is contained in:
Eric 2020-05-08 16:19:35 -05:00
parent 6784583cbe
commit 5b0110e1ef
12 changed files with 88 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
config/initializers/chunked_uploader.rb
View file

@ -1,6 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
CHUNKED_UPLOADER_SERVICE = ENV['CHUNKED_UPLOADER_SERVICE'] ? ActiveStorage::Service.configure(
ENV['CHUNKED_UPLOADER_SERVICE'],
Rails.configuration.active_storage.service_configurations) : ActiveStorage::Blob.service

12
config/initializers/chunked_uploader_service.rb Normal file
View file

@ -0,0 +1,12 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
ActiveSupport.on_load(:active_storage_blob) do
CHUNKED_UPLOAD_SERVICE = ENV['CHUNKED_UPLOAD_SERVICE'] ? ActiveStorage::Service.configure(
ENV['CHUNKED_UPLOAD_SERVICE'],
Rails.configuration.active_storage.service_configurations) : ActiveStorage::Blob.service
end

5
lib/export/export_payments.rb
View file

@ -55,7 +55,10 @@ module ExportPayments
file_date = Time.now.getutc.strftime('%m-%d-%Y--%H-%M-%S')
filename = "tmp/csv-exports/payments-#{file_date}.csv"
url = CHUNKED_UPLOADER.upload(filename, QueryPayments.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv), content_type: 'text/csv', content_disposition: 'attachment')
ChunkedUploader.upload(QueryPayments.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv)) do |io|
CHUNKED_UPLOAD_SERVICE.upload(filename, io, content_type: 'text/csv', content_disposition: 'attachment')
end
url = CHUNKED_UPLOAD_SERVICE.url(filename)
export.url = url
export.status = :completed
export.ended = Time.now

5
lib/export/export_recurring_donations.rb
View file

@ -55,7 +55,10 @@ module ExportRecurringDonations
file_date = Time.now.getutc.strftime('%m-%d-%Y--%H-%M-%S')
filename = "tmp/csv-exports/recurring_donations-#{file_date}.csv"
url = CHUNKED_UPLOADER.upload(filename, QueryRecurringDonations.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv), content_type: 'text/csv', content_disposition: 'attachment')
ChunkedUploader.upload(QueryRecurringDonations.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv)) do |io|
CHUNKED_UPLOAD_SERVICE.upload(filename, io, content_type: 'text/csv', content_disposition: 'attachment')
end
url = CHUNKED_UPLOAD_SERVICE.url(filename)
export.url = url
export.status = :completed
export.ended = Time.now

5
lib/export/export_supporter_notes.rb
View file

@ -54,7 +54,10 @@ module ExportSupporterNotes
file_date = Time.now.getutc.strftime('%m-%d-%Y--%H-%M-%S')
filename = "tmp/csv-exports/supporters-notes-#{file_date}.csv"
url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.supporter_note_export_enumerable(npo_id, params, 30_000).map(&:to_csv), content_type: 'text/csv', content_disposition: 'attachment')
ChunkedUploader.upload(QuerySupporters.supporter_note_export_enumerable(npo_id, params, 30_000).map(&:to_csv)) do |io|
CHUNKED_UPLOAD_SERVICE.upload(filename, io, content_type: 'text/csv', content_disposition: 'attachment')
end
url = CHUNKED_UPLOAD_SERVICE.url(filename)
export.url = url
export.status = :completed
export.ended = Time.now

5
lib/export/export_supporters.rb
View file

@ -52,7 +52,10 @@ module ExportSupporters
file_date = Time.now.getutc.strftime('%m-%d-%Y--%H-%M-%S')
filename = "tmp/csv-exports/supporters-#{file_date}.csv"
url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv), content_type: 'text/csv', content_disposition: 'attachment')
ChunkedUploader.upload(QuerySupporters.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv)) do |io|
CHUNKED_UPLOAD_SERVICE.upload(filename, io, content_type: 'text/csv', content_disposition: 'attachment')
end
url = CHUNKED_UPLOAD_SERVICE.url(filename)
export.url = url
export.status = :completed
export.ended = Time.now

16
spec/lib/export/export_payments_spec.rb
View file

@ -2,13 +2,11 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'rails_helper'
require 'support/test_chunked_uploader'
require 'support/test_upload_service'
describe ExportPayments do
before(:each) do
stub_const('CHUNKED_UPLOADER', TestChunkedUploader)
CHUNKED_UPLOADER.clear
stub_const('CHUNKED_UPLOAD_SERVICE', TestUploadService.new)
end
let(:email) { 'example@example.com' }
@ -158,11 +156,11 @@ describe ExportPayments do
it 'handles exception in upload properly' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: user)
CHUNKED_UPLOADER.raise_error
CHUNKED_UPLOAD_SERVICE.raise_error
Timecop.freeze(2020, 4, 6) do
expect { ExportPayments.run_export(nonprofit.id, {}.to_json, user.id, @export.id) }.to(raise_error do |error|
expect(error).to be_a StandardError
expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE
expect(error.message).to eq TestUploadService::TEST_ERROR_MESSAGE
@export.reload
expect(@export.status).to eq 'failed'
@ -188,13 +186,13 @@ describe ExportPayments do
expect(@export.exception).to be_nil
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
csv = CSV.parse(TestChunkedUploader.output)
csv = CSV.parse(CHUNKED_UPLOAD_SERVICE.output)
expect(csv.length).to eq 3
expect(csv[0]).to eq MockHelpers.payment_export_headers
expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv'
expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment'
expect(CHUNKED_UPLOAD_SERVICE.options[:content_type]).to eq 'text/csv'
expect(CHUNKED_UPLOAD_SERVICE.options[:content_disposition]).to eq 'attachment'
end
end
end

16
spec/lib/export/export_recurring_donations_spec.rb
View file

@ -2,11 +2,11 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'rails_helper'
require 'support/test_chunked_uploader'
require 'support/test_upload_service'
describe ExportRecurringDonations do
before(:each) do
stub_const('CHUNKED_UPLOADER', TestChunkedUploader)
stub_const('CHUNKED_UPLOAD_SERVICE', TestUploadService.new)
@email = 'example@example.com'
@user = force_create(:user, email: @email)
@nonprofit = force_create(:nm_justice)
@ -16,7 +16,7 @@ describe ExportRecurringDonations do
force_create(:donation, nonprofit: @nonprofit, supporter: @supporters[1])]
@recurring_donations = [force_create(:recurring_donation, donation: @donations[0], nonprofit: @nonprofit, edit_token: 'edit_token_1', active: true),
force_create(:recurring_donation, donation: @donations[1], supporter: @supporters[1], nonprofit: @nonprofit, edit_token: 'edit_token_2', active: true)]
CHUNKED_UPLOADER.clear
CHUNKED_UPLOAD_SERVICE.clear
end
context '.initiate_export' do
@ -149,11 +149,11 @@ describe ExportRecurringDonations do
it 'handles exception in upload properly' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: @user)
CHUNKED_UPLOADER.raise_error
CHUNKED_UPLOAD_SERVICE.raise_error
Timecop.freeze(2020, 4, 6) do
expect { ExportRecurringDonations.run_export(@nonprofit.id, {}.to_json, @user.id, @export.id) }.to(raise_error do |error|
expect(error).to be_a StandardError
expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE
expect(error.message).to eq TestUploadService::TEST_ERROR_MESSAGE
@export.reload
expect(@export.status).to eq 'failed'
@ -180,13 +180,13 @@ describe ExportRecurringDonations do
expect(@export.exception).to be_nil
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
csv = CSV.parse(TestChunkedUploader.output)
csv = CSV.parse(CHUNKED_UPLOAD_SERVICE.output)
expect(csv.length).to eq 3
expect(csv[0]).to eq MockHelpers.recurring_donation_export_headers
expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv'
expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment'
expect(CHUNKED_UPLOAD_SERVICE.options[:content_type]).to eq 'text/csv'
expect(CHUNKED_UPLOAD_SERVICE.options[:content_disposition]).to eq 'attachment'
end
end
end

16
spec/lib/export/export_supporter_notes_spec.rb
View file

@ -2,15 +2,15 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'rails_helper'
require 'support/test_chunked_uploader'
require 'support/test_upload_service'
describe ExportSupporterNotes do
before(:each) do
stub_const('CHUNKED_UPLOADER', TestChunkedUploader)
stub_const('CHUNKED_UPLOAD_SERVICE', TestUploadService.new)
supporter_note_for_s1
supporter_note_1_for_s2
supporter_note_2_for_s2
CHUNKED_UPLOADER.clear
CHUNKED_UPLOAD_SERVICE.clear
end
let(:nonprofit) { force_create(:nm_justice) }
@ -178,12 +178,12 @@ describe ExportSupporterNotes do
it 'handles exception in upload properly' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: user)
CHUNKED_UPLOADER.raise_error
CHUNKED_UPLOAD_SERVICE.raise_error
Timecop.freeze(2020, 4, 6) do
expect {
expect { ExportSupporterNotes.run_export(nonprofit.id, {}.to_json, user.id, @export.id) }.to(raise_error do |error|
expect(error).to be_a StandardError
expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE
expect(error.message).to eq TestUploadService::TEST_ERROR_MESSAGE
@export.reload
expect(@export.status).to eq 'failed'
@ -211,13 +211,13 @@ describe ExportSupporterNotes do
expect(@export.exception).to be_nil
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
csv = CSV.parse(TestChunkedUploader.output)
csv = CSV.parse(CHUNKED_UPLOAD_SERVICE.output)
expect(csv.length).to eq 4
expect(csv[0]).to eq export_header
expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv'
expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment'
expect(CHUNKED_UPLOAD_SERVICE.options[:content_type]).to eq 'text/csv'
expect(CHUNKED_UPLOAD_SERVICE.options[:content_disposition]).to eq 'attachment'
end
end
end

15
spec/lib/export/export_supporters_spec.rb
View file

@ -2,16 +2,15 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'rails_helper'
require 'support/test_chunked_uploader'
require 'support/test_upload_service'
describe ExportSupporters do
before(:each) do
stub_const('CHUNKED_UPLOADER', TestChunkedUploader)
stub_const('CHUNKED_UPLOAD_SERVICE', TestUploadService.new)
@nonprofit = force_create(:nm_justice)
@email = 'example@example.com'
@user = force_create(:user, email: @email)
@supporters = 2.times { force_create(:supporter, nonprofit: @nonprofit) }
CHUNKED_UPLOADER.clear
end
let(:export_header) { 'Last Name,First Name,Full Name,Organization,Email,Phone,Address,City,State,Postal Code,Country,Anonymous?,Supporter Id,Total Contributed,Id,Last Payment Received,Notes,Tags'.split(',') }
@ -145,12 +144,12 @@ describe ExportSupporters do
it 'handles exception in upload properly' do
Timecop.freeze(2020, 4, 5) do
@export = force_create(:export, user: @user)
CHUNKED_UPLOADER.raise_error
CHUNKED_UPLOAD_SERVICE.raise_error
Timecop.freeze(2020, 4, 6) do
expect {
expect { ExportSupporters.run_export(@nonprofit.id, {}.to_json, @user.id, @export.id) }.to(raise_error do |error|
expect(error).to be_a StandardError
expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE
expect(error.message).to eq TestUploadService::TEST_ERROR_MESSAGE
@export.reload
expect(@export.status).to eq 'failed'
@ -178,13 +177,13 @@ describe ExportSupporters do
expect(@export.exception).to be_nil
expect(@export.ended).to eq Time.now
expect(@export.updated_at).to eq Time.now
csv = CSV.parse(TestChunkedUploader.output)
csv = CSV.parse(CHUNKED_UPLOAD_SERVICE.output)
expect(csv.length).to eq 3
expect(csv[0]).to eq export_header
expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv'
expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment'
expect(CHUNKED_UPLOAD_SERVICE.options[:content_type]).to eq 'text/csv'
expect(CHUNKED_UPLOAD_SERVICE.options[:content_disposition]).to eq 'attachment'
end
end
end

34
spec/support/test_chunked_uploader.rb
View file

@ -1,34 +0,0 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
class TestChunkedUploader
TEST_ERROR_MESSAGE = 'test exception thrown'
def self.clear
@@output = nil
@@raise_error = false
@@options = nil
end
def self.output
@@output
end
# use this to throw an exception instead of finishing
def self.raise_error
@@raise_error = true
end
def self.options
@@options
end
def self.upload(path, chunk_enum, options = {})
@@options = options
io = StringIO.new('', 'w')
chunk_enum.each do |chunk|
io.write(chunk)
end
raise TEST_ERROR_MESSAGE if @@raise_error
@@output = io.string
'http://fake.url/' + path
end
end

30
spec/support/test_upload_service.rb Executable file
View file

@ -0,0 +1,30 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'stringio'
class TestUploadService < ActiveStorage::Service
TEST_ERROR_MESSAGE = 'test exception thrown'
attr_reader :output, :options
def clear
@output = nil
@raise_error = false
@options = nil
end
# use this to throw an exception instead of finishing
def raise_error
@raise_error = true
end
def upload(key, io, checksum:nil, **options)
@options = options
@output = io.read
raise TEST_ERROR_MESSAGE if @raise_error
end
def url(key, **options)
"http://fake.url/#{key}"
end
end