From 5b0110e1ef7e6b3809470b269dbc4e7d64f0db60 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 8 May 2020 16:19:35 -0500 Subject: [PATCH] Migrate all chunked uploading to the new ChunkedUploader.upload method --- config/initializers/chunked_uploader.rb | 6 ---- .../initializers/chunked_uploader_service.rb | 12 +++++++ lib/export/export_payments.rb | 5 ++- lib/export/export_recurring_donations.rb | 5 ++- lib/export/export_supporter_notes.rb | 5 ++- lib/export/export_supporters.rb | 5 ++- spec/lib/export/export_payments_spec.rb | 16 ++++----- .../export/export_recurring_donations_spec.rb | 16 ++++----- .../lib/export/export_supporter_notes_spec.rb | 16 ++++----- spec/lib/export/export_supporters_spec.rb | 15 ++++---- spec/support/test_chunked_uploader.rb | 34 ------------------- spec/support/test_upload_service.rb | 30 ++++++++++++++++ 12 files changed, 88 insertions(+), 77 deletions(-) delete mode 100644 config/initializers/chunked_uploader.rb create mode 100644 config/initializers/chunked_uploader_service.rb delete mode 100755 spec/support/test_chunked_uploader.rb create mode 100755 spec/support/test_upload_service.rb diff --git a/config/initializers/chunked_uploader.rb b/config/initializers/chunked_uploader.rb deleted file mode 100644 index 99c01870..00000000 --- a/config/initializers/chunked_uploader.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later -CHUNKED_UPLOADER_SERVICE = ENV['CHUNKED_UPLOADER_SERVICE'] ? ActiveStorage::Service.configure( - ENV['CHUNKED_UPLOADER_SERVICE'], - Rails.configuration.active_storage.service_configurations) : ActiveStorage::Blob.service diff --git a/config/initializers/chunked_uploader_service.rb b/config/initializers/chunked_uploader_service.rb new file mode 100644 index 00000000..88905949 --- /dev/null +++ b/config/initializers/chunked_uploader_service.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later + + +ActiveSupport.on_load(:active_storage_blob) do + CHUNKED_UPLOAD_SERVICE = ENV['CHUNKED_UPLOAD_SERVICE'] ? ActiveStorage::Service.configure( + ENV['CHUNKED_UPLOAD_SERVICE'], + Rails.configuration.active_storage.service_configurations) : ActiveStorage::Blob.service +end + + diff --git a/lib/export/export_payments.rb b/lib/export/export_payments.rb index 00f52b63..55b316a4 100644 --- a/lib/export/export_payments.rb +++ b/lib/export/export_payments.rb @@ -55,7 +55,10 @@ module ExportPayments file_date = Time.now.getutc.strftime('%m-%d-%Y--%H-%M-%S') filename = "tmp/csv-exports/payments-#{file_date}.csv" - url = CHUNKED_UPLOADER.upload(filename, QueryPayments.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv), content_type: 'text/csv', content_disposition: 'attachment') + ChunkedUploader.upload(QueryPayments.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv)) do |io| + CHUNKED_UPLOAD_SERVICE.upload(filename, io, content_type: 'text/csv', content_disposition: 'attachment') + end + url = CHUNKED_UPLOAD_SERVICE.url(filename) export.url = url export.status = :completed export.ended = Time.now diff --git a/lib/export/export_recurring_donations.rb b/lib/export/export_recurring_donations.rb index 68d1c5f5..9353d466 100644 --- a/lib/export/export_recurring_donations.rb +++ b/lib/export/export_recurring_donations.rb @@ -55,7 +55,10 @@ module ExportRecurringDonations file_date = Time.now.getutc.strftime('%m-%d-%Y--%H-%M-%S') filename = "tmp/csv-exports/recurring_donations-#{file_date}.csv" - url = CHUNKED_UPLOADER.upload(filename, QueryRecurringDonations.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv), content_type: 'text/csv', content_disposition: 'attachment') + ChunkedUploader.upload(QueryRecurringDonations.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv)) do |io| + CHUNKED_UPLOAD_SERVICE.upload(filename, io, content_type: 'text/csv', content_disposition: 'attachment') + end + url = CHUNKED_UPLOAD_SERVICE.url(filename) export.url = url export.status = :completed export.ended = Time.now diff --git a/lib/export/export_supporter_notes.rb b/lib/export/export_supporter_notes.rb index 4c18041d..a91d259a 100644 --- a/lib/export/export_supporter_notes.rb +++ b/lib/export/export_supporter_notes.rb @@ -54,7 +54,10 @@ module ExportSupporterNotes file_date = Time.now.getutc.strftime('%m-%d-%Y--%H-%M-%S') filename = "tmp/csv-exports/supporters-notes-#{file_date}.csv" - url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.supporter_note_export_enumerable(npo_id, params, 30_000).map(&:to_csv), content_type: 'text/csv', content_disposition: 'attachment') + ChunkedUploader.upload(QuerySupporters.supporter_note_export_enumerable(npo_id, params, 30_000).map(&:to_csv)) do |io| + CHUNKED_UPLOAD_SERVICE.upload(filename, io, content_type: 'text/csv', content_disposition: 'attachment') + end + url = CHUNKED_UPLOAD_SERVICE.url(filename) export.url = url export.status = :completed export.ended = Time.now diff --git a/lib/export/export_supporters.rb b/lib/export/export_supporters.rb index 549c4388..f452636a 100644 --- a/lib/export/export_supporters.rb +++ b/lib/export/export_supporters.rb @@ -52,7 +52,10 @@ module ExportSupporters file_date = Time.now.getutc.strftime('%m-%d-%Y--%H-%M-%S') filename = "tmp/csv-exports/supporters-#{file_date}.csv" - url = CHUNKED_UPLOADER.upload(filename, QuerySupporters.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv), content_type: 'text/csv', content_disposition: 'attachment') + ChunkedUploader.upload(QuerySupporters.for_export_enumerable(npo_id, params, 30_000).map(&:to_csv)) do |io| + CHUNKED_UPLOAD_SERVICE.upload(filename, io, content_type: 'text/csv', content_disposition: 'attachment') + end + url = CHUNKED_UPLOAD_SERVICE.url(filename) export.url = url export.status = :completed export.ended = Time.now diff --git a/spec/lib/export/export_payments_spec.rb b/spec/lib/export/export_payments_spec.rb index 08a9c8e2..d70ba237 100644 --- a/spec/lib/export/export_payments_spec.rb +++ b/spec/lib/export/export_payments_spec.rb @@ -2,13 +2,11 @@ # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later require 'rails_helper' -require 'support/test_chunked_uploader' +require 'support/test_upload_service' describe ExportPayments do before(:each) do - stub_const('CHUNKED_UPLOADER', TestChunkedUploader) - - CHUNKED_UPLOADER.clear + stub_const('CHUNKED_UPLOAD_SERVICE', TestUploadService.new) end let(:email) { 'example@example.com' } @@ -158,11 +156,11 @@ describe ExportPayments do it 'handles exception in upload properly' do Timecop.freeze(2020, 4, 5) do @export = force_create(:export, user: user) - CHUNKED_UPLOADER.raise_error + CHUNKED_UPLOAD_SERVICE.raise_error Timecop.freeze(2020, 4, 6) do expect { ExportPayments.run_export(nonprofit.id, {}.to_json, user.id, @export.id) }.to(raise_error do |error| expect(error).to be_a StandardError - expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE + expect(error.message).to eq TestUploadService::TEST_ERROR_MESSAGE @export.reload expect(@export.status).to eq 'failed' @@ -188,13 +186,13 @@ describe ExportPayments do expect(@export.exception).to be_nil expect(@export.ended).to eq Time.now expect(@export.updated_at).to eq Time.now - csv = CSV.parse(TestChunkedUploader.output) + csv = CSV.parse(CHUNKED_UPLOAD_SERVICE.output) expect(csv.length).to eq 3 expect(csv[0]).to eq MockHelpers.payment_export_headers - expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv' - expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment' + expect(CHUNKED_UPLOAD_SERVICE.options[:content_type]).to eq 'text/csv' + expect(CHUNKED_UPLOAD_SERVICE.options[:content_disposition]).to eq 'attachment' end end end diff --git a/spec/lib/export/export_recurring_donations_spec.rb b/spec/lib/export/export_recurring_donations_spec.rb index 73a43012..d86abd7d 100644 --- a/spec/lib/export/export_recurring_donations_spec.rb +++ b/spec/lib/export/export_recurring_donations_spec.rb @@ -2,11 +2,11 @@ # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later require 'rails_helper' -require 'support/test_chunked_uploader' +require 'support/test_upload_service' describe ExportRecurringDonations do before(:each) do - stub_const('CHUNKED_UPLOADER', TestChunkedUploader) + stub_const('CHUNKED_UPLOAD_SERVICE', TestUploadService.new) @email = 'example@example.com' @user = force_create(:user, email: @email) @nonprofit = force_create(:nm_justice) @@ -16,7 +16,7 @@ describe ExportRecurringDonations do force_create(:donation, nonprofit: @nonprofit, supporter: @supporters[1])] @recurring_donations = [force_create(:recurring_donation, donation: @donations[0], nonprofit: @nonprofit, edit_token: 'edit_token_1', active: true), force_create(:recurring_donation, donation: @donations[1], supporter: @supporters[1], nonprofit: @nonprofit, edit_token: 'edit_token_2', active: true)] - CHUNKED_UPLOADER.clear + CHUNKED_UPLOAD_SERVICE.clear end context '.initiate_export' do @@ -149,11 +149,11 @@ describe ExportRecurringDonations do it 'handles exception in upload properly' do Timecop.freeze(2020, 4, 5) do @export = force_create(:export, user: @user) - CHUNKED_UPLOADER.raise_error + CHUNKED_UPLOAD_SERVICE.raise_error Timecop.freeze(2020, 4, 6) do expect { ExportRecurringDonations.run_export(@nonprofit.id, {}.to_json, @user.id, @export.id) }.to(raise_error do |error| expect(error).to be_a StandardError - expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE + expect(error.message).to eq TestUploadService::TEST_ERROR_MESSAGE @export.reload expect(@export.status).to eq 'failed' @@ -180,13 +180,13 @@ describe ExportRecurringDonations do expect(@export.exception).to be_nil expect(@export.ended).to eq Time.now expect(@export.updated_at).to eq Time.now - csv = CSV.parse(TestChunkedUploader.output) + csv = CSV.parse(CHUNKED_UPLOAD_SERVICE.output) expect(csv.length).to eq 3 expect(csv[0]).to eq MockHelpers.recurring_donation_export_headers - expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv' - expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment' + expect(CHUNKED_UPLOAD_SERVICE.options[:content_type]).to eq 'text/csv' + expect(CHUNKED_UPLOAD_SERVICE.options[:content_disposition]).to eq 'attachment' end end end diff --git a/spec/lib/export/export_supporter_notes_spec.rb b/spec/lib/export/export_supporter_notes_spec.rb index 91aa0566..6688a335 100644 --- a/spec/lib/export/export_supporter_notes_spec.rb +++ b/spec/lib/export/export_supporter_notes_spec.rb @@ -2,15 +2,15 @@ # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later require 'rails_helper' -require 'support/test_chunked_uploader' +require 'support/test_upload_service' describe ExportSupporterNotes do before(:each) do - stub_const('CHUNKED_UPLOADER', TestChunkedUploader) + stub_const('CHUNKED_UPLOAD_SERVICE', TestUploadService.new) supporter_note_for_s1 supporter_note_1_for_s2 supporter_note_2_for_s2 - CHUNKED_UPLOADER.clear + CHUNKED_UPLOAD_SERVICE.clear end let(:nonprofit) { force_create(:nm_justice) } @@ -178,12 +178,12 @@ describe ExportSupporterNotes do it 'handles exception in upload properly' do Timecop.freeze(2020, 4, 5) do @export = force_create(:export, user: user) - CHUNKED_UPLOADER.raise_error + CHUNKED_UPLOAD_SERVICE.raise_error Timecop.freeze(2020, 4, 6) do expect { expect { ExportSupporterNotes.run_export(nonprofit.id, {}.to_json, user.id, @export.id) }.to(raise_error do |error| expect(error).to be_a StandardError - expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE + expect(error.message).to eq TestUploadService::TEST_ERROR_MESSAGE @export.reload expect(@export.status).to eq 'failed' @@ -211,13 +211,13 @@ describe ExportSupporterNotes do expect(@export.exception).to be_nil expect(@export.ended).to eq Time.now expect(@export.updated_at).to eq Time.now - csv = CSV.parse(TestChunkedUploader.output) + csv = CSV.parse(CHUNKED_UPLOAD_SERVICE.output) expect(csv.length).to eq 4 expect(csv[0]).to eq export_header - expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv' - expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment' + expect(CHUNKED_UPLOAD_SERVICE.options[:content_type]).to eq 'text/csv' + expect(CHUNKED_UPLOAD_SERVICE.options[:content_disposition]).to eq 'attachment' end end end diff --git a/spec/lib/export/export_supporters_spec.rb b/spec/lib/export/export_supporters_spec.rb index 7723b597..e083da54 100644 --- a/spec/lib/export/export_supporters_spec.rb +++ b/spec/lib/export/export_supporters_spec.rb @@ -2,16 +2,15 @@ # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later require 'rails_helper' -require 'support/test_chunked_uploader' +require 'support/test_upload_service' describe ExportSupporters do before(:each) do - stub_const('CHUNKED_UPLOADER', TestChunkedUploader) + stub_const('CHUNKED_UPLOAD_SERVICE', TestUploadService.new) @nonprofit = force_create(:nm_justice) @email = 'example@example.com' @user = force_create(:user, email: @email) @supporters = 2.times { force_create(:supporter, nonprofit: @nonprofit) } - CHUNKED_UPLOADER.clear end let(:export_header) { 'Last Name,First Name,Full Name,Organization,Email,Phone,Address,City,State,Postal Code,Country,Anonymous?,Supporter Id,Total Contributed,Id,Last Payment Received,Notes,Tags'.split(',') } @@ -145,12 +144,12 @@ describe ExportSupporters do it 'handles exception in upload properly' do Timecop.freeze(2020, 4, 5) do @export = force_create(:export, user: @user) - CHUNKED_UPLOADER.raise_error + CHUNKED_UPLOAD_SERVICE.raise_error Timecop.freeze(2020, 4, 6) do expect { expect { ExportSupporters.run_export(@nonprofit.id, {}.to_json, @user.id, @export.id) }.to(raise_error do |error| expect(error).to be_a StandardError - expect(error.message).to eq TestChunkedUploader::TEST_ERROR_MESSAGE + expect(error.message).to eq TestUploadService::TEST_ERROR_MESSAGE @export.reload expect(@export.status).to eq 'failed' @@ -178,13 +177,13 @@ describe ExportSupporters do expect(@export.exception).to be_nil expect(@export.ended).to eq Time.now expect(@export.updated_at).to eq Time.now - csv = CSV.parse(TestChunkedUploader.output) + csv = CSV.parse(CHUNKED_UPLOAD_SERVICE.output) expect(csv.length).to eq 3 expect(csv[0]).to eq export_header - expect(TestChunkedUploader.options[:content_type]).to eq 'text/csv' - expect(TestChunkedUploader.options[:content_disposition]).to eq 'attachment' + expect(CHUNKED_UPLOAD_SERVICE.options[:content_type]).to eq 'text/csv' + expect(CHUNKED_UPLOAD_SERVICE.options[:content_disposition]).to eq 'attachment' end end end diff --git a/spec/support/test_chunked_uploader.rb b/spec/support/test_chunked_uploader.rb deleted file mode 100755 index 227a5b79..00000000 --- a/spec/support/test_chunked_uploader.rb +++ /dev/null @@ -1,34 +0,0 @@ -# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later -class TestChunkedUploader - TEST_ERROR_MESSAGE = 'test exception thrown' - def self.clear - @@output = nil - @@raise_error = false - @@options = nil - end - - def self.output - @@output - end - - # use this to throw an exception instead of finishing - def self.raise_error - @@raise_error = true - end - - def self.options - @@options - end - - def self.upload(path, chunk_enum, options = {}) - @@options = options - io = StringIO.new('', 'w') - chunk_enum.each do |chunk| - io.write(chunk) - end - raise TEST_ERROR_MESSAGE if @@raise_error - - @@output = io.string - 'http://fake.url/' + path - end -end diff --git a/spec/support/test_upload_service.rb b/spec/support/test_upload_service.rb new file mode 100755 index 00000000..11fbd029 --- /dev/null +++ b/spec/support/test_upload_service.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later +require 'stringio' +class TestUploadService < ActiveStorage::Service + TEST_ERROR_MESSAGE = 'test exception thrown' + + attr_reader :output, :options + def clear + @output = nil + @raise_error = false + @options = nil + end + + # use this to throw an exception instead of finishing + def raise_error + @raise_error = true + end + + def upload(key, io, checksum:nil, **options) + @options = options + + @output = io.read + raise TEST_ERROR_MESSAGE if @raise_error + end + + def url(key, **options) + "http://fake.url/#{key}" + end +end