diff --git a/lib/chunked_uploader.rb b/lib/chunked_uploader.rb new file mode 100644 index 00000000..ee657846 --- /dev/null +++ b/lib/chunked_uploader.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later +module ChunkedUploader + + # Copy a string using chunks instead of all as one string. This is useful reducing memory usage when you want to do a huge export + # + # This code copies each chunk to a tempfile and then opens the tempfile and passes the IO object to the block + # @param [Enumerable] chunk_enum an enumerable of strings. + # @block accepts an IO for passing to upload + def self.upload(chunk_enum, &block) + file_name = File.join(Dir.tmpdir, SecureRandom.uuid) + + File.open(file_name, 'w') do |file| + chunk_enum.each do |chunk| + file.write(chunk) + end + end + + File.open(file_name, 'r') do |file| + yield(file) + end + + File.delete(file_name) + end +end diff --git a/spec/lib/chunked_uploader_spec.rb b/spec/lib/chunked_uploader_spec.rb new file mode 100644 index 00000000..442c3547 --- /dev/null +++ b/spec/lib/chunked_uploader_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later +require 'rails_helper' + +describe "ChunkedUploader" do + let(:input) { ['something\nexit','anotherbreak\n']} + let(:result) {'something\nexitanotherbreak\n'} + + it 'sends a File to the block' do + ChunkedUploader.upload(input) do |io| + expect(io.class).to eq File + end + end + + it 'it sends the accurate result to the block' do + ChunkedUploader.upload(input) do |io| + expect(io.read).to eq result + end + end + + describe 'chunked uploads' do + let(:input) { (['something\nexit','anotherbreak\n'] * 200000).flatten} + let(:result) {'something\nexitanotherbreak\n' * 200000} + let(:key) { "root/middle/child"} + it 'file service uploads 5MB+ file' do + ChunkedUploader.upload(input) do |io| + ActiveStorage::Blob.service.upload(key, io) + end + expect( ActiveStorage::Blob.service.download(key)).to eq result + end + + describe 's3' do + xit 'uploads 5MB+ file' do + s3_test = ActiveStorage::Service.configure(:s3_test, Rails.configuration.active_storage.service_configurations) + ChunkedUploader.upload(input) do |io| + s3_test.upload(key, io) + end + expect( s3_test.download(key)).to eq result + end + end + end + +end