From e86f9fefe8f2bb57ab13efd45f6668b3cfab03e5 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 4 May 2020 16:31:48 -0500 Subject: [PATCH] Add support for passing the aws-region for the carrierwave migration --- .../cw_to_activestorage_generator.rb | 13 +-- .../templates/initializers/carrierwave.rb.tt | 9 +- .../lib/tasks/houdini_upgrade_tasks.rake | 97 +++++++++++++++++-- .../templates/backup_uploader_columns.rb | 55 +++++++++++ .../delete_uploader_backup_tables.rb | 7 ++ 5 files changed, 162 insertions(+), 19 deletions(-) create mode 100644 gems/houdini_upgrade/lib/tasks/templates/backup_uploader_columns.rb create mode 100644 gems/houdini_upgrade/lib/tasks/templates/delete_uploader_backup_tables.rb diff --git a/gems/houdini_upgrade/lib/generators/cw_to_activestorage/cw_to_activestorage_generator.rb b/gems/houdini_upgrade/lib/generators/cw_to_activestorage/cw_to_activestorage_generator.rb index 19e13e4a..bf4f54f0 100644 --- a/gems/houdini_upgrade/lib/generators/cw_to_activestorage/cw_to_activestorage_generator.rb +++ b/gems/houdini_upgrade/lib/generators/cw_to_activestorage/cw_to_activestorage_generator.rb @@ -2,7 +2,13 @@ class CwToActivestorageGenerator < Rails::Generators::Base source_root File.expand_path('templates', __dir__) class_option :aws_bucket, type: :string, required: true + class_option :aws_region, type: :string, required: false class_option :aws_assethost, type: :string, required: false + + def add_carrierwave_gems + gem "carrierwave", "~> 1" + gem "carrierwave-aws" + end def copy_uploaders if (!File.exists?('app/uploaders')) @@ -34,15 +40,10 @@ class CwToActivestorageGenerator < Rails::Generators::Base end end - def add_carrierwave_gems - gem "carrierwave", "~> 1" - gem "carrierwave-aws" - end - def add_carrierwave_template @aws_bucket = options[:aws_bucket] + @aws_region = options[:aws_region] @aws_assethost = options[:aws_assethost] || "https://#{@aws_bucket}.s3.amazonaws.com" - template 'initializers/carrierwave.rb', 'config/initializers/carrierwave.rb' end end diff --git a/gems/houdini_upgrade/lib/generators/cw_to_activestorage/templates/initializers/carrierwave.rb.tt b/gems/houdini_upgrade/lib/generators/cw_to_activestorage/templates/initializers/carrierwave.rb.tt index 57118451..0bb3676d 100644 --- a/gems/houdini_upgrade/lib/generators/cw_to_activestorage/templates/initializers/carrierwave.rb.tt +++ b/gems/houdini_upgrade/lib/generators/cw_to_activestorage/templates/initializers/carrierwave.rb.tt @@ -7,11 +7,8 @@ CarrierWave.configure do |config| config.aws_bucket = "<%= @aws_bucket %>" config.aws_acl = :public_read config.asset_host = "<%= @aws_assethost %>" - # config.aws_authenticated_url_expiration = 60 * 60 * 24 * 365 - # config.aws_credentials = { - # access_key_id: ENV['CARRIERWAVE_AWS_ACCESS_KEY_ID'], - # secret_access_key: ENV['CARRIERWAVE_SECRET_ACCESS_KEY'], - # config: AWS.config(cache_dir: "#{Rails.root}/tmp/uploads", region: ENV['CARRIERWAVE_SECRET_ACCESS_KEY']) - # } + config.aws_credentials = { + region: "<%= @aws_region %>" + } end \ No newline at end of file diff --git a/gems/houdini_upgrade/lib/tasks/houdini_upgrade_tasks.rake b/gems/houdini_upgrade/lib/tasks/houdini_upgrade_tasks.rake index 3b52fd60..bb98a8a9 100644 --- a/gems/houdini_upgrade/lib/tasks/houdini_upgrade_tasks.rake +++ b/gems/houdini_upgrade/lib/tasks/houdini_upgrade_tasks.rake @@ -29,20 +29,103 @@ end namespace :houdini_upgrade do Rake::Task["install:migrations"].clear_comments desc "Run houdini upgrade to v2" - task :run, [:aws_bucket, :aws_assethost] => :environment do |t, args| + task :run, [:aws_bucket, :aws_region, :aws_assethost] do |t, args| Rake::Task["houdini_upgrade:install:migrations"].invoke Rake::Task["active_storage:install"].invoke Rake::Task["houdini_upgrade:cw_to_activestorage"].invoke(*args) - Rake::Task["houdini_upgrade:migration"].invoke - end - - task :migration do + sh 'bundle' + Rake::Task["db:migrate"].invoke + Rake::Task["houdini_upgrade:migrate_uploads"].invoke + Rake::Task["houdini_upgrade:create_backup_uploader_migration"].invoke sh 'rails db:migrate' + end - task :cw_to_activestorage, [:aws_bucket, :aws_assethost] do |t, args| - tail = ["--aws-bucket=#{args[:aws_bucket]}"] + task :cw_to_activestorage, [:aws_bucket, :aws_region, :aws_assethost] do |t, args| + tail = ["--aws-bucket=#{args[:aws_bucket]}", "--aws-region=#{args[:aws_region]}"] + tail.push("--aws-assethost=#{args[:aws_assethost]}") if args[:aws_assethost] sh "rails generate cw_to_activestorage #{tail.join(' ')}" end + + desc "Migrate your CarrierWave uploads to activestorage" + task :migrate_uploads, [:simulate, :write_out_to_files] => [:environment] do |t, args| + progress_bar = ProgressBar.new(0, "Upload migration progress") + results = [] + Rails.application.eager_load! + # find activerecord descendents + HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table| + klass = table.class_name.constantize + items_to_migrate = klass.where(table.fields.map{|i| i.migrated_name + " IS NOT NULL"}.join(" OR ")) + progress_bar.increment_total(items_to_migrate.count * table.fields.count) + items_to_migrate + .find_each do |record| + table.fields.each do |field| + + if record.send(field.migrated_name + "?") + results << process(upload_url: record.send(field.migrated_name).url.gsub(/\/#{field.migrated_name}\//, "/#{field.name}/"), attachment_name: field.name, record: record) + end + progress_bar.increment + end + end + end + + copied = results.select{|i| i[:success]}.map{|i| i[:value]} + errors = results.select{|i| !i[:success]}.map{|i| i[:value]} + if args.write_out_to_files + CSV.open('copied.csv', 'wb') do |csv| + csv << ['Name', 'Id', "UploaderName", "FileToOpen", "CodeToRun"] + copied.each {|row| csv << row} + end + + CSV.open('errored.csv', 'wb') do |csv| + csv << ['Name', 'Id', "UploaderName", "Error"] + errors.each {|row| csv << row} + end + end + byebug + puts "Copied: #{copied.count}" + puts "Errored: #{errors.count}" + end + + def process(**args) + begin + if args[:upload_url] + filename = File.basename(URI.parse(args[:upload_url]).to_s) + file_to_open = args[:upload_url].start_with?('/') ? "." + args[:upload_url] : args[:upload_url] + + if (!args[:simulate]) + attachment_relation = args[:record].send("#{args[:attachment_name].to_s}") + attachment_relation.attach(io: open(file_to_open), filename: filename) + end + return {success: true, value: [args[:record].id,args[:attachment_name],file_to_open]} + end + return nil + rescue => e + return {success: false, value: [ args[:record].id, args[:attachment_name], e]} + end + end + + task :create_backup_uploader_migration do + if (Dir.glob("db/migrate/*_backup_uploader_columns.rb").none?) + FileUtils.cp __dir + "/templates/backup_uploader_columns.rb", + "db/migrate/#{(DateTime.now + 1.second).strftime('%Y%m%d%H%M%S')}_backup_uploader_columns.rb" + end + end + + task :delete_uploader_backup_tables_migration do + if (Dir.glob("db/migrate/*_backup_uploader_columns.rb").none?) + FileUtils.cp __dir + "/templates/delete_uploader_backup_tables.rb", + "db/migrate/#{(DateTime.now + 1.second).strftime('%Y%m%d%H%M%S')}_delete_uploader_backup_tables.rb" + end + end + + task :cleanup_upgrade_files do + FileUtils.rm_r "app/uploaders" + FileUtils.rm "config/initializers/carrierwave.rb" + gemfile_lines = File.readlines("Gemfile").select{|i| !i.include?("gem 'carrierwave'") && !i.include?("gem 'carrierwave-aws'")} + File.write('Gemfile', gemfile_lines.join("\n")) + cleanup_model_files + end + end diff --git a/gems/houdini_upgrade/lib/tasks/templates/backup_uploader_columns.rb b/gems/houdini_upgrade/lib/tasks/templates/backup_uploader_columns.rb new file mode 100644 index 00000000..31e6ad59 --- /dev/null +++ b/gems/houdini_upgrade/lib/tasks/templates/backup_uploader_columns.rb @@ -0,0 +1,55 @@ +# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later +class BackupUploaderColumns < ActiveRecord::Migration[5.2] + def up + # copy all + HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table| + create_table table.backup_table do |table| + table.references table.name, index:true + table.fields.each do |f| + table.string f.migrated_name + end + end + end + # copy all + HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table| + + execute <<-SQL + INSERT INTO #{table.backup_table} (#{table.foreign_key}, #{table.fields.map(&:migrated_name).join(', ')}) + VALUES ( SELECT id, #{table.fields.map(&:migrated_name).join(', ')} FROM #{table.name}) + SQL; + end + + # delete columns + HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table| + + fields.each do |f| + drop_column table.name, f.migrated_name + end + end + end + + def down + ## readd columns + HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table| + fields.each do |f| + add_column table.name, f.migrated_name, :string + end + end + + # copy all + HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table| + execute <<-SQL + UPDATE #{table.name} SET (#{table.fields.map(&:migrated_name).join(', ')}) = ( + SELECT (#{table.fields.map(&:migrated_name).join(', ')}) FROM #{table.backup_table} + WHERE #{table.backup_table}.#{table.foreign_key} = #{table.name}.id + ) + + SQL; + end + # delete tables + + HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each{ |entity, _| drop_table table.backup_table} + + + end +end \ No newline at end of file diff --git a/gems/houdini_upgrade/lib/tasks/templates/delete_uploader_backup_tables.rb b/gems/houdini_upgrade/lib/tasks/templates/delete_uploader_backup_tables.rb new file mode 100644 index 00000000..3ad763ec --- /dev/null +++ b/gems/houdini_upgrade/lib/tasks/templates/delete_uploader_backup_tables.rb @@ -0,0 +1,7 @@ +# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later +class DeleteUploaderBackupTables < ActiveRecord::Migration[5.2] + def up + # delete tables + HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each{ |table| drop_table table.backup_table} + end +end \ No newline at end of file