Add support for passing the aws-region for the carrierwave migration

gems/houdini_upgrade/lib/generators/cw_to_activestorage/cw_to_activestorage_generator.rb

@@ -2,7 +2,13 @@
 class CwToActivestorageGenerator < Rails::Generators::Base
   source_root File.expand_path('templates', __dir__)
   class_option :aws_bucket, type: :string, required: true
+  class_option :aws_region, type: :string, required: false
   class_option :aws_assethost, type: :string, required: false
+  
+  def add_carrierwave_gems
+    gem "carrierwave", "~> 1"
+    gem "carrierwave-aws"
+  end
 
   def copy_uploaders
     if (!File.exists?('app/uploaders'))
@@ -34,15 +40,10 @@ class CwToActivestorageGenerator < Rails::Generators::Base
     end
   end
 
-  def add_carrierwave_gems
-    gem "carrierwave", "~> 1"
-    gem "carrierwave-aws"
-  end
-
   def add_carrierwave_template
     @aws_bucket = options[:aws_bucket]
+    @aws_region = options[:aws_region]
     @aws_assethost = options[:aws_assethost] || "https://#{@aws_bucket}.s3.amazonaws.com"
-    
     template 'initializers/carrierwave.rb', 'config/initializers/carrierwave.rb'
   end
 end

gems/houdini_upgrade/lib/generators/cw_to_activestorage/templates/initializers/carrierwave.rb.tt

@@ -7,11 +7,8 @@ CarrierWave.configure do |config|
     config.aws_bucket = "<%= @aws_bucket %>"
     config.aws_acl    = :public_read
     config.asset_host = "<%= @aws_assethost %>"
-    # config.aws_authenticated_url_expiration = 60 * 60 * 24 * 365
+    config.aws_credentials = {
-    # config.aws_credentials = {
+        region: "<%= @aws_region %>"
-    #   access_key_id: ENV['CARRIERWAVE_AWS_ACCESS_KEY_ID'],
+    }
-    #   secret_access_key: ENV['CARRIERWAVE_SECRET_ACCESS_KEY'],
-    #   config: AWS.config(cache_dir: "#{Rails.root}/tmp/uploads", region: ENV['CARRIERWAVE_SECRET_ACCESS_KEY'])
-    # }
 end
   

gems/houdini_upgrade/lib/tasks/houdini_upgrade_tasks.rake

@@ -29,20 +29,103 @@ end
  namespace :houdini_upgrade do
   Rake::Task["install:migrations"].clear_comments
   desc "Run houdini upgrade to v2"
-  task :run, [:aws_bucket, :aws_assethost]  => :environment do |t, args|
+  task :run, [:aws_bucket, :aws_region, :aws_assethost] do |t, args|
     Rake::Task["houdini_upgrade:install:migrations"].invoke
     Rake::Task["active_storage:install"].invoke
     Rake::Task["houdini_upgrade:cw_to_activestorage"].invoke(*args)
-    Rake::Task["houdini_upgrade:migration"].invoke
+    sh 'bundle'
-  end
+    Rake::Task["db:migrate"].invoke
-
+    Rake::Task["houdini_upgrade:migrate_uploads"].invoke
-  task :migration do
+    Rake::Task["houdini_upgrade:create_backup_uploader_migration"].invoke
     sh 'rails db:migrate'
+
   end
 
-  task :cw_to_activestorage, [:aws_bucket, :aws_assethost] do |t, args|
+  task :cw_to_activestorage, [:aws_bucket, :aws_region, :aws_assethost] do |t, args|
-    tail = ["--aws-bucket=#{args[:aws_bucket]}"]
+    tail = ["--aws-bucket=#{args[:aws_bucket]}", "--aws-region=#{args[:aws_region]}"]
+    
     tail.push("--aws-assethost=#{args[:aws_assethost]}") if args[:aws_assethost]
     sh "rails generate cw_to_activestorage #{tail.join(' ')}"
   end
+
+  desc "Migrate your CarrierWave uploads to activestorage"
+  task :migrate_uploads, [:simulate, :write_out_to_files] => [:environment] do |t, args|
+    progress_bar = ProgressBar.new(0, "Upload migration progress")
+    results = []
+    Rails.application.eager_load!
+    # find activerecord descendents
+    HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table|
+      klass = table.class_name.constantize
+      items_to_migrate = klass.where(table.fields.map{|i| i.migrated_name + " IS NOT NULL"}.join(" OR "))
+      progress_bar.increment_total(items_to_migrate.count * table.fields.count)
+      items_to_migrate
+        .find_each do |record|
+          table.fields.each do |field|
+            
+            if record.send(field.migrated_name + "?")
+              results << process(upload_url: record.send(field.migrated_name).url.gsub(/\/#{field.migrated_name}\//, "/#{field.name}/"), attachment_name: field.name, record: record)
+            end
+            progress_bar.increment
+          end
+        end
+    end
+
+    copied = results.select{|i| i[:success]}.map{|i| i[:value]}
+    errors = results.select{|i| !i[:success]}.map{|i| i[:value]}
+    if args.write_out_to_files
+      CSV.open('copied.csv', 'wb') do |csv|
+          csv << ['Name', 'Id', "UploaderName", "FileToOpen", "CodeToRun"]
+          copied.each {|row| csv << row}
+      end
+
+      CSV.open('errored.csv', 'wb') do |csv|
+          csv << ['Name', 'Id', "UploaderName", "Error"]
+          errors.each {|row| csv << row}
+      end
+    end
+    byebug
+    puts "Copied: #{copied.count}"
+    puts "Errored: #{errors.count}"
+  end
+
+  def process(**args)
+    begin
+      if args[:upload_url]
+        filename = File.basename(URI.parse(args[:upload_url]).to_s)
+        file_to_open = args[:upload_url].start_with?('/') ? "." + args[:upload_url] : args[:upload_url]
+        
+        if (!args[:simulate])
+            attachment_relation = args[:record].send("#{args[:attachment_name].to_s}")
+            attachment_relation.attach(io: open(file_to_open), filename: filename)
+        end
+        return {success: true, value: [args[:record].id,args[:attachment_name],file_to_open]}
+      end
+      return nil
+    rescue => e
+      return {success: false, value: [ args[:record].id, args[:attachment_name], e]}
+    end
+  end
+
+  task :create_backup_uploader_migration do
+    if (Dir.glob("db/migrate/*_backup_uploader_columns.rb").none?)
+      FileUtils.cp __dir + "/templates/backup_uploader_columns.rb", 
+      "db/migrate/#{(DateTime.now + 1.second).strftime('%Y%m%d%H%M%S')}_backup_uploader_columns.rb"
+    end
+  end
+
+  task :delete_uploader_backup_tables_migration do
+    if (Dir.glob("db/migrate/*_backup_uploader_columns.rb").none?)
+      FileUtils.cp __dir + "/templates/delete_uploader_backup_tables.rb", 
+      "db/migrate/#{(DateTime.now + 1.second).strftime('%Y%m%d%H%M%S')}_delete_uploader_backup_tables.rb"
+    end
+  end
+
+  task :cleanup_upgrade_files do
+    FileUtils.rm_r "app/uploaders"
+    FileUtils.rm "config/initializers/carrierwave.rb"
+    gemfile_lines = File.readlines("Gemfile").select{|i| !i.include?("gem 'carrierwave'") && !i.include?("gem 'carrierwave-aws'")}
+    File.write('Gemfile', gemfile_lines.join("\n"))
+    cleanup_model_files
+  end
+
 end

gems/houdini_upgrade/lib/tasks/templates/backup_uploader_columns.rb

@@ -0,0 +1,55 @@
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+class BackupUploaderColumns < ActiveRecord::Migration[5.2]
+    def up
+        # copy all 
+        HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table|
+            create_table table.backup_table do |table|
+                table.references table.name, index:true
+                table.fields.each do |f|
+                    table.string f.migrated_name
+                end
+            end
+        end
+        # copy all 
+        HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table|
+            
+            execute <<-SQL
+            INSERT INTO #{table.backup_table} (#{table.foreign_key}, #{table.fields.map(&:migrated_name).join(', ')})
+            VALUES ( SELECT id, #{table.fields.map(&:migrated_name).join(', ')} FROM #{table.name})
+            SQL;
+        end
+
+        # delete columns
+        HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table|
+           
+            fields.each do |f|
+                drop_column table.name, f.migrated_name
+            end
+        end
+    end
+
+    def down
+        ## readd columns
+        HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table|
+            fields.each do |f|
+                add_column table.name, f.migrated_name, :string
+            end
+        end
+
+        # copy all 
+        HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each do |table|
+            execute <<-SQL
+            UPDATE  #{table.name}  SET (#{table.fields.map(&:migrated_name).join(', ')}) = (
+                SELECT (#{table.fields.map(&:migrated_name).join(', ')}) FROM #{table.backup_table}
+                WHERE #{table.backup_table}.#{table.foreign_key} = #{table.name}.id
+            )
+
+            SQL;
+        end
+        # delete tables
+
+        HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each{ |entity, _| drop_table table.backup_table}
+        
+    
+    end
+end

gems/houdini_upgrade/lib/tasks/templates/delete_uploader_backup_tables.rb

@@ -0,0 +1,7 @@
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+class DeleteUploaderBackupTables < ActiveRecord::Migration[5.2]
+    def up
+        # delete tables
+        HoudiniUpgrade::UPLOADERS_TO_MIGRATE.each{ |table| drop_table table.backup_table}
+    end
+end