Remove Gemfile, fix bugs in gemspec

Gemfile

@@ -98,3 +98,5 @@ group :production do
 end
 
 gem 'bess', path: 'gems/bess'
+
+gem 'houdini_full_contact', path: 'gems/houdini_full_contact'

Gemfile.lock

@@ -16,6 +16,14 @@ PATH
       wisper (~> 2.0)
       wisper-activejob (~> 1.0.0)
 
+PATH
+  remote: gems/houdini_full_contact
+  specs:
+    houdini_full_contact (0.1.0)
+      bess
+      qx
+      rails (~> 6.0.3, >= 6.0.3.1)
+
 PATH
   remote: gems/ruby-param-validation
   specs:
@@ -439,6 +447,7 @@ DEPENDENCIES
   geocoder (~> 1.6.3)
   hamster (~> 3.0)
   heroku-deflater (~> 0.6.3)
+  houdini_full_contact!
   httparty (~> 0.17.0)
   i18n-js (~> 3.3)
   image_processing (~> 1.10.3)

app/models/full_contact_info.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
-# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
-class FullContactInfo < ApplicationRecord
-  # :email,
-  # :full_name,
-  # :gender,
-  # :city,
-  # :county,
-  # :state_code,
-  # :country,
-  # :continent,
-  # :age,
-  # :age_range,
-  # :location_general,
-  # :supporter_id, :supporter,
-  # :websites
-
-  has_many :full_contact_photos
-  has_many :full_contact_social_profiles
-  has_many :full_contact_orgs
-  has_many :full_contact_topics
-  belongs_to :supporter
-end

app/models/full_contact_org.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
-# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
-class FullContactOrg < ApplicationRecord
-  # :name,
-  # :is_primary,
-  # :name,
-  # :start_date,
-  # :end_date,
-  # :title,
-  # :current,
-  # :full_contact_info_id, :full_contact_info
-
-  belongs_to :full_contact_info
-end

app/models/full_contact_photo.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
-# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
-class FullContactPhoto < ApplicationRecord
-  # :full_contact_info,
-  # :full_contact_info_id,
-  # :type_id, # i.e. twitter, linkedin, facebook
-  # :is_primary, #bool
-  # :url #string
-
-  belongs_to :full_contact_info
-
-  validates_presence_of :full_contact_info
-end

app/models/full_contact_social_profile.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
-# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
-class FullContactSocialProfile < ApplicationRecord
-  # :full_contact_info,
-  # :full_contact_info_id,
-  # :type_id, # i.e. twitter, linkedin, facebook
-  # :username, #string
-  # :uid, # string
-  # :bio, #string
-  # :url #string
-
-  belongs_to :full_contact_info
-
-  validates_presence_of :full_contact_info
-end

app/models/full_contact_topic.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
-# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
-class FullContactTopic < ApplicationRecord
-  # :provider,
-  # :value,
-  # :full_contact_info_id, :full_contact_info
-
-  belongs_to :full_contact_info
-end

app/models/supporter.rb

@@ -33,7 +33,7 @@ class Supporter < ApplicationRecord
   belongs_to :profile
   belongs_to :nonprofit
   belongs_to :import
-  has_many :full_contact_infos
+  
   has_many :payments
   has_many :offsite_payments
   has_many :charges
@@ -85,3 +85,5 @@ class Supporter < ApplicationRecord
     Format::Address.full_address(address, city, state_code)
   end
 end
+
+ActiveSupport.run_load_hooks(:houdini_supporter, Supporter)

gems/full_contact/Gemfile

@@ -1,12 +0,0 @@
-source 'https://rubygems.org'
-git_source(:github) { |repo| "https://github.com/#{repo}.git" }
-
-# Specify your gem's dependencies in full_contact.gemspec.
-gemspec
-
-group :development do
-  gem 'sqlite3'
-end
-
-# To use a debugger
-# gem 'byebug', group: [:development, :test]

gems/full_contact/app/controllers/full_contact/application_controller.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-module FullContact
-  class ApplicationController < ActionController::API
-    # protect_from_forgery with: :exception
-  end
-end

gems/full_contact/app/mailers/full_contact/application_mailer.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-module FullContact
-  class ApplicationMailer < ActionMailer::Base
-    default from: 'from@example.com'
-    layout 'mailer'
-  end
-end

gems/full_contact/config/routes.rb

@@ -1,2 +0,0 @@
-FullContact::Engine.routes.draw do
-end

gems/full_contact/full_contact.gemspec

@@ -1,24 +0,0 @@
-require_relative "lib/full_contact/version"
-
-Gem::Specification.new do |spec|
-  spec.name        = "full_contact"
-  spec.version     = FullContact::VERSION
-  spec.authors     = [""]
-  spec.email       = ["eric@commitchange.com"]
-  spec.homepage    = "TODO"
-  spec.summary     = "TODO: Summary of FullContact."
-  spec.description = "TODO: Description of FullContact."
-  spec.license     = "AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later"
-
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
-  # to allow pushing to a single host or delete this section to allow pushing to any host.
-  spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
-
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
-  spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
-
-  spec.files = Dir["{app,config,db,lib}/**/*", "LICENSE", "AGPL-3.0.txt", "GPL-3.0.txt", "LGPL-3.0.txt", "Rakefile", "README.md"]
-
-  spec.add_dependency "rails", "~> 6.0.3", ">= 6.0.3.1"
-end

gems/full_contact/lib/full_contact.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-require "full_contact/engine"
-
-module FullContact
-  # Your code goes here...
-end

gems/full_contact/lib/full_contact/engine.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-module FullContact
-  class Engine < ::Rails::Engine
-    isolate_namespace FullContact
-    config.generators.api_only = true
-  end
-end

gems/full_contact/lib/tasks/full_contact_tasks.rake

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-# desc "Explaining what the task does"
-# task :full_contact do
-#   # Task goes here
-# end

gems/houdini_full_contact/README.md

@@ -1,5 +1,5 @@
 # FullContact
-Short description and motivation.
+An Houdini add-on to use FullContact's Context API
 
 ## Usage
 How to use my plugin.
@@ -8,7 +8,7 @@ How to use my plugin.
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'full_contact'
+gem 'houdini_full_contact', path: 'gems/houdini_full_contact
 ```
 
 And then execute:

gems/houdini_full_contact/app/models/houdini/full_contact/application_record.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-module FullContact
+class Houdini::FullContact::ApplicationRecord < ActiveRecord::Base
-  class ApplicationRecord < ActiveRecord::Base
     self.abstract_class = true
-  end
 end

gems/houdini_full_contact/app/models/houdini/full_contact/info.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+class Houdini::FullContact::Info < ApplicationRecord
+  self.table_name = 'full_contact_infos'
+
+  has_many :photos, foreign_key: 'full_contact_info_id'
+  has_many :social_profiles, foreign_key: 'full_contact_info_id'
+  has_many :orgs, foreign_key: 'full_contact_info_id'
+  has_many :topics, foreign_key: 'full_contact_info_id'
+
+  belongs_to :supporter, class_name: Houdini.core_classes.fetch(:supporter).to_s
+end

gems/houdini_full_contact/app/models/houdini/full_contact/job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+class Houdini::FullContact::Job < ApplicationRecord
+    self.table_name = 'full_contact_jobs'
+    belongs_to :supporter, class_name: Houdini.core_classes.fetch(:supporter)
+end

gems/houdini_full_contact/app/models/houdini/full_contact/org.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+class Houdini::FullContact::Org < ApplicationRecord
+  self.table_name = 'full_contact_orgs'
+
+  belongs_to :info, foreign_key: 'full_contact_info_id'
+end

gems/houdini_full_contact/app/models/houdini/full_contact/photo.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+class Houdini::FullContact::Photo < ApplicationRecord
+  self.table_name = 'full_contact_photos'
+
+  belongs_to :info, foreign_key: 'full_contact_info_id'
+
+  validates_presence_of :info
+end

gems/houdini_full_contact/app/models/houdini/full_contact/social_profile.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+class Houdini::FullContact::SocialProfile < ApplicationRecord
+  self.table_name = 'full_contact_social_profiles'
+
+  belongs_to :info, foreign_key: 'full_contact_info_id'
+
+  validates_presence_of :info
+end

gems/houdini_full_contact/app/models/houdini/full_contact/topic.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+class Houdini::FullContact::Topic < ApplicationRecord
+  self.table_name = 'full_contact_topics'
+
+  belongs_to :info, foreign_key: 'full_contact_info_id'
+end

gems/houdini_full_contact/bin/rails

@@ -3,7 +3,7 @@
 # installed from the root of your application.
 
 ENGINE_ROOT = File.expand_path('..', __dir__)
-ENGINE_PATH = File.expand_path('../lib/full_contact/engine', __dir__)
+ENGINE_PATH = File.expand_path('../lib/houdini/full_contact/engine', __dir__)
 
 # Set up gems listed in the Gemfile.
 ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
@@ -12,13 +12,13 @@ require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
 require "rails"
 # Pick the frameworks you want:
 require "active_model/railtie"
-require "active_job/railtie"
+# require "active_job/railtie"
 require "active_record/railtie"
 # require "active_storage/engine"
-require "action_controller/railtie"
+# require "action_controller/railtie"
-require "action_mailer/railtie"
+# require "action_mailer/railtie"
-require "action_view/railtie"
+# require "action_view/railtie"
-require "action_cable/engine"
+# require "action_cable/engine"
-require "sprockets/railtie"
+# require "sprockets/railtie"
 # require "rails/test_unit/railtie"
 require "rails/engine/commands"

gems/houdini_full_contact/config/routes.rb

@@ -0,0 +1,2 @@
+Houdini::FullContact::Engine.routes.draw do
+end

gems/houdini_full_contact/db/migrate/20200611182909_create_full_contact.rb

@@ -0,0 +1,63 @@
+class CreateFullContact < ActiveRecord::Migration[6.0]
+  def change
+    create_table :full_contact_infos do |t|
+      t.references :supporters
+      t.string :email
+      t.string :full_name
+      t.string :gender
+      t.string :city
+      t.string :county
+      t.string :state_code
+      t.string :country
+      t.string :continent
+      t.string :age
+      t.string :age_range
+      t.string :location_general
+      t.text :websites
+      t.timestamps
+    end
+    
+    create_table :full_contact_topics do |t|
+      t.references :full_contact_infos
+      t.string :provider
+      t.string :value
+      t.timestamps
+    end
+
+    create_table :full_contact_social_profiles do |t|
+      t.references :full_contact_infos
+      t.string :type_id 
+		  t.string :username 
+		  t.string :uid 
+		  t.text :bio 
+      t.string :url
+      t.integer :followers 
+      t.integer :following
+      t.timestamps
+    end
+
+    create_table :full_contact_social_orgs do |t|
+      t.references :full_contact_infos
+      t.boolean :is_primary
+      t.string :name
+      t.date :start_date
+      t.date :end_date
+      t.string :title
+      t.boolean :current
+      t.timestamps
+    end
+
+    create_table :full_contact_social_photos do |t|
+      t.references :full_contact_infos
+      t.string :type_id
+      t.boolean :is_primary
+      t.text :url
+      t.timestamps
+    end
+
+    create_table :full_contact_jobs do |t|
+      t.references :supporter
+      t.timestamps
+    end
+  end
+end

gems/houdini_full_contact/houdini_full_contact.gemspec

@@ -0,0 +1,26 @@
+require_relative "lib/houdini_full_contact/version"
+
+Gem::Specification.new do |spec|
+  spec.name        = "houdini_full_contact"
+  spec.version     = HoudiniFullContact::VERSION
+  spec.authors     = [""]
+  spec.email       = ["eric@commitchange.com"]
+  # spec.homepage    = "TODO"
+  spec.summary     = " Summary of FullContact."
+  # spec.description = "TODO: Description of FullContact."
+  spec.license     = "AGPL-3.0-or-later WITH WTO-Additional-Permission-3.0-or-later"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  # spec.metadata["homepage_uri"] = spec.homepage
+  # spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  spec.files = Dir["{app,config,db,lib}/**/*", "LICENSE", "AGPL-3.0.txt", "GPL-3.0.txt", "LGPL-3.0.txt", "Rakefile", "README.md"]
+
+  spec.add_dependency "rails", "~> 6.0.3", ">= 6.0.3.1"
+  spec.add_dependency "qx"
+  spec.add_dependency "bess"
+end

gems/houdini_full_contact/lib/houdini.rb

@@ -0,0 +1,4 @@
+require "houdini/full_contact"
+
+module Houdini
+end

gems/houdini_full_contact/lib/houdini/full_contact.rb

@@ -0,0 +1,9 @@
+require "houdini/full_contact/engine"
+
+module Houdini::FullContact
+    extend ActiveSupport::Autoload
+
+    autoload :InsertInfos
+
+    mattr_accessor :api_key
+end

gems/houdini_full_contact/lib/houdini/full_contact/engine.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+module Houdini::FullContact
+  class Engine < ::Rails::Engine
+    isolate_namespace Houdini::FullContact
+    config.generators.api_only = true
+
+    config.houdini.full_contact = ActiveSupport::OrderedOptions.new
+
+    initializer 'houdini.full_contact.supporter_extension' do 
+      ActiveSupport.on_load(:houdini_supporter) do 
+        self.has_many :full_contact_infos, class_name: 'Houdini::FullContact::Info'
+      end
+    end
+
+    initializer 'houdini.full_contact.configs' do 
+      config.before_initialize do |app|
+        Houdini::FullContact.api_key = app.config.houdini.full_contact.api_key || 
+                ENV.fetch('FULL_CONTACT_KEY')
+      end
+    end
+  end
+end

gems/houdini_full_contact/lib/houdini/full_contact/insert_infos.rb

@@ -0,0 +1,155 @@
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+require 'qx'
+require 'rest-client'
+module Houdini::FullContact::InsertInfos
+  # Work off of the full_contact_jobs queue
+  def self.work_queue
+    ids = Houdini::FullContact::Job.pluck(:supporter_id)
+    Houdini::FullContact::Job.delete_all
+    self.bulk(ids) if ids.any?
+  end
+
+
+  # Enqueue full contact jobs for a set of supporter ids
+  def self.enqueue(supporter_ids)
+    Qx.insert_into(:full_contact_jobs)
+      .values(supporter_ids.map{|id| {supporter_id: id}})
+      .ex
+  end
+
+
+  # We need to throttle our requests by 10ms since that is our rate limit on FullContact
+  def self.bulk(supporter_ids)
+    created_ids = []
+    supporter_ids.each do |id|
+      now = Time.current
+      result = self.single id
+      created_ids.push(GetData.hash(result, 'full_contact_info', 'id')) if result.is_a?(Hash)
+      interval = 0.1 - (Time.current - now) # account for time taken in .single
+      sleep interval if interval > 0
+    end
+    return created_ids
+  end
+  
+
+  # Fetch and persist a single full contact record for a single supporter
+  # return an exception if 404 or something else went poop
+  def self.single(supporter_id)
+    supporter = Houdini.core_classes.fetch(:supporter).constantize
+    supp = supporter.find(supporter_id)
+    return if supp.nil? || supp.email.blank?
+
+    begin
+      response = RestClient.post("https://api.fullcontact.com/v3/person.enrich",
+        {
+          "email" => supp.email,
+        }.to_json,
+        {
+          :authorization => "Bearer #{Houdini::FullContact.api_key}",
+          "Reporting-Key" => supp.nonprofit_id
+        })
+      result = JSON.parse(response.body)
+    rescue Exception => e
+      return e
+    end
+
+    location = result['location'] && result['details']['locations'] && result['details']['locations'][0]
+    existing = supp.full_contact_info
+    info_data = {
+      full_name: result['fullName'],
+      gender: result['gender'],
+      city: location && location['city'],
+      state_code: location && location['regionCode'],
+      country: location && location['countryCode'],
+      age_range: result['ageRange'],
+      location_general: result['location'],
+      websites: ((result['details'] && result['details']['urls']) || []).map{|h| h['value']}.join(','),
+      supporter_id: supporter_id
+    }
+
+    if existing
+      full_contact_info = Qx.update(:full_contact_infos)
+        .set(info_data)
+        .timestamps
+        .where(id: existing['id'])
+        .returning('*')
+        .execute.first
+    else
+      full_contact_info = Qx.insert_into(:full_contact_infos)
+        .values(info_data)
+        .returning('*')
+        .timestamps
+        .execute.first
+    end
+
+    if result['details']['photos'].present?
+      photo_data = result['details']['photos'].map{|h| {type_id: h['label'], url: h['value']}}
+      Qx.delete_from("full_contact_photos")
+        .where(full_contact_info_id: full_contact_info['id'])
+        .execute
+      full_contact_photos = Qx.insert_into(:full_contact_photos)
+        .values(photo_data)
+        .common_values(full_contact_info_id: full_contact_info['id'])
+        .timestamps
+        .returning("*")
+        .execute
+    end
+
+    if result['details']['profiles'].present?
+      profile_data = result['details']['profiles'].map{|k,v| {type_id: v['service'], username: v['username'], uid: v['userid'], bio: v['bio'], url: v['url'], followers: v['followers'], following: v['following']} }
+      Qx.delete_from("full_contact_social_profiles")
+        .where(full_contact_info_id: full_contact_info['id'])
+        .execute
+      full_contact_social_profiles = Qx.insert_into(:full_contact_social_profiles)
+        .values(profile_data)
+        .common_values(full_contact_info_id: full_contact_info['id'])
+        .timestamps
+        .returning("*")
+        .execute
+    end
+
+    if result['details'].present? && result['details']['employment'].present?
+      Qx.delete_from('full_contact_orgs')
+        .where(full_contact_info_id: full_contact_info['id'])
+        .execute
+      org_data = result['details']['employment'].map{|h| 
+        start_date = nil
+      end_date = nil
+        start_date = h['start'] && [h['start']['year'], h['start']['month'], h['start']['day']].select{|i| i.present?}.join('-')
+        end_date = h['end'] && [h['end']['year'], h['end']['month'], h['end']['day']].select{|i| i.present?}.join('-')
+        {
+          name: h['name'],
+          start_date: start_date,
+          end_date: end_date,
+          title: h['title'],
+          current: h['current']
+        } }
+        .map{|h| h[:end_date] = Format::Date.parse_partial_str(h[:end_date]); h}
+        .map{|h| h[:start_date] = Format::Date.parse_partial_str(h[:start_date]); h}
+
+      full_contact_orgs = Qx.insert_into(:full_contact_orgs)
+        .values(org_data)
+        .common_values(full_contact_info_id: full_contact_info['id'])
+        .timestamps
+        .returning('*')
+        .execute
+    end
+
+    return {
+      'full_contact_info' => full_contact_info,
+      'full_contact_photos' => full_contact_photos,
+      'full_contact_social_profiles' => full_contact_social_profiles,
+      'full_contact_orgs' => full_contact_orgs
+    }
+  end
+
+  # Delete all orphaned full contact infos that do not have supporters
+  # or full_contact photos, social_profiles, topics, orgs, etc that do not have a parent info
+  def self.cleanup_orphans
+    Info.includes(:supporter).where("supporters.id IS NULL").delete_all
+    Photo.includes(:full_contact_infos).where("full_contact_infos.id IS NULL").delete_all
+    SocialProfiles.includes(:full_contact_infos).where("full_contact_infos.id IS NULL").delete_all
+    Topics.includes(:full_contact_infos).where("full_contact_infos.id IS NULL").delete_all
+    Orgs.includes(:full_contact_infos).where("full_contact_infos.id IS NULL").delete_all
+  end
+end

gems/houdini_full_contact/lib/houdini_full_contact.rb

@@ -1,7 +1,4 @@
 # frozen_string_literal: true
 
 # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-module FullContact
+require_relative "./houdini"
-  class ApplicationJob < ActiveJob::Base
-  end
-end

gems/houdini_full_contact/lib/houdini_full_contact/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-module FullContact
+module HoudiniFullContact
   VERSION = '0.1.0'
 end

gems/houdini_full_contact/lib/tasks/houdini_full_contact_tasks.rake

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
+desc 'For generating Full Contact data'
+
+# Clear old activerecord sessions tables daily
+namespace :houdini do
+    namespace :full_contact do 
+        task work_queue: :environment do
+            loop do
+                sleep(10) until Qx.select('COUNT(*)').from('full_contact_jobs').execute.first['count'] > 0
+                puts 'working...'
+            
+                begin
+                    Houdini::FullContact::InsertInfos.work_queue
+                rescue Exception => e
+                    puts "Exception thrown: #{e}"
+                end
+            end
+            end
+    end
+end
+

lib/insert/insert_full_contact_infos.rb

@@ -1,206 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
-# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
-require 'qx'
-
-module InsertFullContactInfos
-  # Work off of the full_contact_jobs queue
-  def self.work_queue
-    ids = Qx.select('supporter_id').from('full_contact_jobs').ex.map { |h| h['supporter_id'] }
-    Qx.delete_from('full_contact_jobs').where('TRUE').execute
-    bulk(ids) if ids.any?
-  end
-
-  # Enqueue full contact jobs for a set of supporter ids
-  def self.enqueue(supporter_ids)
-    Qx.insert_into(:full_contact_jobs)
-      .values(supporter_ids.map { |id| { supporter_id: id } })
-      .ex
-  end
-
-  # We need to throttle our requests by 10ms since that is our rate limit on FullContact
-  def self.bulk(supporter_ids)
-    created_ids = []
-    supporter_ids.each do |id|
-      now = Time.current
-      result = InsertFullContactInfos.single id
-      created_ids.push(GetData.hash(result, 'full_contact_info', 'id')) if result.is_a?(Hash)
-      interval = 0.1 - (Time.current - now) # account for time taken in .single
-      sleep interval if interval > 0
-    end
-    created_ids
-  end
-
-  # Fetch and persist a single full contact record for a single supporter
-  # return an exception if 404 or something else went poop
-  def self.single(supporter_id)
-    supp = Qx.select('email').from('supporters').where(id: supporter_id).execute.first
-    return if supp.nil? || supp['email'].blank?
-
-    begin
-      result = FullContact.person(email: supp['email']).to_h
-    rescue Exception => e
-      return e
-    end
-
-    if result['status'] == 202 # Queued for search
-      # self.enqueue([supporter_id])
-      return result
-    end
-
-    existing = Qx.select('id').from('full_contact_infos').where(supporter_id: supporter_id).ex.first
-    info_data = {
-      full_name: GetData.hash(result, 'contact_info', 'full_name'),
-      gender: GetData.hash(result, 'demographics', 'gender'),
-      city: GetData.hash(result, 'demographics', 'location_deduced', 'city', 'name'),
-      county: GetData.hash(result, 'demographics', 'location_deduced', 'county', 'name'),
-      state_code: GetData.hash(result, 'demographics', 'location_deduced', 'state', 'code'),
-      country: GetData.hash(result, 'demographics', 'location_deduced', 'country', 'name'),
-      continent: GetData.hash(result, 'demographics', 'location_deduced', 'continent', 'name'),
-      age: GetData.hash(result, 'demographics', 'age'),
-      age_range: GetData.hash(result, 'demographics', 'age_range'),
-      location_general: GetData.hash(result, 'demographics', 'location_general'),
-      websites: (GetData.hash(result, 'contact_info', 'websites') || []).map(&:url).join(','),
-      supporter_id: supporter_id
-    }
-
-    full_contact_info = if existing
-                          Qx.update(:full_contact_infos)
-                            .set(info_data)
-                            .timestamps
-                            .where(id: existing['id'])
-                            .returning('*')
-                            .execute.first
-                        else
-                          Qx.insert_into(:full_contact_infos)
-                            .values(info_data)
-                            .returning('*')
-                            .timestamps
-                            .execute.first
-                        end
-
-    if result['photos'].present?
-      photo_data = result['photos'].map { |h| { type_id: h.type_id, url: h.url, is_primary: h.is_primary } }
-      Qx.delete_from('full_contact_photos')
-        .where(full_contact_info_id: full_contact_info['id'])
-        .execute
-      full_contact_photos = Qx.insert_into(:full_contact_photos)
-                              .values(photo_data)
-                              .common_values(full_contact_info_id: full_contact_info['id'])
-                              .timestamps
-                              .returning('*')
-                              .execute
-    end
-
-    if result['social_profiles'].present?
-      profile_data = result['social_profiles'].map { |h| { type_id: h.type_id, username: h.username, uid: h.id, bio: h.bio, url: h.url, followers: h.followers, following: h.following } }
-      Qx.delete_from('full_contact_social_profiles')
-        .where(full_contact_info_id: full_contact_info['id'])
-        .execute
-      full_contact_social_profiles = Qx.insert_into(:full_contact_social_profiles)
-                                       .values(profile_data)
-                                       .common_values(full_contact_info_id: full_contact_info['id'])
-                                       .timestamps
-                                       .returning('*')
-                                       .execute
-    end
-
-    if result['digital_footprint'] && result['digital_footprint']['topics'].present?
-      profile_data = result['social_profiles']
-                     .map do |h|
-        {
-          type_id: h.type_id,
-          username: h.username,
-          uid: h.id,
-          bio: h.bio,
-          url: h.url,
-          followers: h.followers,
-          following: h.following
-        }
-      end
-
-      vals = result['digital_footprint']['topics'].map(&:value)
-      existing_vals = Qx.select('value').from('full_contact_topics')
-                        .where('value IN ($vals)', vals: vals)
-                        .and_where('full_contact_info_id=$id', id: full_contact_info['id'])
-                        .execute.map { |h| h['value'] }
-
-      topic_data = result['digital_footprint']['topics']
-                   .reject { |h| existing_vals.include?(h.value) }
-                   .map { |h| { value: h.value, provider: h.provider } }
-
-      if topic_data.any?
-        full_contact_topics = Qx.insert_into(:full_contact_topics)
-                                .values(topic_data)
-                                .common_values(full_contact_info_id: full_contact_info['id'])
-                                .timestamps
-                                .returning('*')
-                                .execute
-      end
-    end
-
-    if result['organizations'].present?
-      Qx.delete_from('full_contact_orgs')
-        .where(full_contact_info_id: full_contact_info['id'])
-        .execute
-      org_data = result['organizations'].map do |h|
-                   {
-                     is_primary: h.is_primary,
-                     name: h.name,
-                     start_date: h.start_date,
-                     end_date: h.end_date,
-                     title: h.title,
-                     current: h.current
-                   }
-                 end
-                                        .map { |h| h[:end_date] = Format::Date.parse_partial_str(h[:end_date]); h }
-                                        .map { |h| h[:start_date] = Format::Date.parse_partial_str(h[:start_date]); h }
-
-      full_contact_orgs = Qx.insert_into(:full_contact_orgs)
-                            .values(org_data)
-                            .common_values(full_contact_info_id: full_contact_info['id'])
-                            .timestamps
-                            .returning('*')
-                            .execute
-    end
-
-    {
-      'full_contact_info' => full_contact_info,
-      'full_contact_photos' => full_contact_photos,
-      'full_contact_social_profiles' => full_contact_social_profiles,
-      'full_contact_topics' => full_contact_topics,
-      'full_contact_orgs' => full_contact_orgs
-    }
-  end
-
-  # Delete all orphaned full contact infos that do not have supporters
-  # or full_contact photos, social_profiles, topics, orgs, etc that do not have a parent info
-  def self.cleanup_orphans
-    Qx.delete_from('full_contact_infos')
-      .where('id IN ($ids)', ids: Qx.select('full_contact_infos.id')
-        .from('full_contact_infos')
-        .left_join('supporters', 'full_contact_infos.supporter_id=supporters.id')
-        .where('supporters.id IS NULL')).ex
-    Qx.delete_from('full_contact_photos')
-      .where('id IN ($ids)', ids: Qx.select('full_contact_photos.id')
-        .from('full_contact_photos')
-        .left_join('full_contact_infos', 'full_contact_infos.id=full_contact_photos.full_contact_info_id')
-        .where('full_contact_infos.id IS NULL')).ex
-    Qx.delete_from('full_contact_social_profiles')
-      .where('id IN ($ids)', ids: Qx.select('full_contact_social_profiles.id')
-        .from('full_contact_social_profiles')
-        .left_join('full_contact_infos', 'full_contact_infos.id=full_contact_social_profiles.full_contact_info_id')
-        .where('full_contact_infos.id IS NULL')).ex
-    Qx.delete_from('full_contact_topics')
-      .where('id IN ($ids)', ids: Qx.select('full_contact_topics.id')
-        .from('full_contact_topics')
-        .left_join('full_contact_infos', 'full_contact_infos.id=full_contact_topics.full_contact_info_id')
-        .where('full_contact_infos.id IS NULL')).ex
-    Qx.delete_from('full_contact_orgs')
-      .where('id IN ($ids)', ids: Qx.select('full_contact_orgs.id')
-        .from('full_contact_orgs')
-        .left_join('full_contact_infos', 'full_contact_infos.id=full_contact_orgs.full_contact_info_id')
-        .where('full_contact_infos.id IS NULL')).ex
-  end
-end

lib/insert/insert_import.rb

@@ -7,7 +7,6 @@ require 'required_keys'
 require 'open-uri'
 require 'csv'
 require 'insert/insert_supporter'
-require 'insert/insert_full_contact_infos'
 require 'insert/insert_custom_field_joins'
 require 'insert/insert_tag_joins'
 
@@ -164,7 +163,7 @@ module InsertImport
                .where(id: import['id'])
                .returning('*')
                .execute.first
-    InsertFullContactInfos.enqueue(supporter_ids) if supporter_ids.any?
+    Houdini::FullContact::InsertInfos.enqueue(supporter_ids) if supporter_ids.any?
     ImportCompletedJob.perform_later(Import.find(import['id']))
     import
   end

lib/tasks/full_contact.rake

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
-# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
-desc 'For generating Full Contact data'
-
-# Clear old activerecord sessions tables daily
-task work_full_contact_queue: :environment do
-  loop do
-    sleep(10) until Qx.select('COUNT(*)').from('full_contact_jobs').execute.first['count'] > 0
-    puts 'working...'
-
-    begin
-      InsertFullContactInfos.work_queue
-    rescue Exception => e
-      puts "Exception thrown: #{e}"
-    end
-  end
-end