Finalize houdini_full_contact

config/initializers/houdini_event_publisher.rb

@@ -5,7 +5,8 @@
 
 Wisper.clear if Rails.env.development?
 
-Rails.application.config.houdini.listeners << [NonprofitMailerListener, 
+Rails.application.config.houdini.listeners.push(NonprofitMailerListener,
     CreditCardPaymentListener,
     SepaPaymentListener,
-    TicketListener]
+    TicketListener
+)

gems/bess/lib/houdini/engine.rb

@@ -104,7 +104,7 @@ in the provided locales: #{Houdini.intl.available_locales.join(', ')}") if Houdi
         Houdini.show_state_fields = app.config.houdini.show_state_fields
         Houdini.default_bp = app.config.houdini.default_bp.id
 
-        Houdini.event_publisher.subscribe_all(app.config.houdini.listeners.flatten)
+        Houdini.event_publisher.subscribe_all(app.config.houdini.listeners)
       end
     end
   end

gems/houdini_full_contact/README.md

@@ -1,8 +1,15 @@
 # FullContact
-An Houdini add-on to use FullContact's Context API
+An Houdini add-on to use FullContact's Enrich API. This add-on provides a few features:
+
+* a event listener for supporter_create which downloads information from the Enrich API.
+* adds a has_many relation on Supporter for every set of data about that Supporter downloaded.
+Each item is an instance of `Houdini::FullContact::Info`
 
 ## Usage
-How to use my plugin.
+You can provide your FullContact API key in one of two ways:
+
+* Setting the `FULL_CONTACT_KEY` environment variable or
+* Setting the `houdini.full_contact.api_key` configuration option
 
 ## Installation
 Add this line to your application's Gemfile:
@@ -16,10 +23,7 @@ And then execute:
 $ bundle
 ```
 
-Or install it yourself as:
+And then install the database migrations for houdini_full_contact:
 ```bash
-$ gem install full_contact
+bin/rails houdini_full_contact:install:migrations
 ```
-
-## Contributing
-Contribution directions go here.

gems/houdini_full_contact/app/jobs/houdini/full_contact/application_job.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
+# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
+class Houdini::FullContact::ApplicationJob < ActiveJob::Base
+end

gems/houdini_full_contact/app/jobs/houdini/full_contact/full_contact_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
+# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
+class Houdini::FullContact::FullContactJob < Houdini::FullContact::ApplicationJob
+    queue_as :full_contact_queue
+
+    retry_on Exception, wait: ->(executions) { executions **2.195 }, attempts: Houdini::FullContact.max_attempts || 1
+
+    def perform(supporter)
+        Houdini::FullContact::InsertInfos.single(supporter)
+    end
+end

gems/houdini_full_contact/app/listeners/houdini/full_contact/full_contact_listener.rb

@@ -1,5 +1,13 @@
-module Houdini::FullContact::FullContactListener
-    def supporter_create(supporter)
-        Houdini::FullContact::InsertInfos.enqueue(supporter.id)
+# frozen_string_literal: true
+
+# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
+# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
+class Houdini::FullContact::FullContactListener
+    def name
+        self.class.name
+    end
+
+    def self.supporter_create(supporter)
+        FullContactJob.perform_later(supporter)
     end
 end

gems/houdini_full_contact/lib/houdini/full_contact.rb

@@ -5,5 +5,5 @@ module Houdini::FullContact
 
     autoload :InsertInfos
 
-    mattr_accessor :api_key
+    mattr_accessor :api_key, :max_attempts
 end

gems/houdini_full_contact/lib/houdini/full_contact/engine.rb

@@ -7,6 +7,7 @@ module Houdini::FullContact
     config.generators.api_only = true
 
     config.houdini.full_contact = ActiveSupport::OrderedOptions.new
+    config.houdini.full_contact.max_attempts = 5
 
     initializer 'houdini.full_contact.supporter_extension' do 
       ActiveSupport.on_load(:houdini_supporter) do 
@@ -18,6 +19,7 @@ module Houdini::FullContact
       config.before_initialize do |app|
         Houdini::FullContact.api_key = app.config.houdini.full_contact.api_key || 
                 ENV.fetch('FULL_CONTACT_KEY')
+        Houdini::FullContact.max_attempts = app.config.houdini.full_contact.max_attempts
       end
     end
   end

gems/houdini_full_contact/lib/houdini/full_contact/insert_infos.rb

@@ -1,60 +1,34 @@
+# frozen_string_literal: true
+
 # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
 require 'qx'
 require 'rest-client'
 module Houdini::FullContact::InsertInfos
-  # Work off of the full_contact_jobs queue
-  def self.work_queue
-    ids = Houdini::FullContact::Job.pluck(:supporter_id)
-    Houdini::FullContact::Job.delete_all
-    self.bulk(ids) if ids.any?
-  end
-
-
-  # Enqueue full contact jobs for a set of supporter ids
-  def self.enqueue(supporter_ids)
-    Qx.insert_into(:full_contact_jobs)
-      .values(supporter_ids.map{|id| {supporter_id: id}})
-      .ex
-  end
-
-
-  # We need to throttle our requests by 10ms since that is our rate limit on FullContact
-  def self.bulk(supporter_ids)
-    created_ids = []
-    supporter_ids.each do |id|
-      now = Time.current
-      result = self.single id
-      created_ids.push(GetData.hash(result, 'full_contact_info', 'id')) if result.is_a?(Hash)
-      interval = 0.1 - (Time.current - now) # account for time taken in .single
-      sleep interval if interval > 0
-    end
-    return created_ids
-  end
-  
-
   # Fetch and persist a single full contact record for a single supporter
   # return an exception if 404 or something else went poop
-  def self.single(supporter_id)
-    supporter = Houdini.core_classes.fetch(:supporter).constantize
-    supp = supporter.find(supporter_id)
-    return if supp.nil? || supp.email.blank?
-
+  # @param supporter Supporter
+  def self.single(supporter)
+    return if supporter.nil? || supporter.email.blank?
     begin
       response = RestClient.post("https://api.fullcontact.com/v3/person.enrich",
         {
-          "email" => supp.email,
+          "email" => supporter.email,
         }.to_json,
         {
           :authorization => "Bearer #{Houdini::FullContact.api_key}",
-          "Reporting-Key" => supp.nonprofit_id
+          "Reporting-Key" => supporter.nonprofit_id
         })
-      result = JSON.parse(response.body)
-    rescue Exception => e
-      return e
+        result = JSON.parse(response.body)
+    rescue RestClient::NotFound => e
+      # this means there's no information about this contact so there's nothing to do.
+      # We just return and end
+
+      # NOTE: We pass on other errors because that means something failed. *shrug*
+      return
     end
 
     location = result['location'] && result['details']['locations'] && result['details']['locations'][0]
-    existing = supp.full_contact_info
+    existing = supporter.full_contact_infos.last
     info_data = {
       full_name: result['fullName'],
       gender: result['gender'],
@@ -64,7 +38,13 @@ module Houdini::FullContact::InsertInfos
       age_range: result['ageRange'],
       location_general: result['location'],
       websites: ((result['details'] && result['details']['urls']) || []).map{|h| h['value']}.join(','),
-      supporter_id: supporter_id
+      supporter_id: supporter.id
+    }
+    return {
+      'full_contact_info' => full_contact_info,
+      'full_contact_photos' => full_contact_photos,
+      'full_contact_social_profiles' => full_contact_social_profiles,
+      'full_contact_orgs' => full_contact_orgs
     }
 
     if existing

gems/houdini_full_contact/lib/tasks/houdini_full_contact_tasks.rake

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
-desc 'For generating Full Contact data'
-
-# Clear old activerecord sessions tables daily
-namespace :houdini do
-    namespace :full_contact do 
-        task work_queue: :environment do
-            loop do
-                sleep(10) until Qx.select('COUNT(*)').from('full_contact_jobs').execute.first['count'] > 0
-                puts 'working...'
-            
-                begin
-                    Houdini::FullContact::InsertInfos.work_queue
-                rescue Exception => e
-                    puts "Exception thrown: #{e}"
-                end
-            end
-            end
-    end
-end
-

lib/insert/insert_supporter.rb

@@ -41,7 +41,6 @@ module InsertSupporter
 
     # GeocodeModel.delay.supporter(supporter['id'])
     Houdini.event_publisher.announce(:supporter_create, Supporter.find(supporter['id']))
-    Houdini::FullContact::InsertInfos.enqueue([supporter['id']])
 
     supporter
   end