diff --git a/config/initializers/houdini_event_publisher.rb b/config/initializers/houdini_event_publisher.rb index d82baf3c..c4faee7a 100644 --- a/config/initializers/houdini_event_publisher.rb +++ b/config/initializers/houdini_event_publisher.rb @@ -5,7 +5,8 @@ Wisper.clear if Rails.env.development? -Rails.application.config.houdini.listeners << [NonprofitMailerListener, - CreditCardPaymentListener, - SepaPaymentListener, - TicketListener] +Rails.application.config.houdini.listeners.push(NonprofitMailerListener, + CreditCardPaymentListener, + SepaPaymentListener, + TicketListener +) diff --git a/gems/bess/lib/houdini/engine.rb b/gems/bess/lib/houdini/engine.rb index 07fcadc9..f2a9e6c4 100644 --- a/gems/bess/lib/houdini/engine.rb +++ b/gems/bess/lib/houdini/engine.rb @@ -104,7 +104,7 @@ in the provided locales: #{Houdini.intl.available_locales.join(', ')}") if Houdi Houdini.show_state_fields = app.config.houdini.show_state_fields Houdini.default_bp = app.config.houdini.default_bp.id - Houdini.event_publisher.subscribe_all(app.config.houdini.listeners.flatten) + Houdini.event_publisher.subscribe_all(app.config.houdini.listeners) end end end diff --git a/gems/houdini_full_contact/README.md b/gems/houdini_full_contact/README.md index 65103c1c..26188f58 100644 --- a/gems/houdini_full_contact/README.md +++ b/gems/houdini_full_contact/README.md @@ -1,8 +1,15 @@ # FullContact -An Houdini add-on to use FullContact's Context API +An Houdini add-on to use FullContact's Enrich API. This add-on provides a few features: + +* a event listener for supporter_create which downloads information from the Enrich API. +* adds a has_many relation on Supporter for every set of data about that Supporter downloaded. +Each item is an instance of `Houdini::FullContact::Info` ## Usage -How to use my plugin. +You can provide your FullContact API key in one of two ways: + +* Setting the `FULL_CONTACT_KEY` environment variable or +* Setting the `houdini.full_contact.api_key` configuration option ## Installation Add this line to your application's Gemfile: @@ -16,10 +23,7 @@ And then execute: $ bundle ``` -Or install it yourself as: +And then install the database migrations for houdini_full_contact: ```bash -$ gem install full_contact +bin/rails houdini_full_contact:install:migrations ``` - -## Contributing -Contribution directions go here. diff --git a/gems/houdini_full_contact/app/jobs/houdini/full_contact/application_job.rb b/gems/houdini_full_contact/app/jobs/houdini/full_contact/application_job.rb new file mode 100644 index 00000000..f33f9574 --- /dev/null +++ b/gems/houdini_full_contact/app/jobs/houdini/full_contact/application_job.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later +# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE +class Houdini::FullContact::ApplicationJob < ActiveJob::Base +end \ No newline at end of file diff --git a/gems/houdini_full_contact/app/jobs/houdini/full_contact/full_contact_job.rb b/gems/houdini_full_contact/app/jobs/houdini/full_contact/full_contact_job.rb new file mode 100644 index 00000000..0f1e47a7 --- /dev/null +++ b/gems/houdini_full_contact/app/jobs/houdini/full_contact/full_contact_job.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later +# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE +class Houdini::FullContact::FullContactJob < Houdini::FullContact::ApplicationJob + queue_as :full_contact_queue + + retry_on Exception, wait: ->(executions) { executions **2.195 }, attempts: Houdini::FullContact.max_attempts || 1 + + def perform(supporter) + Houdini::FullContact::InsertInfos.single(supporter) + end +end \ No newline at end of file diff --git a/gems/houdini_full_contact/app/listeners/houdini/full_contact/full_contact_listener.rb b/gems/houdini_full_contact/app/listeners/houdini/full_contact/full_contact_listener.rb index 16baa258..fbd1594d 100644 --- a/gems/houdini_full_contact/app/listeners/houdini/full_contact/full_contact_listener.rb +++ b/gems/houdini_full_contact/app/listeners/houdini/full_contact/full_contact_listener.rb @@ -1,5 +1,13 @@ -module Houdini::FullContact::FullContactListener - def supporter_create(supporter) - Houdini::FullContact::InsertInfos.enqueue(supporter.id) +# frozen_string_literal: true + +# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later +# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE +class Houdini::FullContact::FullContactListener + def name + self.class.name + end + + def self.supporter_create(supporter) + FullContactJob.perform_later(supporter) end end \ No newline at end of file diff --git a/gems/houdini_full_contact/lib/houdini/full_contact.rb b/gems/houdini_full_contact/lib/houdini/full_contact.rb index a6a780b4..8d7c4cfb 100644 --- a/gems/houdini_full_contact/lib/houdini/full_contact.rb +++ b/gems/houdini_full_contact/lib/houdini/full_contact.rb @@ -5,5 +5,5 @@ module Houdini::FullContact autoload :InsertInfos - mattr_accessor :api_key + mattr_accessor :api_key, :max_attempts end \ No newline at end of file diff --git a/gems/houdini_full_contact/lib/houdini/full_contact/engine.rb b/gems/houdini_full_contact/lib/houdini/full_contact/engine.rb index 38dbf778..ce76e8c5 100644 --- a/gems/houdini_full_contact/lib/houdini/full_contact/engine.rb +++ b/gems/houdini_full_contact/lib/houdini/full_contact/engine.rb @@ -7,6 +7,7 @@ module Houdini::FullContact config.generators.api_only = true config.houdini.full_contact = ActiveSupport::OrderedOptions.new + config.houdini.full_contact.max_attempts = 5 initializer 'houdini.full_contact.supporter_extension' do ActiveSupport.on_load(:houdini_supporter) do @@ -18,6 +19,7 @@ module Houdini::FullContact config.before_initialize do |app| Houdini::FullContact.api_key = app.config.houdini.full_contact.api_key || ENV.fetch('FULL_CONTACT_KEY') + Houdini::FullContact.max_attempts = app.config.houdini.full_contact.max_attempts end end end diff --git a/gems/houdini_full_contact/lib/houdini/full_contact/insert_infos.rb b/gems/houdini_full_contact/lib/houdini/full_contact/insert_infos.rb index 86492463..77c8a861 100644 --- a/gems/houdini_full_contact/lib/houdini/full_contact/insert_infos.rb +++ b/gems/houdini_full_contact/lib/houdini/full_contact/insert_infos.rb @@ -1,60 +1,34 @@ +# frozen_string_literal: true + # License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later require 'qx' require 'rest-client' module Houdini::FullContact::InsertInfos - # Work off of the full_contact_jobs queue - def self.work_queue - ids = Houdini::FullContact::Job.pluck(:supporter_id) - Houdini::FullContact::Job.delete_all - self.bulk(ids) if ids.any? - end - - - # Enqueue full contact jobs for a set of supporter ids - def self.enqueue(supporter_ids) - Qx.insert_into(:full_contact_jobs) - .values(supporter_ids.map{|id| {supporter_id: id}}) - .ex - end - - - # We need to throttle our requests by 10ms since that is our rate limit on FullContact - def self.bulk(supporter_ids) - created_ids = [] - supporter_ids.each do |id| - now = Time.current - result = self.single id - created_ids.push(GetData.hash(result, 'full_contact_info', 'id')) if result.is_a?(Hash) - interval = 0.1 - (Time.current - now) # account for time taken in .single - sleep interval if interval > 0 - end - return created_ids - end - - # Fetch and persist a single full contact record for a single supporter # return an exception if 404 or something else went poop - def self.single(supporter_id) - supporter = Houdini.core_classes.fetch(:supporter).constantize - supp = supporter.find(supporter_id) - return if supp.nil? || supp.email.blank? - + # @param supporter Supporter + def self.single(supporter) + return if supporter.nil? || supporter.email.blank? begin response = RestClient.post("https://api.fullcontact.com/v3/person.enrich", { - "email" => supp.email, + "email" => supporter.email, }.to_json, { :authorization => "Bearer #{Houdini::FullContact.api_key}", - "Reporting-Key" => supp.nonprofit_id + "Reporting-Key" => supporter.nonprofit_id }) - result = JSON.parse(response.body) - rescue Exception => e - return e + result = JSON.parse(response.body) + rescue RestClient::NotFound => e + # this means there's no information about this contact so there's nothing to do. + # We just return and end + + # NOTE: We pass on other errors because that means something failed. *shrug* + return end location = result['location'] && result['details']['locations'] && result['details']['locations'][0] - existing = supp.full_contact_info + existing = supporter.full_contact_infos.last info_data = { full_name: result['fullName'], gender: result['gender'], @@ -64,7 +38,13 @@ module Houdini::FullContact::InsertInfos age_range: result['ageRange'], location_general: result['location'], websites: ((result['details'] && result['details']['urls']) || []).map{|h| h['value']}.join(','), - supporter_id: supporter_id + supporter_id: supporter.id + } + return { + 'full_contact_info' => full_contact_info, + 'full_contact_photos' => full_contact_photos, + 'full_contact_social_profiles' => full_contact_social_profiles, + 'full_contact_orgs' => full_contact_orgs } if existing diff --git a/gems/houdini_full_contact/lib/tasks/houdini_full_contact_tasks.rake b/gems/houdini_full_contact/lib/tasks/houdini_full_contact_tasks.rake deleted file mode 100644 index 868fbfbe..00000000 --- a/gems/houdini_full_contact/lib/tasks/houdini_full_contact_tasks.rake +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later -desc 'For generating Full Contact data' - -# Clear old activerecord sessions tables daily -namespace :houdini do - namespace :full_contact do - task work_queue: :environment do - loop do - sleep(10) until Qx.select('COUNT(*)').from('full_contact_jobs').execute.first['count'] > 0 - puts 'working...' - - begin - Houdini::FullContact::InsertInfos.work_queue - rescue Exception => e - puts "Exception thrown: #{e}" - end - end - end - end -end - diff --git a/lib/insert/insert_supporter.rb b/lib/insert/insert_supporter.rb index 458a6154..22c1d577 100644 --- a/lib/insert/insert_supporter.rb +++ b/lib/insert/insert_supporter.rb @@ -41,7 +41,6 @@ module InsertSupporter # GeocodeModel.delay.supporter(supporter['id']) Houdini.event_publisher.announce(:supporter_create, Supporter.find(supporter['id'])) - Houdini::FullContact::InsertInfos.enqueue([supporter['id']]) supporter end