NPO-Accounting/houdini
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Remove Gemfile, fix bugs in gemspec

Browse source
This commit is contained in:
Eric 2020-06-11 13:28:02 -05:00 committed by Eric Schultz
parent b196c6711d
commit 64ebf2f26b
46 changed files with 393 additions and 412 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Gemfile
View file

@ -97,4 +97,6 @@ group :production do
gem 'rack-timeout', '~> 0.5.1'
end
gem 'bess', path: 'gems/bess'
gem 'bess', path: 'gems/bess'
gem 'houdini_full_contact', path: 'gems/houdini_full_contact'

9
Gemfile.lock
View file

@ -16,6 +16,14 @@ PATH
wisper (~> 2.0)
wisper-activejob (~> 1.0.0)
PATH
remote: gems/houdini_full_contact
specs:
houdini_full_contact (0.1.0)
bess
qx
rails (~> 6.0.3, >= 6.0.3.1)
PATH
remote: gems/ruby-param-validation
specs:
@ -439,6 +447,7 @@ DEPENDENCIES
geocoder (~> 1.6.3)
hamster (~> 3.0)
heroku-deflater (~> 0.6.3)
houdini_full_contact!
httparty (~> 0.17.0)
i18n-js (~> 3.3)
image_processing (~> 1.10.3)

25
app/models/full_contact_info.rb
View file

@ -1,25 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
class FullContactInfo < ApplicationRecord
# :email,
# :full_name,
# :gender,
# :city,
# :county,
# :state_code,
# :country,
# :continent,
# :age,
# :age_range,
# :location_general,
# :supporter_id, :supporter,
# :websites
has_many :full_contact_photos
has_many :full_contact_social_profiles
has_many :full_contact_orgs
has_many :full_contact_topics
belongs_to :supporter
end

16
app/models/full_contact_org.rb
View file

@ -1,16 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
class FullContactOrg < ApplicationRecord
# :name,
# :is_primary,
# :name,
# :start_date,
# :end_date,
# :title,
# :current,
# :full_contact_info_id, :full_contact_info
belongs_to :full_contact_info
end

15
app/models/full_contact_photo.rb
View file

@ -1,15 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
class FullContactPhoto < ApplicationRecord
# :full_contact_info,
# :full_contact_info_id,
# :type_id, # i.e. twitter, linkedin, facebook
# :is_primary, #bool
# :url #string
belongs_to :full_contact_info
validates_presence_of :full_contact_info
end

17
app/models/full_contact_social_profile.rb
View file

@ -1,17 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
class FullContactSocialProfile < ApplicationRecord
# :full_contact_info,
# :full_contact_info_id,
# :type_id, # i.e. twitter, linkedin, facebook
# :username, #string
# :uid, # string
# :bio, #string
# :url #string
belongs_to :full_contact_info
validates_presence_of :full_contact_info
end

11
app/models/full_contact_topic.rb
View file

@ -1,11 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
class FullContactTopic < ApplicationRecord
# :provider,
# :value,
# :full_contact_info_id, :full_contact_info
belongs_to :full_contact_info
end

4
app/models/supporter.rb
View file

@ -33,7 +33,7 @@ class Supporter < ApplicationRecord
belongs_to :profile
belongs_to :nonprofit
belongs_to :import
has_many :full_contact_infos
has_many :payments
has_many :offsite_payments
has_many :charges
@ -85,3 +85,5 @@ class Supporter < ApplicationRecord
Format::Address.full_address(address, city, state_code)
end
end
ActiveSupport.run_load_hooks(:houdini_supporter, Supporter)

2
gems/bess/lib/houdini.rb
View file

@ -34,7 +34,7 @@ module Houdini
mattr_accessor :button_host
mattr_accessor :support_email
mattr_accessor :core_classes, default: {supporter: 'Supporter', nonprofit: 'Nonprofit'}
mattr_accessor :event_publisher, default: Houdini::EventPublisher.new

12
gems/full_contact/Gemfile
View file

@ -1,12 +0,0 @@
source 'https://rubygems.org'
git_source(:github) { |repo| "https://github.com/#{repo}.git" }
# Specify your gem's dependencies in full_contact.gemspec.
gemspec
group :development do
gem 'sqlite3'
end
# To use a debugger
# gem 'byebug', group: [:development, :test]

8
gems/full_contact/app/controllers/full_contact/application_controller.rb
View file

@ -1,8 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module FullContact
class ApplicationController < ActionController::API
# protect_from_forgery with: :exception
end
end

9
gems/full_contact/app/mailers/full_contact/application_mailer.rb
View file

@ -1,9 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module FullContact
class ApplicationMailer < ActionMailer::Base
default from: 'from@example.com'
layout 'mailer'
end
end

2
gems/full_contact/config/routes.rb
View file

@ -1,2 +0,0 @@
FullContact::Engine.routes.draw do
end

24
gems/full_contact/full_contact.gemspec
View file

@ -1,24 +0,0 @@
require_relative "lib/full_contact/version"
Gem::Specification.new do |spec|
spec.name = "full_contact"
spec.version = FullContact::VERSION
spec.authors = [""]
spec.email = ["eric@commitchange.com"]
spec.homepage = "TODO"
spec.summary = "TODO: Summary of FullContact."
spec.description = "TODO: Description of FullContact."
spec.license = "AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later"
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
# to allow pushing to a single host or delete this section to allow pushing to any host.
spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
spec.metadata["homepage_uri"] = spec.homepage
spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
spec.files = Dir["{app,config,db,lib}/**/*", "LICENSE", "AGPL-3.0.txt", "GPL-3.0.txt", "LGPL-3.0.txt", "Rakefile", "README.md"]
spec.add_dependency "rails", "~> 6.0.3", ">= 6.0.3.1"
end

8
gems/full_contact/lib/full_contact.rb
View file

@ -1,8 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require "full_contact/engine"
module FullContact
# Your code goes here...
end

9
gems/full_contact/lib/full_contact/engine.rb
View file

@ -1,9 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module FullContact
class Engine < ::Rails::Engine
isolate_namespace FullContact
config.generators.api_only = true
end
end

7
gems/full_contact/lib/tasks/full_contact_tasks.rake
View file

@ -1,7 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
# desc "Explaining what the task does"
# task :full_contact do
# # Task goes here
# end

0
gems/full_contact/.gitignore → gems/houdini_full_contact/.gitignore vendored
View file

0
gems/full_contact/AGPL-3.0.txt → gems/houdini_full_contact/AGPL-3.0.txt
View file

0
gems/full_contact/GPL-3.0.txt → gems/houdini_full_contact/GPL-3.0.txt
View file

0
gems/full_contact/LGPL-3.0.txt → gems/houdini_full_contact/LGPL-3.0.txt
View file

0
gems/full_contact/LICENSE → gems/houdini_full_contact/LICENSE
View file

4
gems/full_contact/README.md → gems/houdini_full_contact/README.md
View file

@ -1,5 +1,5 @@
# FullContact
Short description and motivation.
An Houdini add-on to use FullContact's Context API
## Usage
How to use my plugin.
@ -8,7 +8,7 @@ How to use my plugin.
Add this line to your application's Gemfile:
```ruby
gem 'full_contact'
gem 'houdini_full_contact', path: 'gems/houdini_full_contact
```
And then execute:

0
gems/full_contact/Rakefile → gems/houdini_full_contact/Rakefile
View file

6
gems/full_contact/app/models/full_contact/application_record.rb → gems/houdini_full_contact/app/models/houdini/full_contact/application_record.rb
View file

@ -1,8 +1,6 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module FullContact
class ApplicationRecord < ActiveRecord::Base
class Houdini::FullContact::ApplicationRecord < ActiveRecord::Base
self.abstract_class = true
end
end
end

13
gems/houdini_full_contact/app/models/houdini/full_contact/info.rb Normal file
View file

@ -0,0 +1,13 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
class Houdini::FullContact::Info < ApplicationRecord
self.table_name = 'full_contact_infos'
has_many :photos, foreign_key: 'full_contact_info_id'
has_many :social_profiles, foreign_key: 'full_contact_info_id'
has_many :orgs, foreign_key: 'full_contact_info_id'
has_many :topics, foreign_key: 'full_contact_info_id'
belongs_to :supporter, class_name: Houdini.core_classes.fetch(:supporter).to_s
end

7
gems/houdini_full_contact/app/models/houdini/full_contact/job.rb Normal file
View file

@ -0,0 +1,7 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
class Houdini::FullContact::Job < ApplicationRecord
self.table_name = 'full_contact_jobs'
belongs_to :supporter, class_name: Houdini.core_classes.fetch(:supporter)
end

8
gems/houdini_full_contact/app/models/houdini/full_contact/org.rb Normal file
View file

@ -0,0 +1,8 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
class Houdini::FullContact::Org < ApplicationRecord
self.table_name = 'full_contact_orgs'
belongs_to :info, foreign_key: 'full_contact_info_id'
end

10
gems/houdini_full_contact/app/models/houdini/full_contact/photo.rb Normal file
View file

@ -0,0 +1,10 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
class Houdini::FullContact::Photo < ApplicationRecord
self.table_name = 'full_contact_photos'
belongs_to :info, foreign_key: 'full_contact_info_id'
validates_presence_of :info
end

10
gems/houdini_full_contact/app/models/houdini/full_contact/social_profile.rb Normal file
View file

@ -0,0 +1,10 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
class Houdini::FullContact::SocialProfile < ApplicationRecord
self.table_name = 'full_contact_social_profiles'
belongs_to :info, foreign_key: 'full_contact_info_id'
validates_presence_of :info
end

8
gems/houdini_full_contact/app/models/houdini/full_contact/topic.rb Normal file
View file

@ -0,0 +1,8 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
class Houdini::FullContact::Topic < ApplicationRecord
self.table_name = 'full_contact_topics'
belongs_to :info, foreign_key: 'full_contact_info_id'
end

14
gems/full_contact/bin/rails → gems/houdini_full_contact/bin/rails
View file

@ -3,7 +3,7 @@
# installed from the root of your application.
ENGINE_ROOT = File.expand_path('..', __dir__)
ENGINE_PATH = File.expand_path('../lib/full_contact/engine', __dir__)
ENGINE_PATH = File.expand_path('../lib/houdini/full_contact/engine', __dir__)
# Set up gems listed in the Gemfile.
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
@ -12,13 +12,13 @@ require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
require "rails"
# Pick the frameworks you want:
require "active_model/railtie"
require "active_job/railtie"
# require "active_job/railtie"
require "active_record/railtie"
# require "active_storage/engine"
require "action_controller/railtie"
require "action_mailer/railtie"
require "action_view/railtie"
require "action_cable/engine"
require "sprockets/railtie"
# require "action_controller/railtie"
# require "action_mailer/railtie"
# require "action_view/railtie"
# require "action_cable/engine"
# require "sprockets/railtie"
# require "rails/test_unit/railtie"
require "rails/engine/commands"

2
gems/houdini_full_contact/config/routes.rb Normal file
View file

@ -0,0 +1,2 @@
Houdini::FullContact::Engine.routes.draw do
end

0
gems/houdini_full_contact/db/migrate/.gitkeep Executable file
View file

63
gems/houdini_full_contact/db/migrate/20200611182909_create_full_contact.rb Normal file
View file

@ -0,0 +1,63 @@
class CreateFullContact < ActiveRecord::Migration[6.0]
def change
create_table :full_contact_infos do |t|
t.references :supporters
t.string :email
t.string :full_name
t.string :gender
t.string :city
t.string :county
t.string :state_code
t.string :country
t.string :continent
t.string :age
t.string :age_range
t.string :location_general
t.text :websites
t.timestamps
end
create_table :full_contact_topics do |t|
t.references :full_contact_infos
t.string :provider
t.string :value
t.timestamps
end
create_table :full_contact_social_profiles do |t|
t.references :full_contact_infos
t.string :type_id
t.string :username
t.string :uid
t.text :bio
t.string :url
t.integer :followers
t.integer :following
t.timestamps
end
create_table :full_contact_social_orgs do |t|
t.references :full_contact_infos
t.boolean :is_primary
t.string :name
t.date :start_date
t.date :end_date
t.string :title
t.boolean :current
t.timestamps
end
create_table :full_contact_social_photos do |t|
t.references :full_contact_infos
t.string :type_id
t.boolean :is_primary
t.text :url
t.timestamps
end
create_table :full_contact_jobs do |t|
t.references :supporter
t.timestamps
end
end
end

26
gems/houdini_full_contact/houdini_full_contact.gemspec Normal file
View file

@ -0,0 +1,26 @@
require_relative "lib/houdini_full_contact/version"
Gem::Specification.new do |spec|
spec.name = "houdini_full_contact"
spec.version = HoudiniFullContact::VERSION
spec.authors = [""]
spec.email = ["eric@commitchange.com"]
# spec.homepage = "TODO"
spec.summary = " Summary of FullContact."
# spec.description = "TODO: Description of FullContact."
spec.license = "AGPL-3.0-or-later WITH WTO-Additional-Permission-3.0-or-later"
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
# to allow pushing to a single host or delete this section to allow pushing to any host.
spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
# spec.metadata["homepage_uri"] = spec.homepage
# spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
# spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
spec.files = Dir["{app,config,db,lib}/**/*", "LICENSE", "AGPL-3.0.txt", "GPL-3.0.txt", "LGPL-3.0.txt", "Rakefile", "README.md"]
spec.add_dependency "rails", "~> 6.0.3", ">= 6.0.3.1"
spec.add_dependency "qx"
spec.add_dependency "bess"
end

4
gems/houdini_full_contact/lib/houdini.rb Normal file
View file

@ -0,0 +1,4 @@
require "houdini/full_contact"
module Houdini
end

9
gems/houdini_full_contact/lib/houdini/full_contact.rb Normal file
View file

@ -0,0 +1,9 @@
require "houdini/full_contact/engine"
module Houdini::FullContact
extend ActiveSupport::Autoload
autoload :InsertInfos
mattr_accessor :api_key
end

24
gems/houdini_full_contact/lib/houdini/full_contact/engine.rb Normal file
View file

@ -0,0 +1,24 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module Houdini::FullContact
class Engine < ::Rails::Engine
isolate_namespace Houdini::FullContact
config.generators.api_only = true
config.houdini.full_contact = ActiveSupport::OrderedOptions.new
initializer 'houdini.full_contact.supporter_extension' do
ActiveSupport.on_load(:houdini_supporter) do
self.has_many :full_contact_infos, class_name: 'Houdini::FullContact::Info'
end
end
initializer 'houdini.full_contact.configs' do
config.before_initialize do |app|
Houdini::FullContact.api_key = app.config.houdini.full_contact.api_key ||
ENV.fetch('FULL_CONTACT_KEY')
end
end
end
end

155
gems/houdini_full_contact/lib/houdini/full_contact/insert_infos.rb Normal file
View file

@ -0,0 +1,155 @@
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
require 'qx'
require 'rest-client'
module Houdini::FullContact::InsertInfos
# Work off of the full_contact_jobs queue
def self.work_queue
ids = Houdini::FullContact::Job.pluck(:supporter_id)
Houdini::FullContact::Job.delete_all
self.bulk(ids) if ids.any?
end
# Enqueue full contact jobs for a set of supporter ids
def self.enqueue(supporter_ids)
Qx.insert_into(:full_contact_jobs)
.values(supporter_ids.map{|id| {supporter_id: id}})
.ex
end
# We need to throttle our requests by 10ms since that is our rate limit on FullContact
def self.bulk(supporter_ids)
created_ids = []
supporter_ids.each do |id|
now = Time.current
result = self.single id
created_ids.push(GetData.hash(result, 'full_contact_info', 'id')) if result.is_a?(Hash)
interval = 0.1 - (Time.current - now) # account for time taken in .single
sleep interval if interval > 0
end
return created_ids
end
# Fetch and persist a single full contact record for a single supporter
# return an exception if 404 or something else went poop
def self.single(supporter_id)
supporter = Houdini.core_classes.fetch(:supporter).constantize
supp = supporter.find(supporter_id)
return if supp.nil? || supp.email.blank?
begin
response = RestClient.post("https://api.fullcontact.com/v3/person.enrich",
{
"email" => supp.email,
}.to_json,
{
:authorization => "Bearer #{Houdini::FullContact.api_key}",
"Reporting-Key" => supp.nonprofit_id
})
result = JSON.parse(response.body)
rescue Exception => e
return e
end
location = result['location'] && result['details']['locations'] && result['details']['locations'][0]
existing = supp.full_contact_info
info_data = {
full_name: result['fullName'],
gender: result['gender'],
city: location && location['city'],
state_code: location && location['regionCode'],
country: location && location['countryCode'],
age_range: result['ageRange'],
location_general: result['location'],
websites: ((result['details'] && result['details']['urls']) || []).map{|h| h['value']}.join(','),
supporter_id: supporter_id
}
if existing
full_contact_info = Qx.update(:full_contact_infos)
.set(info_data)
.timestamps
.where(id: existing['id'])
.returning('*')
.execute.first
else
full_contact_info = Qx.insert_into(:full_contact_infos)
.values(info_data)
.returning('*')
.timestamps
.execute.first
end
if result['details']['photos'].present?
photo_data = result['details']['photos'].map{|h| {type_id: h['label'], url: h['value']}}
Qx.delete_from("full_contact_photos")
.where(full_contact_info_id: full_contact_info['id'])
.execute
full_contact_photos = Qx.insert_into(:full_contact_photos)
.values(photo_data)
.common_values(full_contact_info_id: full_contact_info['id'])
.timestamps
.returning("*")
.execute
end
if result['details']['profiles'].present?
profile_data = result['details']['profiles'].map{|k,v| {type_id: v['service'], username: v['username'], uid: v['userid'], bio: v['bio'], url: v['url'], followers: v['followers'], following: v['following']} }
Qx.delete_from("full_contact_social_profiles")
.where(full_contact_info_id: full_contact_info['id'])
.execute
full_contact_social_profiles = Qx.insert_into(:full_contact_social_profiles)
.values(profile_data)
.common_values(full_contact_info_id: full_contact_info['id'])
.timestamps
.returning("*")
.execute
end
if result['details'].present? && result['details']['employment'].present?
Qx.delete_from('full_contact_orgs')
.where(full_contact_info_id: full_contact_info['id'])
.execute
org_data = result['details']['employment'].map{|h|
start_date = nil
end_date = nil
start_date = h['start'] && [h['start']['year'], h['start']['month'], h['start']['day']].select{|i| i.present?}.join('-')
end_date = h['end'] && [h['end']['year'], h['end']['month'], h['end']['day']].select{|i| i.present?}.join('-')
{
name: h['name'],
start_date: start_date,
end_date: end_date,
title: h['title'],
current: h['current']
} }
.map{|h| h[:end_date] = Format::Date.parse_partial_str(h[:end_date]); h}
.map{|h| h[:start_date] = Format::Date.parse_partial_str(h[:start_date]); h}
full_contact_orgs = Qx.insert_into(:full_contact_orgs)
.values(org_data)
.common_values(full_contact_info_id: full_contact_info['id'])
.timestamps
.returning('*')
.execute
end
return {
'full_contact_info' => full_contact_info,
'full_contact_photos' => full_contact_photos,
'full_contact_social_profiles' => full_contact_social_profiles,
'full_contact_orgs' => full_contact_orgs
}
end
# Delete all orphaned full contact infos that do not have supporters
# or full_contact photos, social_profiles, topics, orgs, etc that do not have a parent info
def self.cleanup_orphans
Info.includes(:supporter).where("supporters.id IS NULL").delete_all
Photo.includes(:full_contact_infos).where("full_contact_infos.id IS NULL").delete_all
SocialProfiles.includes(:full_contact_infos).where("full_contact_infos.id IS NULL").delete_all
Topics.includes(:full_contact_infos).where("full_contact_infos.id IS NULL").delete_all
Orgs.includes(:full_contact_infos).where("full_contact_infos.id IS NULL").delete_all
end
end

5
gems/full_contact/app/jobs/full_contact/application_job.rb → gems/houdini_full_contact/lib/houdini_full_contact.rb
View file

@ -1,7 +1,4 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module FullContact
class ApplicationJob < ActiveJob::Base
end
end
require_relative "./houdini"

4
gems/full_contact/lib/full_contact/version.rb → gems/houdini_full_contact/lib/houdini_full_contact/version.rb
View file

@ -1,6 +1,6 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
module FullContact
module HoudiniFullContact
VERSION = '0.1.0'
end
end

23
gems/houdini_full_contact/lib/tasks/houdini_full_contact_tasks.rake Normal file
View file

@ -0,0 +1,23 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
desc 'For generating Full Contact data'
# Clear old activerecord sessions tables daily
namespace :houdini do
namespace :full_contact do
task work_queue: :environment do
loop do
sleep(10) until Qx.select('COUNT(*)').from('full_contact_jobs').execute.first['count'] > 0
puts 'working...'
begin
Houdini::FullContact::InsertInfos.work_queue
rescue Exception => e
puts "Exception thrown: #{e}"
end
end
end
end
end

206
lib/insert/insert_full_contact_infos.rb
View file

@ -1,206 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
require 'qx'
module InsertFullContactInfos
# Work off of the full_contact_jobs queue
def self.work_queue
ids = Qx.select('supporter_id').from('full_contact_jobs').ex.map { |h| h['supporter_id'] }
Qx.delete_from('full_contact_jobs').where('TRUE').execute
bulk(ids) if ids.any?
end
# Enqueue full contact jobs for a set of supporter ids
def self.enqueue(supporter_ids)
Qx.insert_into(:full_contact_jobs)
.values(supporter_ids.map { |id| { supporter_id: id } })
.ex
end
# We need to throttle our requests by 10ms since that is our rate limit on FullContact
def self.bulk(supporter_ids)
created_ids = []
supporter_ids.each do |id|
now = Time.current
result = InsertFullContactInfos.single id
created_ids.push(GetData.hash(result, 'full_contact_info', 'id')) if result.is_a?(Hash)
interval = 0.1 - (Time.current - now) # account for time taken in .single
sleep interval if interval > 0
end
created_ids
end
# Fetch and persist a single full contact record for a single supporter
# return an exception if 404 or something else went poop
def self.single(supporter_id)
supp = Qx.select('email').from('supporters').where(id: supporter_id).execute.first
return if supp.nil? || supp['email'].blank?
begin
result = FullContact.person(email: supp['email']).to_h
rescue Exception => e
return e
end
if result['status'] == 202 # Queued for search
# self.enqueue([supporter_id])
return result
end
existing = Qx.select('id').from('full_contact_infos').where(supporter_id: supporter_id).ex.first
info_data = {
full_name: GetData.hash(result, 'contact_info', 'full_name'),
gender: GetData.hash(result, 'demographics', 'gender'),
city: GetData.hash(result, 'demographics', 'location_deduced', 'city', 'name'),
county: GetData.hash(result, 'demographics', 'location_deduced', 'county', 'name'),
state_code: GetData.hash(result, 'demographics', 'location_deduced', 'state', 'code'),
country: GetData.hash(result, 'demographics', 'location_deduced', 'country', 'name'),
continent: GetData.hash(result, 'demographics', 'location_deduced', 'continent', 'name'),
age: GetData.hash(result, 'demographics', 'age'),
age_range: GetData.hash(result, 'demographics', 'age_range'),
location_general: GetData.hash(result, 'demographics', 'location_general'),
websites: (GetData.hash(result, 'contact_info', 'websites') || []).map(&:url).join(','),
supporter_id: supporter_id
}
full_contact_info = if existing
Qx.update(:full_contact_infos)
.set(info_data)
.timestamps
.where(id: existing['id'])
.returning('*')
.execute.first
else
Qx.insert_into(:full_contact_infos)
.values(info_data)
.returning('*')
.timestamps
.execute.first
end
if result['photos'].present?
photo_data = result['photos'].map { |h| { type_id: h.type_id, url: h.url, is_primary: h.is_primary } }
Qx.delete_from('full_contact_photos')
.where(full_contact_info_id: full_contact_info['id'])
.execute
full_contact_photos = Qx.insert_into(:full_contact_photos)
.values(photo_data)
.common_values(full_contact_info_id: full_contact_info['id'])
.timestamps
.returning('*')
.execute
end
if result['social_profiles'].present?
profile_data = result['social_profiles'].map { |h| { type_id: h.type_id, username: h.username, uid: h.id, bio: h.bio, url: h.url, followers: h.followers, following: h.following } }
Qx.delete_from('full_contact_social_profiles')
.where(full_contact_info_id: full_contact_info['id'])
.execute
full_contact_social_profiles = Qx.insert_into(:full_contact_social_profiles)
.values(profile_data)
.common_values(full_contact_info_id: full_contact_info['id'])
.timestamps
.returning('*')
.execute
end
if result['digital_footprint'] && result['digital_footprint']['topics'].present?
profile_data = result['social_profiles']
.map do |h|
{
type_id: h.type_id,
username: h.username,
uid: h.id,
bio: h.bio,
url: h.url,
followers: h.followers,
following: h.following
}
end
vals = result['digital_footprint']['topics'].map(&:value)
existing_vals = Qx.select('value').from('full_contact_topics')
.where('value IN ($vals)', vals: vals)
.and_where('full_contact_info_id=$id', id: full_contact_info['id'])
.execute.map { |h| h['value'] }
topic_data = result['digital_footprint']['topics']
.reject { |h| existing_vals.include?(h.value) }
.map { |h| { value: h.value, provider: h.provider } }
if topic_data.any?
full_contact_topics = Qx.insert_into(:full_contact_topics)
.values(topic_data)
.common_values(full_contact_info_id: full_contact_info['id'])
.timestamps
.returning('*')
.execute
end
end
if result['organizations'].present?
Qx.delete_from('full_contact_orgs')
.where(full_contact_info_id: full_contact_info['id'])
.execute
org_data = result['organizations'].map do |h|
{
is_primary: h.is_primary,
name: h.name,
start_date: h.start_date,
end_date: h.end_date,
title: h.title,
current: h.current
}
end
.map { |h| h[:end_date] = Format::Date.parse_partial_str(h[:end_date]); h }
.map { |h| h[:start_date] = Format::Date.parse_partial_str(h[:start_date]); h }
full_contact_orgs = Qx.insert_into(:full_contact_orgs)
.values(org_data)
.common_values(full_contact_info_id: full_contact_info['id'])
.timestamps
.returning('*')
.execute
end
{
'full_contact_info' => full_contact_info,
'full_contact_photos' => full_contact_photos,
'full_contact_social_profiles' => full_contact_social_profiles,
'full_contact_topics' => full_contact_topics,
'full_contact_orgs' => full_contact_orgs
}
end
# Delete all orphaned full contact infos that do not have supporters
# or full_contact photos, social_profiles, topics, orgs, etc that do not have a parent info
def self.cleanup_orphans
Qx.delete_from('full_contact_infos')
.where('id IN ($ids)', ids: Qx.select('full_contact_infos.id')
.from('full_contact_infos')
.left_join('supporters', 'full_contact_infos.supporter_id=supporters.id')
.where('supporters.id IS NULL')).ex
Qx.delete_from('full_contact_photos')
.where('id IN ($ids)', ids: Qx.select('full_contact_photos.id')
.from('full_contact_photos')
.left_join('full_contact_infos', 'full_contact_infos.id=full_contact_photos.full_contact_info_id')
.where('full_contact_infos.id IS NULL')).ex
Qx.delete_from('full_contact_social_profiles')
.where('id IN ($ids)', ids: Qx.select('full_contact_social_profiles.id')
.from('full_contact_social_profiles')
.left_join('full_contact_infos', 'full_contact_infos.id=full_contact_social_profiles.full_contact_info_id')
.where('full_contact_infos.id IS NULL')).ex
Qx.delete_from('full_contact_topics')
.where('id IN ($ids)', ids: Qx.select('full_contact_topics.id')
.from('full_contact_topics')
.left_join('full_contact_infos', 'full_contact_infos.id=full_contact_topics.full_contact_info_id')
.where('full_contact_infos.id IS NULL')).ex
Qx.delete_from('full_contact_orgs')
.where('id IN ($ids)', ids: Qx.select('full_contact_orgs.id')
.from('full_contact_orgs')
.left_join('full_contact_infos', 'full_contact_infos.id=full_contact_orgs.full_contact_info_id')
.where('full_contact_infos.id IS NULL')).ex
end
end

3
lib/insert/insert_import.rb
View file

@ -7,7 +7,6 @@ require 'required_keys'
require 'open-uri'
require 'csv'
require 'insert/insert_supporter'
require 'insert/insert_full_contact_infos'
require 'insert/insert_custom_field_joins'
require 'insert/insert_tag_joins'
@ -164,7 +163,7 @@ module InsertImport
.where(id: import['id'])
.returning('*')
.execute.first
InsertFullContactInfos.enqueue(supporter_ids) if supporter_ids.any?
Houdini::FullContact::InsertInfos.enqueue(supporter_ids) if supporter_ids.any?
ImportCompletedJob.perform_later(Import.find(import['id']))
import
end

19
lib/tasks/full_contact.rake
View file

@ -1,19 +0,0 @@
# frozen_string_literal: true
# License: AGPL-3.0-or-later WITH WTO-AP-3.0-or-later
# Full license explanation at https://github.com/houdiniproject/houdini/blob/master/LICENSE
desc 'For generating Full Contact data'
# Clear old activerecord sessions tables daily
task work_full_contact_queue: :environment do
loop do
sleep(10) until Qx.select('COUNT(*)').from('full_contact_jobs').execute.first['count'] > 0
puts 'working...'
begin
InsertFullContactInfos.work_queue
rescue Exception => e
puts "Exception thrown: #{e}"
end
end
end