205 lines
8.9 KiB
Ruby
205 lines
8.9 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
# License: AGPL-3.0-or-later WITH Web-Template-Output-Additional-Permission-3.0-or-later
|
|
require 'qx'
|
|
|
|
module InsertFullContactInfos
|
|
# Work off of the full_contact_jobs queue
|
|
def self.work_queue
|
|
ids = Qx.select('supporter_id').from('full_contact_jobs').ex.map { |h| h['supporter_id'] }
|
|
Qx.delete_from('full_contact_jobs').where('TRUE').execute
|
|
bulk(ids) if ids.any?
|
|
end
|
|
|
|
# Enqueue full contact jobs for a set of supporter ids
|
|
def self.enqueue(supporter_ids)
|
|
Qx.insert_into(:full_contact_jobs)
|
|
.values(supporter_ids.map { |id| { supporter_id: id } })
|
|
.ex
|
|
end
|
|
|
|
# We need to throttle our requests by 10ms since that is our rate limit on FullContact
|
|
def self.bulk(supporter_ids)
|
|
created_ids = []
|
|
supporter_ids.each do |id|
|
|
now = Time.current
|
|
result = InsertFullContactInfos.single id
|
|
created_ids.push(GetData.hash(result, 'full_contact_info', 'id')) if result.is_a?(Hash)
|
|
interval = 0.1 - (Time.current - now) # account for time taken in .single
|
|
sleep interval if interval > 0
|
|
end
|
|
created_ids
|
|
end
|
|
|
|
# Fetch and persist a single full contact record for a single supporter
|
|
# return an exception if 404 or something else went poop
|
|
def self.single(supporter_id)
|
|
supp = Qx.select('email').from('supporters').where(id: supporter_id).execute.first
|
|
return if supp.nil? || supp['email'].blank?
|
|
|
|
begin
|
|
result = FullContact.person(email: supp['email']).to_h
|
|
rescue Exception => e
|
|
return e
|
|
end
|
|
|
|
if result['status'] == 202 # Queued for search
|
|
# self.enqueue([supporter_id])
|
|
return result
|
|
end
|
|
|
|
existing = Qx.select('id').from('full_contact_infos').where(supporter_id: supporter_id).ex.first
|
|
info_data = {
|
|
full_name: GetData.hash(result, 'contact_info', 'full_name'),
|
|
gender: GetData.hash(result, 'demographics', 'gender'),
|
|
city: GetData.hash(result, 'demographics', 'location_deduced', 'city', 'name'),
|
|
county: GetData.hash(result, 'demographics', 'location_deduced', 'county', 'name'),
|
|
state_code: GetData.hash(result, 'demographics', 'location_deduced', 'state', 'code'),
|
|
country: GetData.hash(result, 'demographics', 'location_deduced', 'country', 'name'),
|
|
continent: GetData.hash(result, 'demographics', 'location_deduced', 'continent', 'name'),
|
|
age: GetData.hash(result, 'demographics', 'age'),
|
|
age_range: GetData.hash(result, 'demographics', 'age_range'),
|
|
location_general: GetData.hash(result, 'demographics', 'location_general'),
|
|
websites: (GetData.hash(result, 'contact_info', 'websites') || []).map(&:url).join(','),
|
|
supporter_id: supporter_id
|
|
}
|
|
|
|
full_contact_info = if existing
|
|
Qx.update(:full_contact_infos)
|
|
.set(info_data)
|
|
.timestamps
|
|
.where(id: existing['id'])
|
|
.returning('*')
|
|
.execute.first
|
|
else
|
|
Qx.insert_into(:full_contact_infos)
|
|
.values(info_data)
|
|
.returning('*')
|
|
.timestamps
|
|
.execute.first
|
|
end
|
|
|
|
if result['photos'].present?
|
|
photo_data = result['photos'].map { |h| { type_id: h.type_id, url: h.url, is_primary: h.is_primary } }
|
|
Qx.delete_from('full_contact_photos')
|
|
.where(full_contact_info_id: full_contact_info['id'])
|
|
.execute
|
|
full_contact_photos = Qx.insert_into(:full_contact_photos)
|
|
.values(photo_data)
|
|
.common_values(full_contact_info_id: full_contact_info['id'])
|
|
.timestamps
|
|
.returning('*')
|
|
.execute
|
|
end
|
|
|
|
if result['social_profiles'].present?
|
|
profile_data = result['social_profiles'].map { |h| { type_id: h.type_id, username: h.username, uid: h.id, bio: h.bio, url: h.url, followers: h.followers, following: h.following } }
|
|
Qx.delete_from('full_contact_social_profiles')
|
|
.where(full_contact_info_id: full_contact_info['id'])
|
|
.execute
|
|
full_contact_social_profiles = Qx.insert_into(:full_contact_social_profiles)
|
|
.values(profile_data)
|
|
.common_values(full_contact_info_id: full_contact_info['id'])
|
|
.timestamps
|
|
.returning('*')
|
|
.execute
|
|
end
|
|
|
|
if result['digital_footprint'] && result['digital_footprint']['topics'].present?
|
|
profile_data = result['social_profiles']
|
|
.map do |h|
|
|
{
|
|
type_id: h.type_id,
|
|
username: h.username,
|
|
uid: h.id,
|
|
bio: h.bio,
|
|
url: h.url,
|
|
followers: h.followers,
|
|
following: h.following
|
|
}
|
|
end
|
|
|
|
vals = result['digital_footprint']['topics'].map(&:value)
|
|
existing_vals = Qx.select('value').from('full_contact_topics')
|
|
.where('value IN ($vals)', vals: vals)
|
|
.and_where('full_contact_info_id=$id', id: full_contact_info['id'])
|
|
.execute.map { |h| h['value'] }
|
|
|
|
topic_data = result['digital_footprint']['topics']
|
|
.reject { |h| existing_vals.include?(h.value) }
|
|
.map { |h| { value: h.value, provider: h.provider } }
|
|
|
|
if topic_data.any?
|
|
full_contact_topics = Qx.insert_into(:full_contact_topics)
|
|
.values(topic_data)
|
|
.common_values(full_contact_info_id: full_contact_info['id'])
|
|
.timestamps
|
|
.returning('*')
|
|
.execute
|
|
end
|
|
end
|
|
|
|
if result['organizations'].present?
|
|
Qx.delete_from('full_contact_orgs')
|
|
.where(full_contact_info_id: full_contact_info['id'])
|
|
.execute
|
|
org_data = result['organizations'].map do |h|
|
|
{
|
|
is_primary: h.is_primary,
|
|
name: h.name,
|
|
start_date: h.start_date,
|
|
end_date: h.end_date,
|
|
title: h.title,
|
|
current: h.current
|
|
}
|
|
end
|
|
.map { |h| h[:end_date] = Format::Date.parse_partial_str(h[:end_date]); h }
|
|
.map { |h| h[:start_date] = Format::Date.parse_partial_str(h[:start_date]); h }
|
|
|
|
full_contact_orgs = Qx.insert_into(:full_contact_orgs)
|
|
.values(org_data)
|
|
.common_values(full_contact_info_id: full_contact_info['id'])
|
|
.timestamps
|
|
.returning('*')
|
|
.execute
|
|
end
|
|
|
|
{
|
|
'full_contact_info' => full_contact_info,
|
|
'full_contact_photos' => full_contact_photos,
|
|
'full_contact_social_profiles' => full_contact_social_profiles,
|
|
'full_contact_topics' => full_contact_topics,
|
|
'full_contact_orgs' => full_contact_orgs
|
|
}
|
|
end
|
|
|
|
# Delete all orphaned full contact infos that do not have supporters
|
|
# or full_contact photos, social_profiles, topics, orgs, etc that do not have a parent info
|
|
def self.cleanup_orphans
|
|
Qx.delete_from('full_contact_infos')
|
|
.where('id IN ($ids)', ids: Qx.select('full_contact_infos.id')
|
|
.from('full_contact_infos')
|
|
.left_join('supporters', 'full_contact_infos.supporter_id=supporters.id')
|
|
.where('supporters.id IS NULL')).ex
|
|
Qx.delete_from('full_contact_photos')
|
|
.where('id IN ($ids)', ids: Qx.select('full_contact_photos.id')
|
|
.from('full_contact_photos')
|
|
.left_join('full_contact_infos', 'full_contact_infos.id=full_contact_photos.full_contact_info_id')
|
|
.where('full_contact_infos.id IS NULL')).ex
|
|
Qx.delete_from('full_contact_social_profiles')
|
|
.where('id IN ($ids)', ids: Qx.select('full_contact_social_profiles.id')
|
|
.from('full_contact_social_profiles')
|
|
.left_join('full_contact_infos', 'full_contact_infos.id=full_contact_social_profiles.full_contact_info_id')
|
|
.where('full_contact_infos.id IS NULL')).ex
|
|
Qx.delete_from('full_contact_topics')
|
|
.where('id IN ($ids)', ids: Qx.select('full_contact_topics.id')
|
|
.from('full_contact_topics')
|
|
.left_join('full_contact_infos', 'full_contact_infos.id=full_contact_topics.full_contact_info_id')
|
|
.where('full_contact_infos.id IS NULL')).ex
|
|
Qx.delete_from('full_contact_orgs')
|
|
.where('id IN ($ids)', ids: Qx.select('full_contact_orgs.id')
|
|
.from('full_contact_orgs')
|
|
.left_join('full_contact_infos', 'full_contact_infos.id=full_contact_orgs.full_contact_info_id')
|
|
.where('full_contact_infos.id IS NULL')).ex
|
|
end
|
|
end
|