app/models/maestrano/connector/rails/concerns/entity.rb in maestrano-connector-rails-0.4.4 vs app/models/maestrano/connector/rails/concerns/entity.rb in maestrano-connector-rails-1.0.0
- old
+ new
@@ -1,56 +1,40 @@
module Maestrano::Connector::Rails::Concerns::Entity
extend ActiveSupport::Concern
- module ClassMethods
- # Return an array of all the entities that the connector can synchronize
- # If you add new entities, you need to generate
- # a migration to add them to existing organizations
- def entities_list
- raise "Not implemented"
- end
+ def initialize(organization, connec_client, external_client, opts={})
+ @organization = organization
+ @connec_client = connec_client
+ @external_client = external_client
+ @opts = opts
+ end
+ module ClassMethods
# ----------------------------------------------
# IdMap methods
# ----------------------------------------------
def names_hash
{
connec_entity: connec_entity_name.downcase,
external_entity: external_entity_name.downcase
}
end
- def find_or_create_idmap(organization_and_id)
- Maestrano::Connector::Rails::IdMap.find_or_create_by(names_hash.merge(organization_and_id))
- end
-
# organization_and_id can be either:
# * {connec_id: 'id', organization_id: 'id'}
# * {external_id: 'id', organization_id: 'id'}
# Needs to include either connec_entity or external_entity for complex entities
+ def find_or_create_idmap(organization_and_id)
+ Maestrano::Connector::Rails::IdMap.find_or_create_by(names_hash.merge(organization_and_id))
+ end
def find_idmap(organization_and_id)
Maestrano::Connector::Rails::IdMap.find_by(names_hash.merge(organization_and_id))
end
-
- def create_idmap_from_external_entity(entity, organization)
- h = names_hash.merge({
- external_id: id_from_external_entity_hash(entity),
- name: object_name_from_external_entity_hash(entity),
- organization_id: organization.id
- })
- Maestrano::Connector::Rails::IdMap.create(h)
+ def create_idmap(organization_and_id)
+ Maestrano::Connector::Rails::IdMap.create(names_hash.merge(organization_and_id))
end
- def create_idmap_from_connec_entity(entity, organization)
- h = names_hash.merge({
- connec_id: entity['id'],
- name: object_name_from_connec_entity_hash(entity),
- organization_id: organization.id
- })
- Maestrano::Connector::Rails::IdMap.create(h)
- end
-
# ----------------------------------------------
# Connec! methods
# ----------------------------------------------
def normalized_connec_entity_name
normalize_connec_entity_name(connec_entity_name)
@@ -112,15 +96,20 @@
# Entity Mapper Class
def mapper_class
raise "Not implemented"
end
- # [{reference_class: Entities::.., connec_field: 'account_id', external_field: 'account/something/id'}]
+ # An array of connec fields that are references
def references
[]
end
+ # An array of fields for smart merging. See connec! documentation
+ def connec_matching_fields
+ nil
+ end
+
def can_read_connec?
can_write_external?
end
def can_read_external?
@@ -133,365 +122,378 @@
def can_write_external?
true
end
- def can_update_connec?
- true
- end
-
def can_update_external?
true
end
end
# ----------------------------------------------
# Mapper methods
# ----------------------------------------------
- # Map a Connec! entity to the external format
- def map_to_external(entity, organization)
- ref_hash = {}
- self.class.references.each do |ref|
- ref_hash.merge! ref[:external_field].split('/').reverse.inject(self.class.id_from_ref(entity, ref, false, organization)) { |a, n| { n.to_sym => a } }
- end
-
- self.class.mapper_class.normalize(entity).merge(ref_hash)
+ # Map a Connec! entity to the external model
+ def map_to_external(entity)
+ connec_id = entity[:__connec_id]
+ mapped_entity = self.class.mapper_class.normalize(entity)
+ (connec_id ? mapped_entity.merge(__connec_id: connec_id) : mapped_entity).with_indifferent_access
end
- # Map an external entity to Connec! format
- def map_to_connec(entity, organization)
- ref_hash = {}
- self.class.references.each do |ref|
- ref_hash.merge! ref[:connec_field].split('/').reverse.inject(self.class.id_from_ref(entity, ref, true, organization)) { |a, n| { n.to_sym => a } }
- end
-
- self.class.mapper_class.denormalize(entity).merge(ref_hash)
+ # Map an external entity to Connec! model
+ def map_to_connec(entity)
+ mapped_entity = self.class.mapper_class.denormalize(entity).merge(id: self.class.id_from_external_entity_hash(entity))
+ folded_entity = Maestrano::Connector::Rails::ConnecHelper.fold_references(mapped_entity, self.class.references, @organization)
+ folded_entity.merge!(opts: (mapped_entity[:opts] || {}).merge(matching_fields: self.class.connec_matching_fields)) if self.class.connec_matching_fields
+ folded_entity
end
# ----------------------------------------------
# Connec! methods
# ----------------------------------------------
# Supported options:
# * full_sync
# * $filter (see Connec! documentation)
# * $orderby (see Connec! documentation)
- def get_connec_entities(client, last_synchronization, organization, opts={})
- return [] unless self.class.can_read_connec?
+ def get_connec_entities(last_synchronization)
+ return [] if @opts[:skip_connec] || !self.class.can_read_connec?
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Fetching Connec! #{self.class.connec_entity_name}")
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching Connec! #{self.class.connec_entity_name}")
entities = []
query_params = {}
- query_params[:$orderby] = opts[:$orderby] if opts[:$orderby]
+ query_params[:$orderby] = @opts[:$orderby] if @opts[:$orderby]
# Fetch first page
- if last_synchronization.blank? || opts[:full_sync]
- Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "entity=#{self.class.connec_entity_name}, fetching all data")
- query_params[:$filter] = opts[:$filter] if opts[:$filter]
+ page_number = 0
+ if last_synchronization.blank? || @opts[:full_sync]
+ Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "entity=#{self.class.connec_entity_name}, fetching all data")
+ query_params[:$filter] = @opts[:$filter] if @opts[:$filter]
else
- Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "entity=#{self.class.connec_entity_name}, fetching data since #{last_synchronization.updated_at.iso8601}")
- filter = "updated_at gt '#{last_synchronization.updated_at.iso8601}'"
- filter += " and #{opts[:$filter]}" if opts[:$filter]
- query_params[:$filter] = filter
+ Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "entity=#{self.class.connec_entity_name}, fetching data since #{last_synchronization.updated_at.iso8601}")
+ query_params[:$filter] = "updated_at gt '#{last_synchronization.updated_at.iso8601}'" + (@opts[:$filter] ? " and #{@opts[:$filter]}" : '')
end
- response = client.get("/#{self.class.normalized_connec_entity_name}?#{query_params.to_query}")
- raise "No data received from Connec! when trying to fetch #{self.class.normalized_connec_entity_name}" unless response && !response.body.blank?
- response_hash = JSON.parse(response.body)
- Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "received first page entity=#{self.class.connec_entity_name}, response=#{response.body}")
- if response_hash["#{self.class.normalized_connec_entity_name}"]
- entities << response_hash["#{self.class.normalized_connec_entity_name}"]
- else
- raise "Received unrecognized Connec! data when trying to fetch #{self.class.normalized_connec_entity_name}"
- end
+ uri = "/#{self.class.normalized_connec_entity_name}?#{query_params.to_query}"
+ response_hash = fetch_connec(uri, 0)
+ entities = response_hash["#{self.class.normalized_connec_entity_name}"]
# Fetch subsequent pages
while response_hash['pagination'] && response_hash['pagination']['next']
+ page_number += 1
# ugly way to convert https://api-connec/api/v2/group_id/organizations?next_page_params to /organizations?next_page_params
next_page = response_hash['pagination']['next'].gsub(/^(.*)\/#{self.class.normalized_connec_entity_name}/, self.class.normalized_connec_entity_name)
- response = client.get(next_page)
- raise "No data received from Connec! when trying to fetch subsequent page of #{self.class.connec_entity_name.pluralize}" unless response && !response.body.blank?
- Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "received next page entity=#{self.class.connec_entity_name}, response=#{response.body}")
-
- response_hash = JSON.parse(response.body)
- if response_hash["#{self.class.normalized_connec_entity_name}"]
- entities << response_hash["#{self.class.normalized_connec_entity_name}"]
- else
- raise "Received unrecognized Connec! data when trying to fetch subsequent page of #{self.class.connec_entity_name.pluralize}"
- end
+ response_hash = fetch_connec(uri, page_number)
+ entities << response_hash["#{self.class.normalized_connec_entity_name}"]
end
- entities = entities.flatten
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Received data: Source=Connec!, Entity=#{self.class.connec_entity_name}, Data=#{entities}")
+ entities.flatten!
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Received data: Source=Connec!, Entity=#{self.class.connec_entity_name}, Data=#{entities}")
entities
end
- def push_entities_to_connec(connec_client, mapped_external_entities_with_idmaps, organization)
- push_entities_to_connec_to(connec_client, mapped_external_entities_with_idmaps, self.class.connec_entity_name, organization)
+ def push_entities_to_connec(mapped_external_entities_with_idmaps)
+ push_entities_to_connec_to(mapped_external_entities_with_idmaps, self.class.connec_entity_name)
end
- def push_entities_to_connec_to(connec_client, mapped_external_entities_with_idmaps, connec_entity_name, organization)
+ def push_entities_to_connec_to(mapped_external_entities_with_idmaps, connec_entity_name)
return unless self.class.can_write_connec?
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize} to Connec! #{connec_entity_name.pluralize}")
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize} to Connec! #{connec_entity_name.pluralize}")
- request_per_call = 100
- start = 0
- while start < mapped_external_entities_with_idmaps.size
- # Prepare batch request
- batch_entities = mapped_external_entities_with_idmaps.slice(start, request_per_call)
- batch_request = {sequential: true, ops: []}
- batch_entities.each do |mapped_external_entity_with_idmap|
- external_entity = mapped_external_entity_with_idmap[:entity]
- idmap = mapped_external_entity_with_idmap[:idmap]
- if idmap.connec_id.blank?
- batch_request[:ops] << batch_op('post', external_entity, nil, self.class.normalize_connec_entity_name(connec_entity_name), organization)
- else
- next unless self.class.can_update_connec?
- batch_request[:ops] << batch_op('put', external_entity, idmap.connec_id, self.class.normalize_connec_entity_name(connec_entity_name), organization)
- end
- end
-
- # Batch call
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending batch request to Connec! for #{self.class.normalize_connec_entity_name(connec_entity_name)}. Batch_request_size: #{batch_request[:ops].size}. Call_number: #{(start/request_per_call) + 1}")
- response = connec_client.batch(batch_request)
- Maestrano::Connector::Rails::ConnectorLogger.log('debug', organization, "Received batch response from Connec! for #{self.class.normalize_connec_entity_name(connec_entity_name)}: #{response}")
- raise "No data received from Connec! when trying to send batch request for #{self.class.connec_entity_name.pluralize}" unless response && !response.body.blank?
- response = JSON.parse(response.body)
-
- # Parse barch response
- response['results'].each_with_index do |result, index|
- if result['status'] == 200
- batch_entities[index][:idmap].update_attributes(last_push_to_connec: Time.now, message: nil)
- elsif result['status'] == 201
- batch_entities[index][:idmap].update_attributes(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'], last_push_to_connec: Time.now, message: nil)
- else
- Maestrano::Connector::Rails::ConnectorLogger.log('error', organization, "Error while pushing to Connec!: #{result['body']}")
- batch_entities[index][:idmap].update_attributes(message: result['body'].truncate(255))
- end
- end
- start += request_per_call
- end
+ proc = lambda{|mapped_external_entity_with_idmap| batch_op('post', mapped_external_entity_with_idmap[:entity], nil, self.class.normalize_connec_entity_name(connec_entity_name))}
+ batch_calls(mapped_external_entities_with_idmaps, proc, connec_entity_name)
end
- def batch_op(method, mapped_external_entity, id, connec_entity_name, organization)
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending #{method.upcase} #{connec_entity_name}: #{mapped_external_entity} to Connec! (Preparing batch request)")
+ def batch_op(method, mapped_external_entity, id, connec_entity_name)
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending #{method.upcase} #{connec_entity_name}: #{mapped_external_entity} to Connec! (Preparing batch request)")
{
method: method,
- url: "/api/v2/#{organization.uid}/#{connec_entity_name}" + (id.nil? ? '' : "/#{id}"),
+ url: "/api/v2/#{@organization.uid}/#{connec_entity_name}/#{id}", # id should be nil for POST
params: {
"#{connec_entity_name}".to_sym => mapped_external_entity
}
}
end
- def map_to_external_with_idmap(entity, organization)
- idmap = self.class.find_idmap({connec_id: entity['id'], organization_id: organization.id})
-
- if idmap
- return nil if idmap.external_inactive || !idmap.to_external
-
- if idmap.last_push_to_external && idmap.last_push_to_external > entity['updated_at']
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Discard Connec! #{self.class.connec_entity_name} : #{entity}")
- nil
- else
- idmap.update(name: self.class.object_name_from_connec_entity_hash(entity))
- {entity: map_to_external(entity, organization), idmap: idmap}
- end
- else
- {entity: map_to_external(entity, organization), idmap: self.class.create_idmap_from_connec_entity(entity, organization)}
- end
- end
-
# ----------------------------------------------
# External methods
# ----------------------------------------------
- def get_external_entities(client, last_synchronization, organization, opts={})
- return [] unless self.class.can_read_external?
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
+ def get_external_entities_wrapper(last_synchronization)
+ return [] if @opts[:skip_external] || !self.class.can_read_external?
+ get_external_entities(last_synchronization)
+ end
+
+ def get_external_entities(last_synchronization)
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Fetching #{Maestrano::Connector::Rails::External.external_name} #{self.class.external_entity_name.pluralize}")
raise "Not implemented"
end
- def push_entities_to_external(external_client, mapped_connec_entities_with_idmaps, organization)
- push_entities_to_external_to(external_client, mapped_connec_entities_with_idmaps, self.class.external_entity_name, organization)
+ def push_entities_to_external(mapped_connec_entities_with_idmaps)
+ push_entities_to_external_to(mapped_connec_entities_with_idmaps, self.class.external_entity_name)
end
- def push_entities_to_external_to(external_client, mapped_connec_entities_with_idmaps, external_entity_name, organization)
+ def push_entities_to_external_to(mapped_connec_entities_with_idmaps, external_entity_name)
return unless self.class.can_write_external?
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending Connec! #{self.class.connec_entity_name.pluralize} to #{Maestrano::Connector::Rails::External.external_name} #{external_entity_name.pluralize}")
- mapped_connec_entities_with_idmaps.each do |mapped_connec_entity_with_idmap|
- push_entity_to_external(external_client, mapped_connec_entity_with_idmap, external_entity_name, organization)
+
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending Connec! #{self.class.connec_entity_name.pluralize} to #{Maestrano::Connector::Rails::External.external_name} #{external_entity_name.pluralize}")
+ ids_to_send_to_connec = mapped_connec_entities_with_idmaps.map{ |mapped_connec_entity_with_idmap|
+ push_entity_to_external(mapped_connec_entity_with_idmap, external_entity_name)
+ }.compact
+
+ unless ids_to_send_to_connec.empty?
+ # Send the external ids to connec if it was a creation
+ proc = lambda{|id| batch_op('put', {id: [Maestrano::Connector::Rails::ConnecHelper.id_hash(id[:external_id], @organization)]}, id[:connec_id], self.class.normalize_connec_entity_name(self.class.connec_entity_name)) }
+ batch_calls(ids_to_send_to_connec, proc, self.class.connec_entity_name, true)
end
end
- def push_entity_to_external(external_client, mapped_connec_entity_with_idmap, external_entity_name, organization)
+
+ def push_entity_to_external(mapped_connec_entity_with_idmap, external_entity_name)
idmap = mapped_connec_entity_with_idmap[:idmap]
- connec_entity = mapped_connec_entity_with_idmap[:entity]
+ mapped_connec_entity = mapped_connec_entity_with_idmap[:entity]
begin
+ # Create and return id to send to connec!
if idmap.external_id.blank?
- external_id = create_external_entity(external_client, connec_entity, external_entity_name, organization)
- idmap.update_attributes(external_id: external_id, last_push_to_external: Time.now, message: nil)
+ connec_id = mapped_connec_entity_with_idmap[:idmap].connec_id
+ external_id = create_external_entity(mapped_connec_entity, external_entity_name)
+ idmap.update(external_id: external_id, last_push_to_external: Time.now, message: nil)
+ return {connec_id: connec_id, external_id: external_id, idmap: idmap}
+
+ # Update
else
return unless self.class.can_update_external?
- update_external_entity(external_client, connec_entity, idmap.external_id, external_entity_name, organization)
- idmap.update_attributes(last_push_to_external: Time.now, message: nil)
+ update_external_entity(mapped_connec_entity, idmap.external_id, external_entity_name)
+
+ # Return the id to send it to connec! if the first push of a singleton
+ if self.class.singleton? && idmap.last_push_to_external.nil?
+ connec_id = mapped_connec_entity_with_idmap[:idmap].connec_id
+ idmap.update(last_push_to_external: Time.now, message: nil)
+ return {connec_id: connec_id, external_id: idmap.external_id}
+ else
+ idmap.update(last_push_to_external: Time.now, message: nil)
+ end
+
end
rescue => e
# Store External error
- Maestrano::Connector::Rails::ConnectorLogger.log('error', organization, "Error while pushing to #{Maestrano::Connector::Rails::External.external_name}: #{e}")
- idmap.update_attributes(message: e.message.truncate(255))
+ Maestrano::Connector::Rails::ConnectorLogger.log('error', @organization, "Error while pushing to #{Maestrano::Connector::Rails::External.external_name}: #{e}")
+ idmap.update(message: e.message.truncate(255))
end
+ nil
end
- def create_external_entity(client, mapped_connec_entity, external_entity_name, organization)
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending create #{external_entity_name}: #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
+ def create_external_entity(mapped_connec_entity, external_entity_name)
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending create #{external_entity_name}: #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
raise "Not implemented"
end
- def update_external_entity(client, mapped_connec_entity, external_id, external_entity_name, organization)
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Sending update #{external_entity_name} (id=#{external_id}): #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
+ def update_external_entity(mapped_connec_entity, external_id, external_entity_name)
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending update #{external_entity_name} (id=#{external_id}): #{mapped_connec_entity} to #{Maestrano::Connector::Rails::External.external_name}")
raise "Not implemented"
end
# This method is called during the webhook workflow only. It should return the array of filtered entities
# The aim is to have the same filtering as with the Connec! filters on API calls in the webhooks
- def filter_connec_entities(entities, organization, opts={})
+ def filter_connec_entities(entities)
entities
end
+
# ----------------------------------------------
# General methods
# ----------------------------------------------
# * Discards entities that do not need to be pushed because they have not been updated since their last push
# * Discards entities from one of the two source in case of conflict
# * Maps not discarded entities and associates them with their idmap, or create one if there isn't any
- # * Return a hash {connec_entities: [], external_entities: []}
- def consolidate_and_map_data(connec_entities, external_entities, organization, opts={})
- return consolidate_and_map_singleton(connec_entities, external_entities, organization, opts) if self.class.singleton?
+ # * Returns a hash {connec_entities: [], external_entities: []}
+ def consolidate_and_map_data(connec_entities, external_entities)
+ return consolidate_and_map_singleton(connec_entities, external_entities) if self.class.singleton?
- mapped_external_entities = external_entities.map{|entity|
- idmap = self.class.find_idmap({external_id: self.class.id_from_external_entity_hash(entity), organization_id: organization.id})
+ mapped_connec_entities = consolidate_and_map_connec_entities(connec_entities, external_entities, self.class.references, self.class.external_entity_name)
+ mapped_external_entities = consolidate_and_map_external_entities(external_entities, self.class.connec_entity_name)
- # No idmap: creating one, nothing else to do
- unless idmap
- next {entity: map_to_connec(entity, organization), idmap: self.class.create_idmap_from_external_entity(entity, organization)}
+ return {connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
+ end
+
+ def consolidate_and_map_connec_entities(connec_entities, external_entities, references, external_entity_name)
+ connec_entities.map{|entity|
+ entity = Maestrano::Connector::Rails::ConnecHelper.unfold_references(entity, references, @organization)
+ next nil unless entity
+ connec_id = entity.delete(:__connec_id)
+
+ if entity['id'].blank?
+ idmap = self.class.create_idmap(organization_id: @organization.id, name: self.class.object_name_from_connec_entity_hash(entity), external_entity: external_entity_name.downcase, connec_id: connec_id)
+ next map_connec_entity_with_idmap(entity, external_entity_name, idmap)
end
+ idmap = self.class.find_or_create_idmap(external_id: entity['id'], organization_id: @organization.id, external_entity: external_entity_name.downcase, connec_id: connec_id)
+ idmap.update(name: self.class.object_name_from_connec_entity_hash(entity))
+
+ next nil if idmap.external_inactive || !idmap.to_external || (!@opts[:full_sync] && not_modified_since_last_push_to_external?(idmap, entity))
+
+ # Check for conflict with entities from external
+ solve_conflict(entity, external_entities, external_entity_name, idmap)
+ }.compact
+ end
+
+ def consolidate_and_map_external_entities(external_entities, connec_entity_name)
+ external_entities.map{|entity|
+ entity_id = self.class.id_from_external_entity_hash(entity)
+ idmap = self.class.find_or_create_idmap(external_id: entity_id, organization_id: @organization.id, connec_entity: connec_entity_name.downcase)
+
# Not pushing entity to Connec!
next nil unless idmap.to_connec
# Not pushing to Connec! and flagging as inactive if inactive in external application
inactive = self.class.inactive_from_external_entity_hash?(entity)
- idmap.update(external_inactive: inactive)
+ idmap.update(external_inactive: inactive, name: self.class.object_name_from_external_entity_hash(entity))
next nil if inactive
# Entity has not been modified since its last push to connec!
- next nil if self.class.not_modified_since_last_push_to_connec?(idmap, entity, self, organization)
+ next nil if !@opts[:full_sync] && not_modified_since_last_push_to_connec?(idmap, entity)
- idmap.update(name: self.class.object_name_from_external_entity_hash(entity))
- # Check for conflict with entities from connec!
- self.class.solve_conflict(entity, self, connec_entities, self.class.connec_entity_name, idmap, organization, opts)
+ map_external_entity_with_idmap(entity, connec_entity_name, idmap)
}.compact
-
- mapped_connec_entities = connec_entities.map{|entity|
- map_to_external_with_idmap(entity, organization)
- }.compact
-
- return {connec_entities: mapped_connec_entities, external_entities: mapped_external_entities}
end
- def consolidate_and_map_singleton(connec_entities, external_entities, organization, opts={})
+ def consolidate_and_map_singleton(connec_entities, external_entities)
return {connec_entities: [], external_entities: []} if external_entities.empty? && connec_entities.empty?
- idmap = self.class.find_or_create_idmap({organization_id: organization.id})
+ idmap = self.class.find_or_create_idmap({organization_id: @organization.id})
+ # No to_connec, to_external and inactive consideration here as we don't expect those workflow for singleton
if external_entities.empty?
keep_external = false
elsif connec_entities.empty?
keep_external = true
- elsif !opts[:connec_preemption].nil?
- keep_external = !opts[:connec_preemption]
+ elsif @opts.has_key?(:connec_preemption)
+ keep_external = !@opts[:connec_preemption]
else
- keep_external = self.class.is_external_more_recent?(connec_entities.first, external_entities.first, self)
+ keep_external = !is_connec_more_recent?(connec_entities.first, external_entities.first)
end
+
if keep_external
idmap.update(external_id: self.class.id_from_external_entity_hash(external_entities.first), name: self.class.object_name_from_external_entity_hash(external_entities.first))
- return {connec_entities: [], external_entities: [{entity: map_to_connec(external_entities.first, organization), idmap: idmap}]}
+ return {connec_entities: [], external_entities: [{entity: map_to_connec(external_entities.first), idmap: idmap}]}
else
- idmap.update(connec_id: connec_entities.first['id'], name: self.class.object_name_from_connec_entity_hash(connec_entities.first))
- return {connec_entities: [{entity: map_to_external(connec_entities.first, organization), idmap: idmap}], external_entities: []}
+ entity = Maestrano::Connector::Rails::ConnecHelper.unfold_references(connec_entities.first, self.class.references, @organization)
+ idmap.update(name: self.class.object_name_from_connec_entity_hash(entity), connec_id: entity.delete(:__connec_id))
+ idmap.update(external_id: self.class.id_from_external_entity_hash(external_entities.first)) unless external_entities.empty?
+ return {connec_entities: [{entity: map_to_external(entity), idmap: idmap}], external_entities: []}
end
end
# ----------------------------------------------
# After and before sync
# ----------------------------------------------
- def before_sync(connec_client, external_client, last_synchronization, organization, opts)
+ def before_sync(last_synchronization)
# Does nothing by default
end
- def after_sync(connec_client, external_client, last_synchronization, organization, opts)
+ def after_sync(last_synchronization)
# Does nothing by default
end
+
# ----------------------------------------------
# Internal helper methods
# ----------------------------------------------
- module ClassMethods
- def not_modified_since_last_push_to_connec?(idmap, entity, entity_instance, organization)
- result = idmap.last_push_to_connec && idmap.last_push_to_connec > entity_instance.class.last_update_date_from_external_entity_hash(entity)
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Discard #{Maestrano::Connector::Rails::External::external_name} #{entity_instance.class.external_entity_name} : #{entity}") if result
- result
+ private
+ # array_with_idmap must be an array of hashes with a key idmap
+ # proc is a lambda to create a batch_op from an element of the array
+ def batch_calls(array_with_idmap, proc, connec_entity_name, id_update_only=false)
+ request_per_call = @opts[:request_per_batch_call] || 100
+ start = 0
+ while start < array_with_idmap.size
+ # Prepare batch request
+ batch_entities = array_with_idmap.slice(start, request_per_call)
+ batch_request = {sequential: true, ops: []}
+
+ batch_entities.each do |id|
+ batch_request[:ops] << proc.call(id)
+ end
+
+ # Batch call
+ log_info = id_update_only ? 'with only ids' : ''
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Sending batch request to Connec! #{log_info} for #{self.class.normalize_connec_entity_name(connec_entity_name)}. Batch_request_size: #{batch_request[:ops].size}. Call_number: #{(start/request_per_call) + 1}")
+ response = @connec_client.batch(batch_request)
+ Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "Received batch response from Connec! for #{self.class.normalize_connec_entity_name(connec_entity_name)}: #{response}")
+ raise "No data received from Connec! when trying to send batch request #{log_info} for #{self.class.connec_entity_name.pluralize}" unless response && !response.body.blank?
+ response = JSON.parse(response.body)
+
+ # Parse batch response
+ response['results'].each_with_index do |result, index|
+ if result['status'] == 200
+ batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find{|id| id['provider'] == 'connec'}['id'], last_push_to_connec: Time.now, message: nil) unless id_update_only # id_update_only only apply for 200 as it's doing PUTs
+ elsif result['status'] == 201
+ batch_entities[index][:idmap].update(connec_id: result['body'][self.class.normalize_connec_entity_name(connec_entity_name)]['id'].find{|id| id['provider'] == 'connec'}['id'], last_push_to_connec: Time.now, message: nil)
+ else
+ Maestrano::Connector::Rails::ConnectorLogger.log('error', @organization, "Error while pushing to Connec!: #{result['body']}")
+ batch_entities[index][:idmap].update(message: result['body'].to_s.truncate(255))
+ end
+ end
+ start += request_per_call
+ end
end
+
+ def not_modified_since_last_push_to_connec?(idmap, entity)
+ not_modified = idmap.last_push_to_connec && idmap.last_push_to_connec > self.class.last_update_date_from_external_entity_hash(entity)
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Discard #{Maestrano::Connector::Rails::External::external_name} #{self.class.external_entity_name} : #{entity}") if not_modified
+ not_modified
+ end
- def is_external_more_recent?(connec_entity, external_entity, entity_instance)
- connec_entity['updated_at'] < entity_instance.class.last_update_date_from_external_entity_hash(external_entity)
+ def not_modified_since_last_push_to_external?(idmap, entity)
+ not_modified = idmap.last_push_to_external && idmap.last_push_to_external > entity['updated_at']
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Discard Connec! #{self.class.connec_entity_name} : #{entity}") if not_modified
+ not_modified
end
- def solve_conflict(external_entity, entity_instance, connec_entities, connec_entity_name, idmap, organization, opts)
- if idmap.connec_id && connec_entity = connec_entities.detect{|connec_entity| connec_entity['id'] == idmap.connec_id}
+ def is_connec_more_recent?(connec_entity, external_entity)
+ connec_entity['updated_at'] > self.class.last_update_date_from_external_entity_hash(external_entity)
+ end
+
+ def solve_conflict(connec_entity, external_entities, external_entity_name, idmap)
+ if external_entity = external_entities.find{|external_entity| connec_entity['id'] == self.class.id_from_external_entity_hash(external_entity)}
# We keep the most recently updated entity
- if !opts[:connec_preemption].nil?
- keep_external = !opts[:connec_preemption]
+ if @opts.has_key?(:connec_preemption)
+ keep_connec = @opts[:connec_preemption]
else
- keep_external = is_external_more_recent?(connec_entity, external_entity, entity_instance)
+ keep_connec = is_connec_more_recent?(connec_entity, external_entity)
end
- if keep_external
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{entity_instance.class.external_entity_name} #{external_entity} and Connec! #{connec_entity_name} #{connec_entity}. Entity from external kept")
- connec_entities.delete(connec_entity)
- entity_instance.map_external_entity_with_idmap(external_entity, connec_entity_name, idmap, organization)
+ if keep_connec
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from external kept")
+ external_entities.delete(external_entity)
+ map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
else
- Maestrano::Connector::Rails::ConnectorLogger.log('info', organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{entity_instance.class.external_entity_name} #{external_entity} and Connec! #{connec_entity_name} #{connec_entity}. Entity from Connec! kept")
+ Maestrano::Connector::Rails::ConnectorLogger.log('info', @organization, "Conflict between #{Maestrano::Connector::Rails::External::external_name} #{external_entity_name} #{external_entity} and Connec! #{self.class.connec_entity_name} #{connec_entity}. Entity from Connec! kept")
nil
end
else
- entity_instance.map_external_entity_with_idmap(external_entity, connec_entity_name, idmap, organization)
+ map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
end
end
- def id_from_ref(entity, ref, is_external, organization)
- # field can be address/billing/country_id
- field = is_external ? ref[:external_field] : ref[:connec_field]
- field = field.split('/')
- id = entity
- field.each do |f|
- id &&= id[f]
- end
+ def map_connec_entity_with_idmap(connec_entity, external_entity_name, idmap)
+ {entity: map_to_external(connec_entity), idmap: idmap}
+ end
- if is_external
- idmap = ref[:reference_class].find_idmap({external_id: id, organization_id: organization.id})
- idmap && idmap.connec_id
- else
- idmap = ref[:reference_class].find_idmap({connec_id: id, organization_id: organization.id})
- idmap && idmap.external_id
- end
+ def map_external_entity_with_idmap(external_entity, connec_entity_name, idmap)
+ {entity: map_to_connec(external_entity), idmap: idmap}
end
- end
- def map_external_entity_with_idmap(external_entity, connec_entity_name, idmap, organization)
- {entity: map_to_connec(external_entity, organization), idmap: idmap}
- end
+ def fetch_connec(uri, page_number)
+ response = @connec_client.get(uri)
+ raise "No data received from Connec! when trying to fetch page #{page_number} of #{self.class.normalized_connec_entity_name}" unless response && !response.body.blank?
+
+ response_hash = JSON.parse(response.body)
+ Maestrano::Connector::Rails::ConnectorLogger.log('debug', @organization, "received first page entity=#{self.class.connec_entity_name}, response=#{response_hash}")
+ raise "Received unrecognized Connec! data when trying to fetch page #{page_number} of #{self.class.normalized_connec_entity_name}: #{response_hash}" unless response_hash["#{self.class.normalized_connec_entity_name}"]
+
+ response_hash
+ end
+
end
\ No newline at end of file