module Picky module Backends # # class Redis < Backend attr_reader :client, :realtime def initialize options = {} maybe_load_hiredis check_hiredis_gem check_redis_gem @client = options[:client] || ::Redis.new(:db => (options[:db] || 15)) @realtime = options[:realtime] end def maybe_load_hiredis require 'hiredis' rescue LoadError # It's ok. end def check_hiredis_gem require 'redis/connection/hiredis' rescue LoadError # It's ok, the next check will fail if this one does. end def check_redis_gem require 'redis' rescue LoadError => e warn_gem_missing 'redis', 'the Redis client' end # Returns an object that on #initial, #load returns an object that responds to: # [:token] # => [id, id, id, id, id] (an array of ids) # def create_inverted bundle List.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:inverted", realtime: realtime end # Returns an object that on #initial, #load returns an object that responds to: # [:token] # => 1.23 (a weight) # def create_weights bundle Float.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:weights", realtime: realtime end # Returns an object that on #initial, #load returns an object that responds to: # [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to) # def create_similarity bundle List.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:similarity", realtime: realtime end # Returns an object that on #initial, #load returns an object that responds to: # [:key] # => value (a value for this config key) # def create_configuration bundle String.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:configuration", realtime: realtime end # Returns an object that on #initial, #load returns an object that responds to: # [id] # => [:sym1, :sym2] # def create_realtime bundle List.new client, "#{bundle.identifier}:realtime", realtime: realtime end # Does the Redis version already include # scripting support? # def redis_with_scripting? at_least_version redis_version, [2, 6, 0] end # Compares two versions each in an array [major, minor, patch] # format and returns true if the first version is higher # or the same as the second one. False if not. # # Note: Destructive. # def at_least_version major_minor_patch, should_be 3.times { return false if major_minor_patch.shift < should_be.shift } true end # Returns an array describing the # current Redis version. # # Note: This method assumes that clients answer # to #info with a hash (string/symbol keys) # detailing the infos. # # Example: # backend.redis_version # => [2, 4, 1] # def redis_version infos = client.info version_string = infos['redis_version'] || infos[:redis_version] version_string.split('.').map &:to_i end # Returns the total weight for the combinations. # def weight combinations # Note: A nice experiment that generated far too many strings. # # if redis_with_scripting? # @@weight_script = "local sum = 0; for i=1,#(KEYS),2 do local value = redis.call('hget', KEYS[i], KEYS[i+1]); if value then sum = sum + value end end return sum;" # # require 'digest/sha1' # @@weight_sent_once = nil # # # Scripting version of #ids. # # # class << self # def weight combinations # namespaces_keys = combinations.inject([]) do |namespaces_keys, combination| # namespaces_keys << "#{combination.bundle.identifier}:weights" # namespaces_keys << combination.token.text # end # # # Assume it's using EVALSHA. # # # begin # client.evalsha @@weight_sent_once, # namespaces_keys.size, # *namespaces_keys # rescue RuntimeError => e # # Make the server have a SHA-1 for the script. # # # @@weight_sent_once = Digest::SHA1.hexdigest @@weight_script # client.eval @@weight_script, # namespaces_keys.size, # *namespaces_keys # end # end # end # else # class << self # def weight combinations combinations.score # end # end # end # # Call the newly installed version. # # # weight combinations end # Returns the result ids for the allocation. # # Developers wanting to program fast intersection # routines, can do so analogue to this in their own # backend implementations. # # Note: We use the amount and offset hints to speed Redis up. # def ids combinations, amount, offset # TODO This is actually not correct: # A dumped/loaded Redis backend should use # the Redis backend calculation method. # So loaded? would be more appropriate. # if realtime # Just checked once on the first call. # if redis_with_scripting? @@ids_script = "local intersected = redis.call('zinterstore', ARGV[1], #(KEYS), unpack(KEYS)); if intersected == 0 then redis.call('del', ARGV[1]); return {}; end local results = redis.call('zrange', ARGV[1], tonumber(ARGV[2]), tonumber(ARGV[3])); redis.call('del', ARGV[1]); return results;" require 'digest/sha1' @@ids_sent_once = nil # Scripting version of #ids. # class << self def ids combinations, amount, offset identifiers = combinations.inject([]) do |identifiers, combination| identifiers << "#{combination.identifier}" end # Assume it's using EVALSHA. # begin if identifiers.size > 1 client.evalsha @@ids_sent_once, identifiers.size, *identifiers, generate_intermediate_result_id, offset, (offset + amount) else client.zrange identifiers.first, offset, (offset + amount) end rescue RuntimeError => e # Make the server have a SHA-1 for the script. # @@ids_sent_once = Digest::SHA1.hexdigest @@ids_script client.eval @@ids_script, identifiers.size, *identifiers, generate_intermediate_result_id, offset, (offset + amount) end end end else # Non-Scripting version of #ids. # class << self def ids combinations, amount, offset identifiers = combinations.inject([]) do |identifiers, combination| identifiers << "#{combination.identifier}" end result_id = generate_intermediate_result_id # Little optimization. # if identifiers.size > 1 # Intersect and store. # intersected = client.zinterstore result_id, identifiers # Return clean and early if there has been no intersection. # if intersected.zero? client.del result_id return [] end # Get the stored result. # results = client.zrange result_id, offset, (offset + amount) # Delete the stored result as it was only for temporary purposes. # # Note: I could also not delete it, but that # would not be clean at all. # client.del result_id else results = client.zrange identifiers.first, offset, (offset + amount) end results end end end else class << self def ids combinations, _, _ # Get the ids for each combination. # id_arrays = combinations.inject([]) do |total, combination| total << combination.ids end # Call the optimized C algorithm. # # Note: It orders the passed arrays by size. # Performant::Array.memory_efficient_intersect id_arrays end end end # Call the newly installed version. # ids combinations, amount, offset end # Generate a multiple host/process safe result id. # # Note: Generated when this class loads. # require 'socket' def self.extract_host @host ||= Socket.gethostname end def host self.class.extract_host end extract_host def pid @pid ||= Process.pid end # Use the host and pid (generated lazily in child processes) for the result. # def generate_intermediate_result_id @intermediate_result_id ||= "#{host}:#{pid}:picky:result" end end end end