lib/redis_failover/client.rb in redis_failover-0.5.2 vs lib/redis_failover/client.rb in redis_failover-0.5.3

@@ -3,10 +3,11 @@
 module RedisFailover
   # Redis failover-aware client.
   class Client
     include Util
 
+    ZNODE_UPDATE_TIMEOUT = 9
     RETRY_WAIT_TIME = 3
     REDIS_READ_OPS = Set[
       :echo,
       :exists,
       :get,
@@ -63,11 +64,11 @@
 
     # Creates a new failover redis client.
     #
     # Options:
     #
-    #   :zkservers - comma-separated zookeeper host:port pairs (required)
+    #   :zkservers - comma-separated ZooKeeper host:port pairs (required)
     #   :znode_path - the Znode path override for redis server list (optional)
     #   :password - password for redis nodes (optional)
     #   :namespace - namespace for redis nodes (optional)
     #   :logger - logger override (optional)
     #   :retry_failure - indicate if failures should be retried (default true)
@@ -107,10 +108,11 @@
 
     private
 
     def setup_zookeeper_client
       @zkclient = ZkClient.new(@zkservers)
+      update_znode_timestamp
 
       # when session expires, purge client list
       @zkclient.on_session_expiration do
         purge_clients
       end
@@ -126,14 +128,16 @@
       end
 
       # register a watcher for future changes
       @zkclient.watcher.register(@znode) do |event|
         if event.node_created? || event.node_changed?
+          update_znode_timestamp
           build_clients
         elsif event.node_deleted?
-          @zkclient.stat(@znode, :watch => true)
+          update_znode_timestamp
           purge_clients
+          @zkclient.stat(@znode, :watch => true)
         else
           logger.error("Unknown ZK node event: #{event.inspect}")
         end
       end
     end
@@ -141,23 +145,27 @@
     def redis_operation?(method)
       Redis.public_instance_methods(false).include?(method)
     end
 
     def dispatch(method, *args, &block)
+      unless recently_heard_from_node_manager?
+        purge_clients
+        raise MissingNodeManagerError.new(ZNODE_UPDATE_TIMEOUT)
+      end
+
       verify_supported!(method)
       tries = 0
-
       begin
         if REDIS_READ_OPS.include?(method)
           # send read operations to a slave
           slave.send(method, *args, &block)
         else
           # direct everything else to master
           master.send(method, *args, &block)
         end
       rescue *CONNECTIVITY_ERRORS => ex
-        logger.error("Error while handling operation `#{method}` - #{ex.message}")
+        logger.error("Error while handling operation `#{method}` - #{ex.inspect}")
         logger.error(ex.backtrace.join("\n"))
 
         if tries < @max_retries
           tries += 1
           build_clients
@@ -200,11 +208,11 @@
           new_slaves = new_clients_for(*nodes[:slaves])
           @master = new_master
           @slaves = new_slaves
         rescue StandardError, *CONNECTIVITY_ERRORS => ex
           purge_clients
-          logger.error("Failed to fetch nodes from #{@zkservers} - #{ex.message}")
+          logger.error("Failed to fetch nodes from #{@zkservers} - #{ex.inspect}")
           logger.error(ex.backtrace.join("\n"))
 
           if tries < @max_retries
             tries += 1
             sleep(RETRY_WAIT_TIME) && retry
@@ -289,8 +297,17 @@
         logger.info("Purging current redis clients")
         disconnect(@master, *@slaves)
         @master = nil
         @slaves = []
       end
+    end
+
+    def update_znode_timestamp
+      @last_znode_timestamp = Time.now
+    end
+
+    def recently_heard_from_node_manager?
+      return false unless @last_znode_timestamp
+      Time.now - @last_znode_timestamp <= ZNODE_UPDATE_TIMEOUT
     end
   end
 end