lib/oxidized/source/http.rb in oxidized-0.29.1 vs lib/oxidized/source/http.rb in oxidized-0.30.0

@@ -4,10 +4,11 @@
       @cfg = Oxidized.config.source.http
       super
     end
 
     def setup
+      Oxidized.setup_logger
       return unless @cfg.url.empty?
 
       raise NoConfig, 'no source http url config, edit ~/.config/oxidized/config'
     end
 
@@ -16,13 +17,18 @@
     require "uri"
     require "json"
 
     def load(node_want = nil)
       nodes = []
-      data = JSON.parse(read_http(node_want))
-      data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
-      data.each do |node|
+      uri = URI.parse(@cfg.url)
+      data = JSON.parse(read_http(uri, node_want))
+      node_data = data
+      node_data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
+      node_data = pagination(data, node_want) if @cfg.pagination?
+
+      # at this point we have all the nodes; pagination or not
+      node_data.each do |node|
         next if node.empty?
 
         # map node parameters
         keys = {}
         @cfg.map.each do |key, want_position|
@@ -54,11 +60,26 @@
         object = object[want] if object.respond_to? :each
       end
       object
     end
 
-    def read_http(node_want)
-      uri = URI.parse(@cfg.url)
+    def pagination(data, node_want)
+      node_data = []
+      raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name?
+
+      next_key = @cfg.pagination_key_name
+      loop do
+        node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
+        break if data[next_key].nil?
+
+        new_uri = URI.parse(data[next_key]) if data.has_key?(next_key)
+        data = JSON.parse(read_http(new_uri, node_want))
+        node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
+      end
+      node_data
+    end
+
+    def read_http(uri, node_want)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = true if uri.scheme == 'https'
       http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
 
       # Add read_timeout to handle case of big list of nodes (default value is 60 seconds)