spidr.rb in spidr-0.5.0

- old
+ new

@@ -4,14 +4,14 @@
   # Common proxy port.
   COMMON_PROXY_PORT = 8080
 
   # Default proxy information.
   DEFAULT_PROXY = {
-    :host => nil,
-    :port => COMMON_PROXY_PORT,
-    :user => nil,
-    :password => nil
+    host:     nil,
+    port:     COMMON_PROXY_PORT,
+    user:     nil,
+    password: nil
   }
 
   #
   # Proxy information used by all newly created Agent objects by default.
   #
@@ -42,11 +42,11 @@
   #
   # @return [Hash]
   #   The new proxy information.
   #
   def Spidr.proxy=(new_proxy)
-    @@spidr_proxy = {:port => COMMON_PROXY_PORT}.merge(new_proxy)
+    @@spidr_proxy = {port: COMMON_PROXY_PORT}.merge(new_proxy)
   end
 
   #
   # Disables the proxy settings used by all newly created Agent objects.
   #
@@ -74,10 +74,34 @@
   def Spidr.user_agent=(new_agent)
     @@spidr_user_agent = new_agent
   end
 
   #
+  # Specifies whether `robots.txt` should be honored globally.
+  #
+  # @return [Boolean]
+  #
+  # @since 0.5.0
+  #
+  def Spidr.robots?
+    @robots
+  end
+
+  #
+  # Enables or disables `robots.txt` globally.
+  #
+  # @param [Boolean] mode
+  #
+  # @return [Boolean]
+  #
+  # @since 0.5.0
+  #
+  def Spidr.robots=(mode)
+    @robots = mode
+  end
+
+  #
   # @see Agent.start_at
   #
   def Spidr.start_at(url,options={},&block)
     Agent.start_at(url,options,&block)
   end
@@ -92,7 +116,10 @@
   #
   # @see Agent.site
   #
   def Spidr.site(url,options={},&block)
     Agent.site(url,options,&block)
+  end
+
+  def Spidr.robots
   end
 end