lib/ronin/web/web.rb in ronin-0.1.1 vs lib/ronin/web/web.rb in ronin-0.1.2

@@ -24,10 +24,11 @@
 require 'ronin/network/http'
 
 require 'uri/http'
 require 'mechanize'
 require 'open-uri'
+require 'spidr'
 
 module Ronin
   module Web
     #
     # Returns the default Ronin Web proxy port.
@@ -67,14 +68,16 @@
 
         if (Web.proxy[:user] || Web.proxy[:password])
           userinfo = "#{Web.proxy[:user]}:#{Web.proxy[:password]}"
         end
 
-        return URI::HTTP.build(:host => Web.proxy[:host],
-                               :port => Web.proxy[:port],
-                               :userinfo => userinfo,
-                               :path => '/')
+        return URI::HTTP.build(
+          :host => Web.proxy[:host],
+          :port => Web.proxy[:port],
+          :userinfo => userinfo,
+          :path => '/'
+        )
       end
     end
 
     #
     # Returns the supported Web User-Agent Aliases.
@@ -258,8 +261,71 @@
     def Web.post_body(url,options={},&block)
       body = Web.post(url,options).body
 
       block.call(body) if block
       return body
+    end
+
+    #
+    # Creates a new Spidr::Agent object with the given _options_ and
+    # _block_. If a _block_ is given, it will be passed the newly created
+    # Spidr::Agent object.
+    #
+    # _options_ may contain the following keys:
+    # <tt>:proxy</tt>:: The proxy to use while spidering. Defaults to
+    #                   Web.proxy.
+    # <tt>:user_agent</tt>:: The User-Agent string to send. Defaults to
+    #                        Web.user_agent.
+    # <tt>:referer</tt>:: The referer URL to send.
+    # <tt>:delay</tt>:: Duration in seconds to pause between spidering each
+    #                   link. Defaults to 0.
+    # <tt>:host</tt>:: The host-name to visit.
+    # <tt>:hosts</tt>:: An +Array+ of host patterns to visit.
+    # <tt>:ignore_hosts</tt>:: An +Array+ of host patterns to not visit.
+    # <tt>:ports</tt>:: An +Array+ of port patterns to visit.
+    # <tt>:ignore_ports</tt>:: An +Array+ of port patterns to not visit.
+    # <tt>:links</tt>:: An +Array+ of link patterns to visit.
+    # <tt>:ignore_links</tt>:: An +Array+ of link patterns to not visit.
+    # <tt>:exts</tt>:: An +Array+ of File extension patterns to visit.
+    # <tt>:ignore_exts</tt>:: An +Array+ of File extension patterns to not
+    #                         visit.
+    #
+    def Web.spider_agent(options={},&block)
+      options = Web.spider_default_options.merge(options)
+
+      return Spidr::Agent.new(options,&block)
+    end
+
+    #
+    # Creates a new Spidr::Agent object with the given _options_ and will
+    # begin spidering the specified host _name_. If a _block_ is given it
+    # will be passed the newly created Spidr::Agent object, before the
+    # agent begins spidering.
+    #
+    def Web.spider_host(name,options={},&block)
+      options = Web.spider_default_options.merge(options)
+
+      return Spidr::Agent.host(name,options,&block)
+    end
+
+    #
+    # Creates a new Spidr::Agent object with the given _options_ and will
+    # begin spidering the host of the specified _url_. If a _block_ is
+    # given it will be passed the newly created Spidr::Agent object, before
+    # the agent begins spidering.
+    #
+    def Web.spider_site(url,options={},&block)
+      options = Web.spider_default_options.merge(options)
+
+      return Spidr::Agent.site(url,options,&block)
+    end
+
+    protected
+
+    #
+    # Returns the default options for Spidr::Agent.
+    #
+    def Web.spider_default_options
+      {:proxy => Web.proxy, :user_agent => Web.user_agent}
     end
   end
 end