require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')

describe Robots do
  include HttpStubs
  before(:each) do
    setup_stubs
    @cobweb = Cobweb.new :quiet => true, :cache => nil
  end
  
  describe "default user-agent" do
    before(:each) do
      pending("thin not installed") unless THIN_INSTALLED
      @options = {:url => "http://localhost:3532/"}
    end
    
    it "should parse a valid robots.txt" do
      lambda {Robots.new(@options)}.should_not raise_error
    end
    
    it "should allow urls marked as allow" do
      robot = Robots.new(@options)
      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_true
    end
    it "should disallow urls specified as disallow" do
      robot = Robots.new(@options)
      robot.allowed?("http://localhost/globalmarketfinder/").should be_false
      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_false
    end
    it "should allow urls not listed" do
      robot = Robots.new(@options)
      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_true
    end
     
  end
  
  describe "google user-agent" do
    before(:each) do
      @options = {:url => "http://localhost/", :user_agent => "google"}
    end
    it "should parse a valid robots.txt" do
      lambda {Robots.new(@options)}.should_not raise_error
    end
    
    it "should disallow all urls" do
      robot = Robots.new(@options)
      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_false
      robot.allowed?("http://localhost/globalmarketfinder/").should be_false
      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_false
      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_false
    end
    
  end
    
  describe "cybermapper user-agent" do
    before(:each) do
      @options = {:url => "http://localhost/", :user_agent => "cybermapper"}
    end
    it "should parse a valid robots.txt" do
      lambda {Robots.new(@options)}.should_not raise_error
    end
    
    it "should disallow all urls" do
      robot = Robots.new(@options)
      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_true
      robot.allowed?("http://localhost/globalmarketfinder/").should be_true
      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_true
      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_true
    end
    
  end
    
end