Sha256: 82454d0a65803c6c41a0fbfc92c87ab6caf9cba065a04190f181c9ea61354b68
Contents?: true
Size: 1.6 KB
Versions: 15
Compression:
Stored size: 1.6 KB
Contents
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
describe CobwebCrawler do
before(:each) do
@base_url = "http://www.baseurl.com/"
@default_headers = {"Cache-Control" => "private, max-age=0",
"Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
"Expires" => "-1",
"Content-Type" => "text/html; charset=UTF-8",
"Content-Encoding" => "gzip",
"Transfer-Encoding" => "chunked",
"Server" => "gws",
"X-XSS-Protection" => "1; mode=block"}
end
describe "with mock" do
it "should generate a cobweb_crawler object" do
CobwebCrawler.new.should be_an_instance_of CobwebCrawler
end
describe "crawl" do
it "should crawl a site" do
# temporary tests to run crawler - proper specs to follow.. honest
crawler = CobwebCrawler.new({:cache => false, :quiet => false, :debug => false})
statistics = crawler.crawl("http://rockwellcottage.heroku.com/")
ap statistics
end
it "should take a block" do
# temporary tests to run crawler - proper specs to follow.. honest
crawler = CobwebCrawler.new({:cache => false, :quiet => false, :debug => false})
statistics = crawler.crawl("http://www.rockwellcottage.com/") do |content, statistics|
ap content[:url]
ap statistics[:average_length]
end
ap statistics
end
end
end
end
Version data entries
15 entries across 15 versions & 1 rubygems