spec/spidey/strategies/mongo_spec.rb in spidey-mongo-0.1.0 vs spec/spidey/strategies/mongo_spec.rb in spidey-mongo-0.2.0
- old
+ new
@@ -1,61 +1,62 @@
require 'spec_helper'
+require 'mongo'
describe Spidey::Strategies::Mongo do
- class TestSpider < Spidey::AbstractSpider
+ class TestMongoSpider < Spidey::AbstractSpider
include Spidey::Strategies::Mongo
handle "http://www.cnn.com", :process_home
-
+
def result_key(data)
data[:detail_url]
end
end
-
+
before(:each) do
@db = Mongo::Connection.new['spidey-mongo-test']
- @spider = TestSpider.new(
+ @spider = TestMongoSpider.new(
url_collection: @db['urls'],
result_collection: @db['results'],
error_collection: @db['errors'])
end
-
+
after(:each) do
%w{ urls results errors }.each{ |col| @db[col].drop }
end
-
+
it "should add initial URLs to collection" do
doc = @db['urls'].find_one(url: "http://www.cnn.com")
doc['handler'].should == :process_home
- doc['spider'].should == 'TestSpider'
+ doc['spider'].should == 'TestMongoSpider'
end
-
+
it "should not add duplicate URLs" do
@spider.send :handle, "http://www.cnn.com", :process_home
@db['urls'].find(url: "http://www.cnn.com").count.should == 1
end
-
+
it "should add results" do
@spider.record detail_url: 'http://www.cnn.com', foo: 'bar'
@db['results'].count.should == 1
doc = @db['results'].find_one
doc['detail_url'].should == 'http://www.cnn.com'
doc['foo'].should == 'bar'
- doc['spider'].should == 'TestSpider'
+ doc['spider'].should == 'TestMongoSpider'
end
-
+
it "should update existing result" do
@db['results'].insert key: 'http://foo.bar', detail_url: 'http://foo.bar'
@spider.record detail_url: 'http://foo.bar', foo: 'bar'
@db['results'].count.should == 1
end
-
+
it "should add error" do
@spider.add_error error: Exception.new("WTF"), url: "http://www.cnn.com", handler: :blah
doc = @db['errors'].find_one
doc['error'].should == 'Exception'
doc['url'].should == 'http://www.cnn.com'
doc['handler'].should == :blah
doc['message'].should == 'WTF'
- doc['spider'].should == 'TestSpider'
+ doc['spider'].should == 'TestMongoSpider'
end
-
+
end
\ No newline at end of file