spec/spidey/strategies/mongo_spec.rb in spidey-mongo-0.0.3 vs spec/spidey/strategies/mongo_spec.rb in spidey-mongo-0.0.4
- old
+ new
@@ -1,22 +1,29 @@
require 'spec_helper'
describe Spidey::Strategies::Mongo do
class TestSpider < Spidey::AbstractSpider
include Spidey::Strategies::Mongo
- set_result_key ->(data) { data[:detail_url] }
handle "http://www.cnn.com", :process_home
+
+ def result_key(data)
+ data[:detail_url]
+ end
end
before(:each) do
@db = Mongo::Connection.new['spidey-mongo-test']
@spider = TestSpider.new(
url_collection: @db['urls'],
result_collection: @db['results'],
error_collection: @db['errors'])
end
+ after(:each) do
+ %w{ urls results errors }.each{ |col| @db[col].drop }
+ end
+
it "should add initial URLs to collection" do
doc = @db['urls'].find_one(url: "http://www.cnn.com")
doc['handler'].should == :process_home
end
@@ -29,9 +36,15 @@
@spider.record detail_url: 'http://www.cnn.com', foo: 'bar'
@db['results'].count.should == 1
doc = @db['results'].find_one
doc['detail_url'].should == 'http://www.cnn.com'
doc['foo'].should == 'bar'
+ end
+
+ it "should update existing result" do
+ @db['results'].insert key: 'http://foo.bar', detail_url: 'http://foo.bar'
+ @spider.record detail_url: 'http://foo.bar', foo: 'bar'
+ @db['results'].count.should == 1
end
it "should add error" do
@spider.add_error error: Exception.new("WTF"), url: "http://www.cnn.com", handler: :blah
doc = @db['errors'].find_one
\ No newline at end of file