lib/spidr/agent.rb in spidr-0.2.3 vs lib/spidr/agent.rb in spidr-0.2.4

- old
+ new

@@ -700,15 +700,16 @@ # # @return [Boolean] # Specifies whether the given URL should be visited. # def visit?(url) - (!(visited?(url)) && + !(visited?(url)) && visit_scheme?(url.scheme) && visit_host?(url.host) && visit_port?(url.port) && visit_link?(url.to_s) && - visit_ext?(url.path)) + visit_url?(url) && + visit_ext?(url.path) end # # Adds a given URL to the failures list. #