lib/spieker/link_scraper.rb in spieker-0.0.9 vs lib/spieker/link_scraper.rb in spieker-0.0.10

@@ -13,11 +13,12 @@
       @lang = lang
       Capybara.app_host = app_host
 
       Capybara.register_driver :tolq do |app|
         profile = Selenium::WebDriver::Firefox::Profile.new
-        profile['general.useragent.override'] = "Mozilla/5.0 (compatible; Tolq Spieker/#{Spieker::VERSION}; +http://www.tolq.com)"
+        # Sorry internet, but we need the whitelisting
+        profile['general.useragent.override'] = "Mozilla/5.0 (compatible; Googlebot TolqSpieker/#{Spieker::VERSION}; +http://www.tolq.com)"
 
         Capybara::Selenium::Driver.new(app, :profile => profile)
       end
 
       Capybara.current_driver = :tolq
@@ -54,10 +55,10 @@
                 end
 
         begin
           # Our javascript adds a class if the content has been succesfully submitted
           page.find(:css, 'html.tolq-content-updated')
-        rescue Capybara::Ambiguous, Capybara::ElementNotFound => e
+        rescue Capybara::Ambiguous, Capybara::ElementNotFound, Net::ReadTimeout => e
           puts "Something went wrong with submitting the content: #{e.message}"
         end
         links
       rescue => e
         puts "Error parsing #{@url.to_s}, #{e.message}"