lib/benchmark/http/spider.rb in benchmark-http-0.15.0 vs lib/benchmark/http/spider.rb in benchmark-http-0.15.1
- old
+ new
@@ -26,10 +26,11 @@
require 'async/http/client'
require 'async/http/endpoint'
require 'async/await'
require 'uri'
+require 'console'
module Benchmark
module HTTP
class Spider
include Async::Await
@@ -45,11 +46,11 @@
body = response.read
begin
filter = LinksFilter.parse(body)
rescue
- Async.logger.error(self) {$!}
+ Console.logger.error(self) {$!}
return []
end
if filter.base
base = base + filter.base
@@ -63,19 +64,19 @@
if full_url.host == url.host && full_url.kind_of?(URI::HTTP)
yield full_url
end
rescue ArgumentError, URI::InvalidURIError
- Async.logger.warn(self) {"Could not fetch #{href}, relative to #{base}!"}
+ Console.logger.warn(self) {"Could not fetch #{href}, relative to #{base}!"}
next # Don't accumulate an item into the resulting array.
end
end.compact
end
async def fetch(statistics, client, url, depth = @depth, fetched = Set.new, &block)
if depth&.zero?
- Async.logger.warn(self) {"Exceeded depth while trying to visit #{url}!"}
+ Console.logger.warn(self) {"Exceeded depth while trying to visit #{url}!"}
return
elsif fetched.include?(url)
return
elsif @ignore&.match?(url.path)
return
@@ -90,15 +91,15 @@
yield("HEAD", url, response) if block_given?
if response.redirection?
location = url + response.headers['location']
if location.host == url.host
- Async.logger.debug(self) {"Following redirect to #{location}..."}
+ Console.logger.debug(self) {"Following redirect to #{location}..."}
fetch(statistics, client, location, depth&.-(1), fetched, &block).wait
return
else
- Async.logger.debug(self) {"Ignoring redirect to #{location}."}
+ Console.logger.debug(self) {"Ignoring redirect to #{location}."}
return
end
end
content_type = response.headers['content-type']
@@ -115,12 +116,12 @@
extract_links(url, response) do |href|
fetch(statistics, client, href, depth&.-(1), fetched, &block)
end.each(&:wait)
rescue Async::TimeoutError
- Async.logger.error(self) {"Timeout while fetching #{url}"}
+ Console.logger.error(self) {"Timeout while fetching #{url}"}
rescue StandardError
- Async.logger.error(self) {$!}
+ Console.logger.error(self) {$!}
end
sync def call(urls, &block)
statistics = Statistics.new