lib/benchmark/http/spider.rb in benchmark-http-0.15.0 vs lib/benchmark/http/spider.rb in benchmark-http-0.15.1

- old
+ new

@@ -26,10 +26,11 @@ require 'async/http/client' require 'async/http/endpoint' require 'async/await' require 'uri' +require 'console' module Benchmark module HTTP class Spider include Async::Await @@ -45,11 +46,11 @@ body = response.read begin filter = LinksFilter.parse(body) rescue - Async.logger.error(self) {$!} + Console.logger.error(self) {$!} return [] end if filter.base base = base + filter.base @@ -63,19 +64,19 @@ if full_url.host == url.host && full_url.kind_of?(URI::HTTP) yield full_url end rescue ArgumentError, URI::InvalidURIError - Async.logger.warn(self) {"Could not fetch #{href}, relative to #{base}!"} + Console.logger.warn(self) {"Could not fetch #{href}, relative to #{base}!"} next # Don't accumulate an item into the resulting array. end end.compact end async def fetch(statistics, client, url, depth = @depth, fetched = Set.new, &block) if depth&.zero? - Async.logger.warn(self) {"Exceeded depth while trying to visit #{url}!"} + Console.logger.warn(self) {"Exceeded depth while trying to visit #{url}!"} return elsif fetched.include?(url) return elsif @ignore&.match?(url.path) return @@ -90,15 +91,15 @@ yield("HEAD", url, response) if block_given? if response.redirection? location = url + response.headers['location'] if location.host == url.host - Async.logger.debug(self) {"Following redirect to #{location}..."} + Console.logger.debug(self) {"Following redirect to #{location}..."} fetch(statistics, client, location, depth&.-(1), fetched, &block).wait return else - Async.logger.debug(self) {"Ignoring redirect to #{location}."} + Console.logger.debug(self) {"Ignoring redirect to #{location}."} return end end content_type = response.headers['content-type'] @@ -115,12 +116,12 @@ extract_links(url, response) do |href| fetch(statistics, client, href, depth&.-(1), fetched, &block) end.each(&:wait) rescue Async::TimeoutError - Async.logger.error(self) {"Timeout while fetching #{url}"} + Console.logger.error(self) {"Timeout while fetching #{url}"} rescue StandardError - Async.logger.error(self) {$!} + Console.logger.error(self) {$!} end sync def call(urls, &block) statistics = Statistics.new