Sha256: a19567745797cdcecb571047490cac7587f4a64d338d51810c79e28fb576f68b
Contents?: true
Size: 425 Bytes
Versions: 16
Compression:
Stored size: 425 Bytes
Contents
require 'rubygems'
require 'mechanize'
agent = Mechanize.new
stack = agent.get(ARGV[0]).links
while l = stack.pop
next unless l.uri
host = l.uri.host
next unless host.nil? or host == agent.history.first.uri.host
next if agent.visited? l.href
puts "crawling #{l.uri}"
begin
page = l.click
next unless Mechanize::Page === page
stack.push(*page.links)
rescue Mechanize::ResponseCodeError
end
end
Version data entries
16 entries across 16 versions & 5 rubygems