Sha256: 2dd0b5097185371ab73d51c2aa3e2b911a8dac73adb1432945e03386d71d5cac
Contents?: true
Size: 777 Bytes
Versions: 34
Compression:
Stored size: 777 Bytes
Contents
#!/usr/bin/env ruby
# script to automate the new site discovery through by crawling all unique sites in the site store
require "wmap"
def print_usage
puts "Program to crawl all unique sites within the site store, then update the store accordingly. \nUsage: spiderBot"
end
puts Wmap.banner
print_usage
Log_dir=File.dirname(__FILE__)+'/../logs/'
Wmap.wlog("Execute the command: spiderBot","spiderBot",Log_dir+"wmap.log")
abort "Incorrect program argument - no argument need! Proper Usage: spiderBot" unless ARGV.length==0
dis=Wmap::SiteTracker.instance
sites=dis.get_uniq_sites
crawler=Wmap::UrlCrawler.new(:max_parallel=>40)
crawler.crawls(sites)
v_sites=crawler.discovered_urls_by_crawler.keys.map {|x| crawler.url_2_site(x) }
v_sites.uniq!
dis.adds(v_sites)
dis.save!
Version data entries
34 entries across 34 versions & 1 rubygems