#!/usr/bin/env ruby
# script to automate the new site discovery through by crawling all unique sites in the site store
require "wmap"

def print_usage
	puts "Program to crawl all unique sites within the site store, then update the store accordingly. \nUsage: spiderBot"
end

puts Wmap.banner
print_usage
Log_dir=File.dirname(__FILE__)+'/../logs/'
Wmap.wlog("Execute the command: spiderBot","spiderBot",Log_dir+"wmap.log")

abort "Incorrect program argument - no argument need! Proper Usage: spiderBot" unless ARGV.length==0
dis=Wmap::SiteTracker.instance
sites=dis.get_uniq_sites

crawler=Wmap::UrlCrawler.new(:max_parallel=>40)
crawler.crawls(sites)
v_sites=crawler.discovered_urls_by_crawler.keys.map {|x| crawler.url_2_site(x) }
v_sites.uniq!

dis.adds(v_sites)
dis.save!