#!/usr/bin/env ruby
# Wmap main executable - intelligent enough to handle most command argument inputs from the user.
# The discovery result is automatically compared and saved into the the tracking data repository.
#
# Usage: wmap -t <Target Host | URL | IP | CIDR | or a seed file with any of the above combo> -d <Optional Discovery Result Directory>
require "wmap"
require "optparse"

# program command line options
options = {:data_dir => nil, :target => nil, :verbose => false}
parser = OptionParser.new do|opts|
	opts.banner = Wmap.banner
	opts.on('-d', '--data_dir data_dir', 'Web Mapper local cache data directory') do |data_dir|
		options[:data_dir] = data_dir;
	end
	opts.on('-t', '--target target', 'Web Mapper target') do |target|
		options[:target] = target;
	end
	opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
		options[:verbose] = v;
	end
	opts.on('-h', '--help', 'Displays Help') do
		puts opts
		exit 0
	end
end
parser.parse!

# print program banner
puts Wmap.banner
# print_usage unless options[:target]

# Preparing - check out the working logs directory
if options[:data_dir]
	# Log to the instance running directory
	Log_dir = Pathname.new(options[:data_dir]).join('logs')
else
	# Log the command entry
	Log_dir=Pathname.new(Gem.loaded_specs['wmap'].full_gem_path).join('logs')
end
Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)

# Start wmap logging
Wmap.wlog("Execute the command: wmap -t #{options[:target]}","wmap",Log_dir.join("wmap.log").to_s)
urls = Array.new
# first step - construct the host list
scanner = Wmap::PortScanner.new(:verbose=>options[:verbose], :socket_timeout=>600) # default time-out of 600 milliseconds
hosts=Array.new
if File.exist?(options[:target])
	puts "Parsing the discovery seed file: \"#{options[:target]}\" "
	seeds=scanner.file_2_list(options[:target])-[nil,""]
	domains=Array.new
	cidrs=Array.new
	raise "Error: empty seed file or no legal entry found!" if seeds.nil? or seeds.empty?
	seeds.map do |x|
		x=x.split(%r{(,|\s+)})[0]
		urls.push(x) if scanner.is_url?(x)
		domains.push(x) if scanner.is_domain_root?(x) or Wmap.sub_domain_known?(x)
		# invoke bruter if the hostname contains a numeric number.
		domains.push(x) if scanner.is_fqdn?(x) and (x.split('.')[0] =~ /\d+/)
		hosts.push(x) if scanner.is_fqdn?(x) or scanner.is_ip?(x)
		cidrs.push(x) if scanner.is_cidr?(x)
	end
	puts "Parsing done. "
	hosts+=Wmap::DnsBruter.new(:verbose=>options[:verbose]).dns_brute_workers(domains.uniq).values.flatten if domains.size > 0
	cidrs.map { |x| hosts+= scanner.cidr_2_ips(x) } if cidrs.size > 0
elsif scanner.is_url?(options[:target])
	puts "Processing the URL: #{options[:target]}"
	urls.push(options[:target])
elsif Wmap.domain_known?(options[:target]) or Wmap.sub_domain_known?(options[:target])
	puts "Processing the domain: #{options[:target]}"
	hosts+=Wmap::DnsBruter.new(:verbose=>options[:verbose]).dns_brute_worker(options[:target]).values.flatten
elsif scanner.is_fqdn?(options[:target])
	puts "Processing the host: #{options[:target]}"
	hosts.push(options[:target])
	my_hosts=Wmap::DnsBruter.new(:verbose=>options[:verbose]).dns_brute_worker(options[:target]).values.flatten if (options[:target].split('.')[0] =~ /\d+/)
	hosts+=my_hosts unless my_hosts.nil?
elsif scanner.is_cidr?(options[:target])
	puts "Processing the network block: #{options[:target]}"
	hosts+=scanner.cidr_2_ips(options[:target])
elsif scanner.is_ip?(options[:target])
	hosts.push(options[:target])
else
	print_usage
end


# second step - port discovery on the above host list, and to build the URL seeds
puts "Build up URL list for the web crawler ..."
urls0=scanner.scans(hosts)
urls+=urls0
urls.uniq!
scanner=nil


# third step - crawling on the URL seeds
if options[:target] && options[:data_dir]
	puts "Fire up the crawler with the optional directory setter."
	crawler = Wmap::UrlCrawler.new(:data_dir => options[:data_dir])
elsif options[:target]
	puts "Fire up the crawler."
	crawler = Wmap::UrlCrawler.new(:verbose=>options[:verbose])
else
	abort "Error firing up UrlCrawler instance!"
end
Wmap.wlog(urls, "wmap", Log_dir+"url_seeds.log") if urls.size > 0   # save port scan results for debugging
crawler.crawls(urls) if urls.size>0
dis_urls=crawler.discovered_urls_by_crawler
#c_start=crawler.crawl_start
#c_done=crawler.crawl_done
dis_sites=Hash.new
unless dis_urls.empty?
	dis_urls.keys.map do |url|
		site=crawler.url_2_site(url)
		dis_sites[site]=true unless dis_sites.key?(site)
	end
end
puts "Discovered sites: "

if dis_sites.empty?
	puts "No web site is discovered. "
else
	dis_sites.keys.map {|x| puts x}
end


# fourth step - trace the discovery results into a local log file for debugging and other purposes
Wmap.wlog(dis_urls.keys, "wmap", Log_dir+"discovered_urls.log") unless dis_urls.empty?
Wmap.wlog(dis_sites.keys, "wmap", Log_dir+"discovered_sites.log") unless dis_sites.empty?
#crawler.wlog(c_start.keys,Log_dir+"crawler.log")
#crawler.wlog(c_done.keys,Log_dir+"crawler.log")
crawler=nil


# fifth step - save discovery results into the inventory data repository
case dis_sites.keys
when nil,[]
	puts "No new site found. There is no change to the site tracking data repository. "
else
	puts "Automatically save the discovery results into the site tracking data repository: "
	if options[:target] && options[:data_dir]
		puts "Start the SiteTracker with the optional directory setter. "
		inventory=Wmap::SiteTracker.instance
		inventory.data_dir = options[:data_dir]
		inventory.sites_file = inventory.data_dir + "/" + "sites"
		inventory.load_site_stores_from_file(inventory.sites_file)
	elsif options[:target]
		puts "Start the SiteTracker. "
		inventory=Wmap::SiteTracker.instance
	else
		abort "Error firing up SiteTracker instance!"
	end
	new_sites=inventory.adds(dis_sites.keys-["",nil])
	if new_sites.size>0 && options[:data_dir]
		inventory.save!(inventory.sites_file)
	elsif new_sites.size>0
		inventory.save!
	end
	inventory=nil
	puts "Done! New found sites are successfully saved. " if new_sites.size > 0
end


# seventh step - update the hosts repository
if options[:target] && options[:data_dir]
	puts "Invoke the HostTracker with optional directory setter."
	host_tracker = Wmap::HostTracker.instance
	host_tracker.verbose=options[:verbose]
	host_tracker.data_dir = options[:data_dir]
	host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
	host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
elsif options[:target]
	puts puts "Invoke the HostTracker."
	host_tracker = Wmap::HostTracker.instance
	host_tracker.verbose=options[:verbose]
else
	abort "Error firing up HostTracker instance!"
end
new_hosts = dis_sites.keys.map {|x| host_tracker.url_2_host(x)}
hosts += new_hosts
hosts.uniq!
if hosts.size > 0
	hostnames=hosts.dup.delete_if { |h| host_tracker.is_ip?(h) }
	if hostnames.size > 0
		puts "Update the local hosts data repository with: #{hostnames}"
		new_hosts=host_tracker.adds(hostnames-["",nil])
		if new_hosts.size>0 && options[:data_dir]
			host_tracker.save!(host_tracker.hosts_file)
		elsif new_hosts.size>0
			host_tracker.save!
		end
	end
end
host_tracker=nil