Contents

#!/usr/bin/env ruby
require 'krawler'
require 'optparse'

options = {}
optparse = OptionParser.new do |opts|
  opts.banner = 'Usage: krawl [base url] [options]'

  opts.separator ''
  opts.separator 'Specific options:'

  opts.on('-e', '--exclude regex', 'Exclude matching paths') do |e|
    options[:e] = e
  end

  opts.on('-s', '--sub-restrict', 'Restrict to sub paths of base url', 'Default: false') do |s|
    options[:s] = true
  end

  opts.on('-c', '--concurrent count', 'Crawl with count number of concurrent connections', 'Default: 4') do |c|
    options[:c] = c.to_i
  end

  opts.on('-r', '--randomize', 'Randomize crawl path', 'Default: true') do |r|
    options[:r] = r
  end

  opts.separator ''

  opts.on('-h', '--help', 'Show this help message.') { puts opts; exit }

end
optparse.parse!

if ARGV.empty? || !(ARGV[0] =~ /^http/)
  puts optparse
  exit(-1)
end

Krawler::Base.new(ARGV[0] || 'http://localhost:3000/', {
  :exclude   => options[:e],
  :restrict  => options[:s],
  :threads   => options[:c],
  :randomize => options[:r]
}).base

Version data entries

7 entries across 7 versions & 1 rubygems

Version	Path
krawler-1.0.6	bin/krawl
krawler-1.0.5	bin/krawl
krawler-1.0.4	bin/krawl
krawler-1.0.3	bin/krawl
krawler-1.0.2	bin/krawl
krawler-1.0.1	bin/krawl
krawler-1.0.0	bin/krawl