--- !ruby/object:Gem::Specification
name: cobweb
version: !ruby/object:Gem::Version
  version: 1.0.15
platform: ruby
authors:
- Stewart McKee
autorequire: 
bindir: bin
cert_chain: []
date: 2013-08-19 00:00:00.000000000 Z
dependencies:
- !ruby/object:Gem::Dependency
  name: redis
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: nokogiri
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: addressable
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: rspec
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: awesome_print
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: sinatra
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: thin
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: haml
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: namespaced_redis
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: json
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
- !ruby/object:Gem::Dependency
  name: slop
  requirement: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
  type: :runtime
  prerelease: false
  version_requirements: !ruby/object:Gem::Requirement
    requirements:
    - - ! '>='
      - !ruby/object:Gem::Version
        version: '0'
description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
  crawl extremely large sites which is much more performant than multi-threaded crawlers.  It
  is also a standalone crawler that has a sophisticated statistics monitoring interface
  to monitor the progress of the crawls.
email: stewart@rockwellcottage.com
executables: []
extensions: []
extra_rdoc_files:
- README.textile
files:
- spec/cobweb/cobweb_crawl_helper_spec.rb
- spec/cobweb/cobweb_crawl_spec.rb
- spec/cobweb/cobweb_crawler_spec.rb
- spec/cobweb/cobweb_links_spec.rb
- spec/cobweb/cobweb_spec.rb
- spec/cobweb/content_link_parser_spec.rb
- spec/cobweb/crawl_job_spec.rb
- spec/cobweb/crawl_worker_spec.rb
- spec/cobweb/robots_spec.rb
- spec/http_stubs.rb
- spec/samples/robots.txt
- spec/samples/sample_html_links.html
- spec/samples/sample_server.rb
- spec/samples/sample_site/boxgrid>withsillyname.html
- spec/samples/sample_site/css/accordion.css
- spec/samples/sample_site/css/datatable.css
- spec/samples/sample_site/css/datepicker.css
- spec/samples/sample_site/css/form-buttons.css
- spec/samples/sample_site/css/forms.css
- spec/samples/sample_site/css/jquery.fancybox-1.3.4.css
- spec/samples/sample_site/css/jquery.treeview.css
- spec/samples/sample_site/css/link-buttons.css
- spec/samples/sample_site/css/login.css
- spec/samples/sample_site/css/menu.css
- spec/samples/sample_site/css/messages.css
- spec/samples/sample_site/css/modalbox.css
- spec/samples/sample_site/css/statics.css
- spec/samples/sample_site/css/style.css
- spec/samples/sample_site/css/style_text.css
- spec/samples/sample_site/css/tabs.css
- spec/samples/sample_site/css/wysiwyg-editor.css
- spec/samples/sample_site/css/wysiwyg.css
- spec/samples/sample_site/css/wysiwyg.modal.css
- spec/samples/sample_site/dashboard.html
- spec/samples/sample_site/forms.html
- spec/samples/sample_site/gallery.html
- spec/samples/sample_site/gfx/back-menu.gif
- spec/samples/sample_site/gfx/back-submenu.gif
- spec/samples/sample_site/gfx/background.gif
- spec/samples/sample_site/gfx/box-hide.png
- spec/samples/sample_site/gfx/box-search.png
- spec/samples/sample_site/gfx/box-title.gif
- spec/samples/sample_site/gfx/code.gif
- spec/samples/sample_site/gfx/datepicker-arrows.gif
- spec/samples/sample_site/gfx/fancybox/blank.gif
- spec/samples/sample_site/gfx/fancybox/fancy_close.png
- spec/samples/sample_site/gfx/fancybox/fancy_loading.png
- spec/samples/sample_site/gfx/fancybox/fancy_nav_left.png
- spec/samples/sample_site/gfx/fancybox/fancy_nav_right.png
- spec/samples/sample_site/gfx/fancybox/fancy_title_left.png
- spec/samples/sample_site/gfx/fancybox/fancy_title_main.png
- spec/samples/sample_site/gfx/fancybox/fancy_title_over.png
- spec/samples/sample_site/gfx/fancybox/fancy_title_right.png
- spec/samples/sample_site/gfx/fancybox/fancybox-x.png
- spec/samples/sample_site/gfx/fancybox/fancybox.png
- spec/samples/sample_site/gfx/forms/date-next.gif
- spec/samples/sample_site/gfx/forms/date-prev.gif
- spec/samples/sample_site/gfx/forms/forms-checkbox.gif
- spec/samples/sample_site/gfx/forms/forms-date.gif
- spec/samples/sample_site/gfx/forms/forms-file.gif
- spec/samples/sample_site/gfx/forms/forms-input-big.gif
- spec/samples/sample_site/gfx/forms/forms-input-medium.gif
- spec/samples/sample_site/gfx/forms/forms-input-small.gif
- spec/samples/sample_site/gfx/forms/forms-input-xl.gif
- spec/samples/sample_site/gfx/forms/forms-radio.gif
- spec/samples/sample_site/gfx/forms/forms-selectbox-small.gif
- spec/samples/sample_site/gfx/forms/forms-selectbox.gif
- spec/samples/sample_site/gfx/forms/forms-textarea-big.gif
- spec/samples/sample_site/gfx/forms/forms-textarea-medium.gif
- spec/samples/sample_site/gfx/forms/forms-textarea-small.gif
- spec/samples/sample_site/gfx/forms/forms-textarea-xl.gif
- spec/samples/sample_site/gfx/icon-delete.png
- spec/samples/sample_site/gfx/icon-edit.png
- spec/samples/sample_site/gfx/icon-home.gif
- spec/samples/sample_site/gfx/img-delete.png
- spec/samples/sample_site/gfx/img-hover.png
- spec/samples/sample_site/gfx/img-zoom.png
- spec/samples/sample_site/gfx/jquery.wysiwyg.gif
- spec/samples/sample_site/gfx/label-icons.gif
- spec/samples/sample_site/gfx/label.gif
- spec/samples/sample_site/gfx/li-down.gif
- spec/samples/sample_site/gfx/li.gif
- spec/samples/sample_site/gfx/link-button-big.gif
- spec/samples/sample_site/gfx/link-button-medium.gif
- spec/samples/sample_site/gfx/link-button.gif
- spec/samples/sample_site/gfx/loading-2.gif
- spec/samples/sample_site/gfx/loading.gif
- spec/samples/sample_site/gfx/logo.png
- spec/samples/sample_site/gfx/modal-title.gif
- spec/samples/sample_site/gfx/photos/00.jpg
- spec/samples/sample_site/gfx/photos/01.jpg
- spec/samples/sample_site/gfx/photos/01xl.jpg
- spec/samples/sample_site/gfx/photos/02.jpg
- spec/samples/sample_site/gfx/photos/02xl.jpg
- spec/samples/sample_site/gfx/photos/03.jpg
- spec/samples/sample_site/gfx/photos/03xl.jpg
- spec/samples/sample_site/gfx/photos/04.jpg
- spec/samples/sample_site/gfx/photos/04xl.jpg
- spec/samples/sample_site/gfx/photos/05.jpg
- spec/samples/sample_site/gfx/photos/05xl.jpg
- spec/samples/sample_site/gfx/photos/06.jpg
- spec/samples/sample_site/gfx/photos/06xl.jpg
- spec/samples/sample_site/gfx/photos/07.jpg
- spec/samples/sample_site/gfx/photos/07xl.jpg
- spec/samples/sample_site/gfx/photos/08.jpg
- spec/samples/sample_site/gfx/photos/08xl.jpg
- spec/samples/sample_site/gfx/photos/09.jpg
- spec/samples/sample_site/gfx/photos/09xl.jpg
- spec/samples/sample_site/gfx/photos/10.jpg
- spec/samples/sample_site/gfx/photos/10xl.jpg
- spec/samples/sample_site/gfx/photos/11.jpg
- spec/samples/sample_site/gfx/photos/11xl.jpg
- spec/samples/sample_site/gfx/photos/12.jpg
- spec/samples/sample_site/gfx/photos/12xl.jpg
- spec/samples/sample_site/gfx/photos/13.jpg
- spec/samples/sample_site/gfx/photos/13xl.jpg
- spec/samples/sample_site/gfx/photos/14.jpg
- spec/samples/sample_site/gfx/photos/14xl.jpg
- spec/samples/sample_site/gfx/photos/15.jpg
- spec/samples/sample_site/gfx/photos/15xl.jpg
- spec/samples/sample_site/gfx/search-button.gif
- spec/samples/sample_site/gfx/search-input.gif
- spec/samples/sample_site/gfx/slider-button.gif
- spec/samples/sample_site/gfx/system-messages.gif
- spec/samples/sample_site/gfx/table-asc-arrow.gif
- spec/samples/sample_site/gfx/table-desc-arrow.gif
- spec/samples/sample_site/gfx/table-first.gif
- spec/samples/sample_site/gfx/table-last.gif
- spec/samples/sample_site/gfx/table-next.gif
- spec/samples/sample_site/gfx/table-number.gif
- spec/samples/sample_site/gfx/table-prev.gif
- spec/samples/sample_site/gfx/table-rows.gif
- spec/samples/sample_site/gfx/table-search.gif
- spec/samples/sample_site/gfx/table-thead.gif
- spec/samples/sample_site/gfx/tooltip.gif
- spec/samples/sample_site/gfx/treeview/ajax-loader.gif
- spec/samples/sample_site/gfx/treeview/file.gif
- spec/samples/sample_site/gfx/treeview/folder-closed.gif
- spec/samples/sample_site/gfx/treeview/folder.gif
- spec/samples/sample_site/gfx/treeview/minus.gif
- spec/samples/sample_site/gfx/treeview/plus.gif
- spec/samples/sample_site/gfx/treeview/treeview-default-line.gif
- spec/samples/sample_site/gfx/treeview/treeview-default.gif
- spec/samples/sample_site/index.html
- spec/samples/sample_site/js/controls/wysiwyg.image.js
- spec/samples/sample_site/js/controls/wysiwyg.link.js
- spec/samples/sample_site/js/controls/wysiwyg.table.js
- spec/samples/sample_site/js/customInput.jquery.js
- spec/samples/sample_site/js/excanvas.min.js
- spec/samples/sample_site/js/hoverIntent.js
- spec/samples/sample_site/js/inline.js
- spec/samples/sample_site/js/jquery-1.7.1.min.js
- spec/samples/sample_site/js/jquery-ui-select.js
- spec/samples/sample_site/js/jquery-ui-timepicker-addon.js
- spec/samples/sample_site/js/jquery-ui.js
- spec/samples/sample_site/js/jquery.dataTables.js
- spec/samples/sample_site/js/jquery.fancybox-1.3.4.js
- spec/samples/sample_site/js/jquery.filestyle.mini.js
- spec/samples/sample_site/js/jquery.flot.js
- spec/samples/sample_site/js/jquery.flot.resize.min.js
- spec/samples/sample_site/js/jquery.graphtable-0.2.js
- spec/samples/sample_site/js/jquery.tipsy.js
- spec/samples/sample_site/js/jquery.treeview.js
- spec/samples/sample_site/js/jquery.wysiwyg.js
- spec/samples/sample_site/js/plugins/wysiwyg.rmFormat.js
- spec/samples/sample_site/js/superfish.js
- spec/samples/sample_site/js/supersubs.js
- spec/samples/sample_site/more.html
- spec/samples/sample_site/robots.txt
- spec/samples/sample_site/tables.html
- spec/samples/sample_site/text/museosans-webfont.eot
- spec/samples/sample_site/text/museosans-webfont.svg
- spec/samples/sample_site/text/museosans-webfont.ttf
- spec/samples/sample_site/text/museosans-webfont.woff
- spec/samples/sample_site/text/museosans_bold-webfont.eot
- spec/samples/sample_site/text/museosans_bold-webfont.svg
- spec/samples/sample_site/text/museosans_bold-webfont.ttf
- spec/samples/sample_site/text/museosans_bold-webfont.woff
- spec/samples/sample_site/typography.html
- spec/spec.opts
- spec/spec_helper.rb
- lib/cobweb.rb
- lib/cobweb_crawl_helper.rb
- lib/cobweb_crawler.rb
- lib/cobweb_dsl.rb
- lib/cobweb_finished_job.rb
- lib/cobweb_links.rb
- lib/cobweb_process_job.rb
- lib/cobweb_version.rb
- lib/content_link_parser.rb
- lib/crawl.rb
- lib/crawl_finished_worker.rb
- lib/crawl_helper.rb
- lib/crawl_job.rb
- lib/crawl_object.rb
- lib/crawl_process_worker.rb
- lib/crawl_worker.rb
- lib/document.rb
- lib/encoding_safe_process_job.rb
- lib/export_command.rb
- lib/hash_util.rb
- lib/redirect_error.rb
- lib/report_command.rb
- lib/robots.rb
- lib/server.rb
- lib/sidekiq/cobweb_helper.rb
- lib/stats.rb
- lib/string.rb
- lib/uri_helper.rb
- views/home.haml
- views/layout.haml
- views/statistics.haml
- public/css/accordion.css
- public/css/custom.css
- public/css/datatable.css
- public/css/datepicker.css
- public/css/form-buttons.css
- public/css/forms.css
- public/css/jquery.fancybox-1.3.4.css
- public/css/jquery.treeview.css
- public/css/link-buttons.css
- public/css/login.css
- public/css/menu.css
- public/css/messages.css
- public/css/modalbox.css
- public/css/statics.css
- public/css/style.css
- public/css/style_text.css
- public/css/tabs.css
- public/css/wysiwyg-editor.css
- public/css/wysiwyg.css
- public/css/wysiwyg.modal.css
- public/gfx/back-menu.gif
- public/gfx/back-submenu.gif
- public/gfx/background.gif
- public/gfx/box-hide.png
- public/gfx/box-search.png
- public/gfx/box-title.gif
- public/gfx/code.gif
- public/gfx/datepicker-arrows.gif
- public/gfx/fancybox/blank.gif
- public/gfx/fancybox/fancy_close.png
- public/gfx/fancybox/fancy_loading.png
- public/gfx/fancybox/fancy_nav_left.png
- public/gfx/fancybox/fancy_nav_right.png
- public/gfx/fancybox/fancy_title_left.png
- public/gfx/fancybox/fancy_title_main.png
- public/gfx/fancybox/fancy_title_over.png
- public/gfx/fancybox/fancy_title_right.png
- public/gfx/fancybox/fancybox-x.png
- public/gfx/fancybox/fancybox.png
- public/gfx/forms/date-next.gif
- public/gfx/forms/date-prev.gif
- public/gfx/forms/forms-checkbox.gif
- public/gfx/forms/forms-date.gif
- public/gfx/forms/forms-file.gif
- public/gfx/forms/forms-input-big.gif
- public/gfx/forms/forms-input-medium.gif
- public/gfx/forms/forms-input-small.gif
- public/gfx/forms/forms-input-xl.gif
- public/gfx/forms/forms-radio.gif
- public/gfx/forms/forms-selectbox-small.gif
- public/gfx/forms/forms-selectbox.gif
- public/gfx/forms/forms-textarea-big.gif
- public/gfx/forms/forms-textarea-medium.gif
- public/gfx/forms/forms-textarea-small.gif
- public/gfx/forms/forms-textarea-xl.gif
- public/gfx/icon-delete.png
- public/gfx/icon-edit.png
- public/gfx/icon-home.gif
- public/gfx/img-delete.png
- public/gfx/img-hover.png
- public/gfx/img-zoom.png
- public/gfx/jquery.wysiwyg.gif
- public/gfx/label-icons.gif
- public/gfx/label.gif
- public/gfx/li-down.gif
- public/gfx/li.gif
- public/gfx/link-button-big.gif
- public/gfx/link-button-medium.gif
- public/gfx/link-button.gif
- public/gfx/loading-2.gif
- public/gfx/loading.gif
- public/gfx/logo.png
- public/gfx/modal-title.gif
- public/gfx/photos/00.jpg
- public/gfx/photos/01.jpg
- public/gfx/photos/01xl.jpg
- public/gfx/photos/02.jpg
- public/gfx/photos/02xl.jpg
- public/gfx/photos/03.jpg
- public/gfx/photos/03xl.jpg
- public/gfx/photos/04.jpg
- public/gfx/photos/04xl.jpg
- public/gfx/photos/05.jpg
- public/gfx/photos/05xl.jpg
- public/gfx/photos/06.jpg
- public/gfx/photos/06xl.jpg
- public/gfx/photos/07.jpg
- public/gfx/photos/07xl.jpg
- public/gfx/photos/08.jpg
- public/gfx/photos/08xl.jpg
- public/gfx/photos/09.jpg
- public/gfx/photos/09xl.jpg
- public/gfx/photos/10.jpg
- public/gfx/photos/10xl.jpg
- public/gfx/photos/11.jpg
- public/gfx/photos/11xl.jpg
- public/gfx/photos/12.jpg
- public/gfx/photos/12xl.jpg
- public/gfx/photos/13.jpg
- public/gfx/photos/13xl.jpg
- public/gfx/photos/14.jpg
- public/gfx/photos/14xl.jpg
- public/gfx/photos/15.jpg
- public/gfx/photos/15xl.jpg
- public/gfx/search-button.gif
- public/gfx/search-input.gif
- public/gfx/slider-button.gif
- public/gfx/system-messages.gif
- public/gfx/table-asc-arrow.gif
- public/gfx/table-desc-arrow.gif
- public/gfx/table-first.gif
- public/gfx/table-last.gif
- public/gfx/table-next.gif
- public/gfx/table-number.gif
- public/gfx/table-prev.gif
- public/gfx/table-rows.gif
- public/gfx/table-search.gif
- public/gfx/table-thead.gif
- public/gfx/tooltip.gif
- public/gfx/treeview/ajax-loader.gif
- public/gfx/treeview/file.gif
- public/gfx/treeview/folder-closed.gif
- public/gfx/treeview/folder.gif
- public/gfx/treeview/minus.gif
- public/gfx/treeview/plus.gif
- public/gfx/treeview/treeview-default-line.gif
- public/gfx/treeview/treeview-default.gif
- public/js/controls/wysiwyg.image.js
- public/js/controls/wysiwyg.link.js
- public/js/controls/wysiwyg.table.js
- public/js/customInput.jquery.js
- public/js/excanvas.min.js
- public/js/hoverIntent.js
- public/js/inline.js
- public/js/jquery-1.7.1.min.js
- public/js/jquery-ui-select.js
- public/js/jquery-ui-timepicker-addon.js
- public/js/jquery-ui.js
- public/js/jquery.dataTables.js
- public/js/jquery.fancybox-1.3.4.js
- public/js/jquery.filestyle.mini.js
- public/js/jquery.flot.js
- public/js/jquery.flot.resize.min.js
- public/js/jquery.graphtable-0.2.js
- public/js/jquery.tipsy.js
- public/js/jquery.treeview.js
- public/js/jquery.wysiwyg.js
- public/js/plugins/wysiwyg.rmFormat.js
- public/js/superfish.js
- public/js/supersubs.js
- README.textile
homepage: http://github.com/stewartmckee/cobweb
licenses: []
metadata: {}
post_install_message: 
rdoc_options: []
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
  requirements:
  - - ! '>='
    - !ruby/object:Gem::Version
      version: '0'
required_rubygems_version: !ruby/object:Gem::Requirement
  requirements:
  - - ! '>='
    - !ruby/object:Gem::Version
      version: '0'
requirements: []
rubyforge_project: 
rubygems_version: 2.0.3
signing_key: 
specification_version: 4
summary: Cobweb is a web crawler that can use resque to cluster crawls to quickly
  crawl extremely large sites faster than multi-threaded crawlers.  It is also a standalone
  crawler that has a sophisticated statistics monitoring interface to monitor the
  progress of the crawls.
test_files: []