Sha256: ba72a9d6e48dbaa5679643662d59cd5bcceb5e029e21ae6300500e774b7374db

Contents?: true

Size: 702 Bytes

Versions: 19

Compression:

Stored size: 702 Bytes

Contents

require 'anemone/http'

module Anemone
  class Tentacle

    #
    # Create a new Tentacle
    #
    def initialize(link_queue, page_queue, opts = {})
      @link_queue = link_queue
      @page_queue = page_queue
      @http = Anemone::HTTP.new(opts)
      @opts = opts
    end

    #
    # Gets links from @link_queue, and returns the fetched
    # Page objects into @page_queue
    #
    def run
      loop do
        link, referer, depth = @link_queue.deq

        break if link == :END

        @http.fetch_pages(link, referer, depth).each { |page| @page_queue << page }

        delay
      end
    end

    private

    def delay
      sleep @opts[:delay] if @opts[:delay] > 0
    end

  end
end

Version data entries

19 entries across 19 versions & 4 rubygems

Version Path
sutch-anemone-0.7.2.2 lib/anemone/tentacle.rb
sutch-anemone-0.7.2.1 lib/anemone/tentacle.rb
sutch-anemone-0.7.2 lib/anemone/tentacle.rb
rodneyc-anemone-0.7.1.3 lib/anemone/tentacle.rb
rodneyc-anemone-0.7.1.2 lib/anemone/tentacle.rb
rodneyc-anemone-0.7.1.1 lib/anemone/tentacle.rb
anemone-0.7.2 lib/anemone/tentacle.rb
anemone-0.7.1 lib/anemone/tentacle.rb
anemone-0.7.0 lib/anemone/tentacle.rb
anemone-0.6.1 lib/anemone/tentacle.rb
anemone-0.6.0 lib/anemone/tentacle.rb
anemone-0.5.0 lib/anemone/tentacle.rb
spk-anemone-0.4.0 lib/anemone/tentacle.rb
anemone-0.4.0 lib/anemone/tentacle.rb
anemone-0.3.2 lib/anemone/tentacle.rb
spk-anemone-0.3.1 lib/anemone/tentacle.rb
anemone-0.3.1 lib/anemone/tentacle.rb
spk-anemone-0.3.0 lib/anemone/tentacle.rb
anemone-0.3.0 lib/anemone/tentacle.rb