Sha256: 48b13f876a4ddb8a1edd5919d517dbfb36be381803af9fabca0fcc4ee176c03d

Contents?: true

Size: 1.32 KB

Versions: 22

Compression:

Stored size: 1.32 KB

Contents

module CMSScanner
  module Finders
    class Finder
      # Module to provide an easy way to enumerate items such as plugins, themes etc
      module Enumerator
        # @param [ Hash ] The target urls
        # @param [ Hash ] opts
        # @option opts [ Boolean ] :show_progression Wether or not to display the progress bar
        # @option opts [ Regexp ] :exclude_content
        #
        # @yield [ Typhoeus::Response, String ]
        def enumerate(target_urls, opts = {})
          bar = progress_bar(total: target_urls.size) if opts[:show_progression]

          target_urls.each do |url, id|
            request = browser.forge_request(url, request_params)

            request.on_complete do |res|
              bar.progress += 1 if opts[:show_progression]

              next if target.homepage_or_404?(res)
              next if opts[:exclude_content] && res.body.match(opts[:exclude_content])

              yield res, id
            end

            hydra.queue(request)
          end

          hydra.run
        end

        # @return [ Hash ]
        def request_params
          # disabling the cache, as it causes a 'stack level too deep' exception
          # with a large number of requests :/
          # See https://github.com/typhoeus/typhoeus/issues/408
          { cache_ttl: 0 }
        end
      end
    end
  end
end

Version data entries

22 entries across 22 versions & 1 rubygems

Version Path
cms_scanner-0.0.37.2 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.37.1 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.37 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.36 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.35.1 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.35 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.34 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.33 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.32 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.31 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.30 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.29 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.28 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.27 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.26 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.25 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.24 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.23 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.22 lib/cms_scanner/finders/finder/enumerator.rb
cms_scanner-0.0.21 lib/cms_scanner/finders/finder/enumerator.rb