Contents

# frozen_string_literal: true

require "colorize"
require "parallel"
require "uri"

module Miteru
  class Crawler
    attr_reader :downloader
    attr_reader :feeds

    def initialize
      @downloader = Downloader.new(Miteru.configuration.download_to)

      @feeds = Feeds.new
      @notifier = Notifier.new
    end

    def crawl(url)
      website = Website.new(url)
      downloader.download_kits(website.kits) if website.has_kits? && auto_download?
      notify(website) if website.has_kits? || verbose?
    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
      nil
    end

    def execute
      threads = Miteru.configuration.threads
      suspicious_urls = feeds.suspicious_urls
      puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?

      Parallel.each(suspicious_urls, in_threads: threads) do |url|
        crawl url
      end
    end

    def self.execute
      new.execute
    end

    def notify(website)
      @notifier.notify(url: website.url, kits: website.kits, message: website.message)
    end

    def auto_download?
      Miteru.configuration.auto_download?
    end

    def verbose?
      Miteru.configuration.verbose?
    end
  end
end

Version data entries

4 entries across 4 versions & 1 rubygems

Version	Path
miteru-0.12.12	lib/miteru/crawler.rb
miteru-0.12.11	lib/miteru/crawler.rb
miteru-0.12.10	lib/miteru/crawler.rb
miteru-0.12.9	lib/miteru/crawler.rb