lib/miteru/orchestrator.rb in miteru-2.4.0 vs lib/miteru/orchestrator.rb in miteru-3.0.0

- old
+ new

@@ -2,33 +2,11 @@ module Miteru class Orchestrator < Service def call logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose? - - if sidekiq? - sidekiq_call - else - parallel_call - end - end - - def sidekiq_call array_of_args = non_cached_websites.map { |website| [website.url, website.source] } Jobs::CrawleJob.perform_bulk(array_of_args, batch_size: Miteru.config.sidekiq_batch_size) - end - - def parallel_call - logger.info("Use #{threads} thread(s).") if verbose? - Parallel.each(non_cached_websites, in_threads: threads) do |website| - logger.info("Website:#{website.truncated_url} crawling started.") if verbose? - result = Crawler.result(website) - if result.success? - logger.info("Crawler:#{website.truncated_url} succeeded.") - else - logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.") - end - end end # # @return [Array<Miteru::Website>] #