Sha256: 430ec4e11616b9b75ece3a9eb148555c490833e6f4bf842fd72acb2527736c8c
Contents?: true
Size: 1.18 KB
Versions: 7
Compression:
Stored size: 1.18 KB
Contents
module NightcrawlerSwift
class MultithreadSync < Command
DEFAULT_POOL_SIZE = 5
def initialize
require 'concurrent/utilities'
require 'concurrent/executors'
@logger = NightcrawlerSwift.logger
end
def execute args = {}
pool_size = args[:pool_size] || DEFAULT_POOL_SIZE
dir_path = args[:dir_path]
@logger.info "[NightcrawlerSwift] dir_path: #{dir_path}"
@logger.info "[NightcrawlerSwift] multithread sync, #{Concurrent.processor_count} processors"
assets = Dir["#{dir_path}/**/**"].
reject {|fullpath| File.directory?(fullpath)}.
map {|fullpath|
path = fullpath.gsub("#{dir_path}/", "")
OpenStruct.new(path: path, fullpath: fullpath)
}
pool = Concurrent::FixedThreadPool.new pool_size
assets.each do |asset|
pool.post do
@logger.info "[NightcrawlerSwift] #{asset.path}"
upload = Upload.new
upload.execute asset.path, File.open(asset.fullpath, "r")
end
end
sleep(1) while pool.queue_length > 0
@logger.info "[NightcrawlerSwift] shutting down"
pool.shutdown
pool.wait_for_termination
end
end
end
Version data entries
7 entries across 7 versions & 1 rubygems