Sha256: 606a04b62cd4a09a4074a31e10c09e55fa5b6d587f8c887f92c6cdfb08f18e88
Contents?: true
Size: 1.64 KB
Versions: 1
Compression:
Stored size: 1.64 KB
Contents
require 'httparty' require 'tempfile' require 'addressable' module BrowseEverything class Retriever attr_accessor :chunk_size def initialize @chunk_size = 16384 end def download(spec, target = nil) if target.nil? ext = File.extname(spec['file_name']) base = File.basename(spec['file_name'], ext) target = Dir::Tmpname.create([base, ext]) {} end File.open(target, 'wb') do |output| retrieve(spec) do |chunk, retrieved, total| output.write(chunk) yield(target, retrieved, total) if block_given? end end target end def retrieve(spec) if spec.key?('expires') && Time.parse(spec['expires']) < Time.now raise ArgumentError, "Download spec expired at #{spec['expires']}" end url = ::Addressable::URI.parse(spec['url']) file_size = spec.fetch('file_size', 0).to_i retrieved = 0 case url.scheme when 'file' File.open(url.path, 'rb') do |f| until f.eof? chunk = f.read(chunk_size) retrieved += chunk.length yield(chunk, retrieved, file_size) end end when /https?/ headers = spec['auth_header'] || {} headers.each_pair do |k, v| headers[k] = v.tr('+', ' ') end stream_body = file_size > 500.megabytes HTTParty.get(url.to_s, stream_body: stream_body, headers: headers) do |chunk| retrieved += chunk.length yield(chunk, retrieved, file_size) end else raise URI::BadURIError, "Unknown URI scheme: #{url.scheme}" end end end end
Version data entries
1 entries across 1 versions & 1 rubygems
Version | Path |
---|---|
browse-everything-0.14.1 | lib/browse_everything/retriever.rb |