Sha256: 606a04b62cd4a09a4074a31e10c09e55fa5b6d587f8c887f92c6cdfb08f18e88

Contents?: true

Size: 1.64 KB

Versions: 1

Compression:

Stored size: 1.64 KB

Contents

require 'httparty'
require 'tempfile'
require 'addressable'

module BrowseEverything
  class Retriever
    attr_accessor :chunk_size

    def initialize
      @chunk_size = 16384
    end

    def download(spec, target = nil)
      if target.nil?
        ext = File.extname(spec['file_name'])
        base = File.basename(spec['file_name'], ext)
        target = Dir::Tmpname.create([base, ext]) {}
      end

      File.open(target, 'wb') do |output|
        retrieve(spec) do |chunk, retrieved, total|
          output.write(chunk)
          yield(target, retrieved, total) if block_given?
        end
      end
      target
    end

    def retrieve(spec)
      if spec.key?('expires') && Time.parse(spec['expires']) < Time.now
        raise ArgumentError, "Download spec expired at #{spec['expires']}"
      end

      url = ::Addressable::URI.parse(spec['url'])
      file_size = spec.fetch('file_size', 0).to_i
      retrieved = 0
      case url.scheme
      when 'file'
        File.open(url.path, 'rb') do |f|
          until f.eof?
            chunk = f.read(chunk_size)
            retrieved += chunk.length
            yield(chunk, retrieved, file_size)
          end
        end
      when /https?/
        headers = spec['auth_header'] || {}
        headers.each_pair do |k, v|
          headers[k] = v.tr('+', ' ')
        end

        stream_body = file_size > 500.megabytes

        HTTParty.get(url.to_s, stream_body: stream_body, headers: headers) do |chunk|
          retrieved += chunk.length
          yield(chunk, retrieved, file_size)
        end
      else
        raise URI::BadURIError, "Unknown URI scheme: #{url.scheme}"
      end
    end
  end
end

Version data entries

1 entries across 1 versions & 1 rubygems

Version Path
browse-everything-0.14.1 lib/browse_everything/retriever.rb