lib/down/net_http.rb in down-4.0.1 vs lib/down/net_http.rb in down-4.1.0

@@ -10,11 +10,11 @@
 require "cgi"
 
 module Down
   class NetHttp < Backend
     def initialize(options = {})
-      @options = options
+      @options = { "User-Agent" => "Down/#{Down::VERSION}" }.merge(options)
     end
 
     def download(uri, options = {})
       options = @options.merge(options)
 
@@ -22,11 +22,10 @@
       max_redirects       = options.delete(:max_redirects) || 2
       progress_proc       = options.delete(:progress_proc)
       content_length_proc = options.delete(:content_length_proc)
 
       open_uri_options = {
-        "User-Agent" => "Down/#{VERSION}",
         content_length_proc: proc { |size|
           if size && max_size && size > max_size
             raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
           end
           content_length_proc.call(size) if content_length_proc
@@ -60,13 +59,11 @@
       tries = max_redirects + 1
 
       begin
         uri = URI(uri)
 
-        if uri.class != URI::HTTP && uri.class != URI::HTTPS
-          raise Down::InvalidUrl, "URL scheme needs to be http or https"
-        end
+        fail Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
 
         if uri.user || uri.password
           open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
           uri.user = nil
           uri.password = nil
@@ -81,11 +78,11 @@
             open_uri_options["Cookie"] = exception.io.meta["set-cookie"]
           end
 
           retry
         else
-          raise Down::TooManyRedirects, "too many redirects"
+          fail Down::TooManyRedirects, "too many redirects"
         end
       rescue OpenURI::HTTPError => exception
         code, message = exception.io.status
         response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
         response = response_class.new(nil, code, message)
@@ -105,25 +102,19 @@
       # does.
       open_uri_file = downloaded_file
       downloaded_file = copy_to_tempfile(uri.path, open_uri_file)
       OpenURI::Meta.init downloaded_file, open_uri_file
 
-      downloaded_file.extend DownloadedFile
+      downloaded_file.extend Down::NetHttp::DownloadedFile
       downloaded_file
     end
 
     def open(uri, options = {})
       options = @options.merge(options)
+      uri     = URI(uri)
 
-      begin
-        uri = URI(uri)
-        if uri.class != URI::HTTP && uri.class != URI::HTTPS
-          raise Down::InvalidUrl, "URL scheme needs to be http or https"
-        end
-      rescue URI::InvalidURIError
-        raise Down::InvalidUrl, "URL was invalid"
-      end
+      fail Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
 
       http_class = Net::HTTP
 
       if options[:proxy]
         proxy = URI(options[:proxy])
@@ -150,10 +141,12 @@
 
       http.read_timeout = options[:read_timeout] if options.key?(:read_timeout)
       http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)
 
       request_headers = options.select { |key, value| key.is_a?(String) }
+      request_headers["Accept-Encoding"] = "" # otherwise FiberError can be raised
+
       get = Net::HTTP::Get.new(uri.request_uri, request_headers)
       get.basic_auth(uri.user, uri.password) if uri.user || uri.password
 
       request = Fiber.new do
         http.start do
@@ -162,20 +155,24 @@
             response.instance_variable_set("@read", true)
           end
         end
       end
 
-      begin
-        response = request.resume
+      response = request.resume
 
-        response_error!(response) unless (200..299).cover?(response.code.to_i)
-      rescue => exception
-        request_error!(exception)
+      response_error!(response) unless (200..299).cover?(response.code.to_i)
+
+      body_chunks = Enumerator.new do |yielder|
+        begin
+          response.read_body { |chunk| yielder << chunk }
+        rescue => exception
+          request_error!(exception)
+        end
       end
 
       Down::ChunkedIO.new(
-        chunks:     response.enum_for(:read_body),
+        chunks:     body_chunks,
         size:       response["Content-Length"] && response["Content-Length"].to_i,
         encoding:   response.type_params["charset"],
         rewindable: options.fetch(:rewindable, true),
         on_close:   -> { request.resume }, # close HTTP connnection
         data: {
@@ -185,10 +182,12 @@
                       headers.merge!(name => value)
                     },
           response: response,
         },
       )
+    rescue => exception
+      request_error!(exception)
     end
 
     private
 
     def copy_to_tempfile(basename, io)
@@ -233,12 +232,10 @@
            Errno::EHOSTUNREACH
         raise Down::ConnectionError, exception.message
       when SocketError
         raise Down::ConnectionError, "domain name could not be resolved"
       when Errno::ETIMEDOUT,
-           Timeout::Error,
-           Net::OpenTimeout,
-           Net::ReadTimeout
+           Timeout::Error
         raise Down::TimeoutError, "request timed out"
       when defined?(OpenSSL) && OpenSSL::SSL::SSLError
         raise Down::SSLError, exception.message
       else
         raise exception