lib/down/net_http.rb in down-5.1.1 vs lib/down/net_http.rb in down-5.2.0

@@ -10,17 +10,23 @@
 require "fileutils"
 
 module Down
   # Provides streaming downloads implemented with Net::HTTP and open-uri.
   class NetHttp < Backend
+    URI_NORMALIZER = -> (url) do
+      addressable_uri = Addressable::URI.parse(url)
+      addressable_uri.normalize.to_s
+    end
+
     # Initializes the backend with common defaults.
     def initialize(*args, **options)
       @options = merge_options({
-        headers:       { "User-Agent" => "Down/#{Down::VERSION}" },
-        max_redirects: 2,
-        open_timeout:  30,
-        read_timeout:  30,
+        headers:        { "User-Agent" => "Down/#{Down::VERSION}" },
+        max_redirects:  2,
+        open_timeout:   30,
+        read_timeout:   30,
+        uri_normalizer: URI_NORMALIZER,
       }, *args, **options)
     end
 
     # Downloads a remote file to disk using open-uri. Accepts any open-uri
     # options, and a few more.
@@ -31,10 +37,11 @@
       max_redirects       = options.delete(:max_redirects)
       progress_proc       = options.delete(:progress_proc)
       content_length_proc = options.delete(:content_length_proc)
       destination         = options.delete(:destination)
       headers             = options.delete(:headers)
+      uri_normalizer      = options.delete(:uri_normalizer)
 
       # Use open-uri's :content_lenth_proc or :progress_proc to raise an
       # exception early if the file is too large.
       #
       # Also disable following redirects, as we'll provide our own
@@ -72,11 +79,11 @@
       end
 
       open_uri_options.merge!(options)
       open_uri_options.merge!(headers)
 
-      uri = ensure_uri(addressable_normalize(url))
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Handle basic authentication in the remote URL.
       if uri.user || uri.password
         open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
         uri.user = nil
@@ -94,15 +101,17 @@
     end
 
     # Starts retrieving the remote file using Net::HTTP and returns an IO-like
     # object which downloads the response body on-demand.
     def open(url, *args, **options)
-      uri     = ensure_uri(addressable_normalize(url))
       options = merge_options(@options, *args, **options)
 
-      max_redirects = options.delete(:max_redirects)
+      max_redirects  = options.delete(:max_redirects)
+      uri_normalizer = options.delete(:uri_normalizer)
 
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
+
       # Create a Fiber that halts when response headers are received.
       request = Fiber.new do
         net_http_request(uri, options, follows_remaining: max_redirects) do |response|
           Fiber.yield response
         end
@@ -255,12 +264,14 @@
 
       headers = options[:headers].to_h
       headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
 
       get = Net::HTTP::Get.new(uri.request_uri, headers)
-      get.basic_auth(uri.user, uri.password) if uri.user || uri.password
 
+      user, password = options[:http_basic_authentication] || [uri.user, uri.password]
+      get.basic_auth(user, password) if user || password
+
       [http, get]
     end
 
     # Yields chunks of the response body to the block.
     def stream_body(response, &block)
@@ -283,23 +294,26 @@
 
       uri
     end
 
     # Makes sure that the URL is properly encoded.
-    def addressable_normalize(url)
+    def normalize_uri(url, uri_normalizer:)
       URI(url)
     rescue URI::InvalidURIError
-      addressable_uri = Addressable::URI.parse(url)
-      addressable_uri.normalize.to_s
+      uri_normalizer.call(url)
     end
 
     # When open-uri raises an exception, it doesn't expose the response object.
     # Fortunately, the exception object holds response data that can be used to
     # rebuild the Net::HTTP response object.
     def rebuild_response_from_open_uri_exception(exception)
       code, message = exception.io.status
 
-      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |code|
+        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(code[0]) do
+          Net::HTTPUnknownResponse
+        end
+      end
       response       = response_class.new(nil, code, message)
 
       exception.io.metas.each do |name, values|
         values.each { |value| response.add_field(name, value) }
       end