require "net/http" require "uri" module IOStreams module Utils class ReliableHTTP attr_reader :username, :password, :max_redirects, :url # Reliable HTTP implementation with support for: # * HTTP Redirects # * Basic authentication # * Raises an exception anytime the HTTP call is not successful. # * TODO: Automatic retries with a logarithmic backoff strategy. # # Parameters: # url: [String] # URI of the file to download. # Example: # https://www5.fdic.gov/idasp/Offices2.zip # http://hostname/path/file_name # # Full url showing all the optional elements that can be set via the url: # https://username:password@hostname/path/file_name # # username: [String] # When supplied, basic authentication is used with the username and password. # # password: [String] # Password to use use with basic authentication when the username is supplied. # # max_redirects: [Integer] # Maximum number of http redirects to follow. def initialize(url, username: nil, password: nil, max_redirects: 10) uri = URI.parse(url) unless %w[http https].include?(uri.scheme) raise(ArgumentError, "Invalid URL. Required Format: 'http:///', or 'https:///'") end @username = username || uri.user @password = password || uri.password @max_redirects = max_redirects @url = url end # Read a file using an http get. # # For example: # IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').reader {|file| puts file.read} # # Read the file without unzipping and streaming the first file in the zip: # IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').stream(:none).reader {|file| puts file.read} # # Notes: # * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first. def post(&block) handle_redirects(Net::HTTP::Post, url, max_redirects, &block) end def get(&block) handle_redirects(Net::HTTP::Get, url, max_redirects, &block) end private def handle_redirects(request_class, uri, max_redirects, &block) uri = URI.parse(uri) unless uri.is_a?(URI) result = nil raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if max_redirects < 1 Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http| request = request_class.new(uri) request.basic_auth(username, password) if username http.request(request) do |response| raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}") if response.is_a?(Net::HTTPNotFound) if response.is_a?(Net::HTTPUnauthorized) raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.") end if response.is_a?(Net::HTTPRedirection) new_uri = response["location"] return handle_redirects(request_class, new_uri, max_redirects: max_redirects - 1, &block) end unless response.is_a?(Net::HTTPSuccess) raise(IOStreams::Errors::CommunicationsFailure, "Invalid response code: #{response.code}") end yield(response) if block_given? result = response end end result end end end end