require 'uri' require 'hpricot' require 'json' require 'cgi' require 'timeout' require 'net/http' module Itrigga module NetHelper module_function def typhoeus_present? defined?(Typhoeus) end def do_get(url, time_out=5, retries_on_timeout=5, max_redirects = 3) retrycount = 0 resp = nil begin resp = get_with_timeout( url, time_out ) handle_response( url, resp, retries_on_timeout, max_redirects ) rescue Timeout::Error if(retrycount.to_i < retries_on_timeout.to_i) retrycount+=1 retry else raise IOError.new( "HTTP request timed out #{retrycount} times" ) end end end def typhoeus_request( opts={} ) opts[:timeout] ||= 30000 # ms opts[:follow_location] ||= true opts[:disable_ssl_peer_verification] = true if opts[:disable_ssl_peer_verification].nil? request = Typhoeus::Request.new(opts[:url], opts) request.on_complete do |response| if response.success? return response.body elsif response.timed_out? # aw hell no log("got a time out") elsif response.code == 0 # Could not get an http response, something's wrong. log(response.curl_error_message) else # Received a non-successful http response. log("HTTP request failed: " + response.code.to_s) end end end def get_response( url ) if typhoeus_present? typhoeus_request( url ) else Net::HTTP.get_response(URI.parse(url)) end end def get_with_timeout( url, time_out) resp = nil if defined?(SystemTimer) resp = SystemTimer.timeout_after(time_out) do get_response(url) end else resp = timeout(time_out) do get_response(url) end end resp end resp = timeout(time_out) do end resp end def handle_response( url, resp, retries_on_timeout=5, max_redirects = 3 ) if resp.is_a? Net::HTTPSuccess then resp.body elsif resp.is_a? Net::HTTPRedirection if max_redirects > 0 do_get( URI.parse(url).merge(resp['location']).to_s, retries_on_timeout, max_redirects - 1 ) else raise IOError.new("too many redirects!") end else resp.error! end end def get( options = {} ) opts = {:timeout=>5, :retries_on_timeout=>5, :max_redirects => 3, :headers=>{} }.merge(options) raise ArgumentError.new(":url is required" ) unless opts[:url] if (opts[:username] || opts[:headers]).to_s.empty? do_get(opts[:url], opts[:timeout], opts[:retries_on_timeout], opts[:max_redirects]) else retrycount = 0 resp = begin timeout( opts[:timeout] ) do raw_get(opts) end rescue TimeoutError if(retrycount < opts[:retries_on_timeout]) retrycount+=1 retry else raise IOError.new( "HTTP request timed out #{retrycount} times" ) end end resp end end def raw_get(opts) resp = nil establish_session_if_needed(opts) if opts[:username] resp = get_with_auth(opts) retries = 0 while resp.is_a? Net::HTTPRedirection do retries += 1 raise IOError.new( "HTTP request timed out #{retries} times" ) if retries > (opts[:max_redirects] || 3) resp = get_with_auth(opts.merge(:parsed_url=>URI.parse(resp['location']))) end resp.body else response = opts[:http_session].request_get(opts[:parsed_url].path, opts[:headers]) response.body end end def get_with_auth( opts ) establish_session_if_needed(opts) req = Net::HTTP::Get.new(opts[:parsed_url].path) req.basic_auth( opts[:username], opts[:password] ) if opts[:username] resp = opts[:http_session].request(req, opts[:headers]) end def establish_session_if_needed(opts) opts[:parsed_url] ||= URI.parse(opts[:url]) opts[:http_session] ||= Net::HTTP.new(opts[:parsed_url].host, opts[:parsed_url].port) opts[:http_session].use_ssl = true if opts[:parsed_url].scheme == 'https' end def query_string( h, opts={:encode_values=>false, :skip_empty=>false} ) params = [] h.each{ |k,v| params << "#{k.to_s}=#{ opts[:encode_values] ? url_encode(v) : v }" unless v.to_s.empty? && opts[:skip_empty] } params.sort.join('&') end def url_encode( s ) URI.escape( s.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]") ) end def discover_favicon_url(root_url, time_out = 5) favicon_urls = ['favicon.ico'] + discover_icon_hrefs(root_url) favicon_urls.compact.each do |url| abs_url = URI.join(root_url, url).to_s return abs_url if get_with_timeout( abs_url, time_out ).is_a?(Net::HTTPSuccess) end nil end def discover_icon_hrefs(url) pd = open(url) { |f| parse_with_hpricot(f) } links = pd.search("//link[@rel='shortcut icon']") + pd.search("//link[@rel='icon']") links.map{ |link| link.attributes['href'] }.compact.to_a end # Have difficulty mocking/stubbing the top-level Hpricot call # This wrapper makes it testable def parse_with_hpricot(html) Hpricot.parse(html) end def ip_of(server) IPSocket::getaddress(server) end # uses the url shortener service defined for this site # by default will use bit.ly def shorten_url(opts={}) opts[:attempts] ||= 0 return opts[:url] if opts[:attempts] > 3 api_url = self.format_url_shortener_api_url(opts[:url], opts[:config]) begin response = NetHelper.get :url => api_url rescue Exception => e sleep 10 unless defined?(RAILS_ENV) && RAILS_ENV == "test" return shorten_url(:url=>opts[:url], :attempts=>opts[:attempts]+1) end data = JSON.parse(response).recursive_symbolize_keys! if data[:status_code] == 200 return data[:data][:url] else # theres been a problem, try again if we have tries left sleep 10 unless defined?(RAILS_ENV) && RAILS_ENV == "test" return shorten_url(:url=>opts[:url], :attempts=>opts[:attempts]+1) end end # parses the :url template for the url shortener service def format_url_shortener_api_url(raw_url, config_hash) escaped_url = CGI::escape(raw_url) config_hash[:url].gsub("{{username}}",config_hash[:username] || "").gsub("{{api_key}}",config_hash[:api_key] || "").gsub("{{raw_url}}",escaped_url) end # # Transfer a file using scp between servers # # Options: # :host - the target server name. Hash. Must have keys: # => :port - the port on which to connect # => :ssh_key_path - the absolute path to the ssh key file which it will use to connect # => :user - the user to connect as # => :host - the hostname to connect to # :target_path - the absolute path on the target server to put the file # :file - the file to transfer. Must be an absolute path to the file # def transfer_file_scp(opts = {}) raise ArgumentError.new("File '#{opts[:file]}' does not exist!") unless opts[:file] && File.exist?(opts[:file]) raise ArgumentError.new("host is required") unless opts[:host] raise ArgumentError.new("No target_path defined") unless opts[:target_path] #host = TRIGGA_CONFIG.hosts.detect{|host| host[:display_name].to_s == opts[:target].to_s} # now that everything checks out make sure the destination dir path exists. If not then create it command = "ssh -p#{opts[:host][:port]} -i #{opts[:host][:ssh_key_path]} #{opts[:host][:user]}@#{opts[:host][:host]} 'mkdir -p #{File.dirname(opts[:target_path])}'" `#{command}` # now transfer the file across command = "scp -P#{opts[:host][:port]} -i #{opts[:host][:ssh_key_path]} #{opts[:file]} #{opts[:host][:user]}@#{opts[:host][:host]}:#{opts[:target_path]}" `#{command}` end end end