require 'rest-client' require 'itrigga/core_ext' Dir[File.join(File.expand_path(File.dirname(__FILE__)),"net_helper","*.rb")].each{|f| require f} require 'uri' require 'hpricot' require 'json' require 'cgi' require 'timeout' require 'net/http' module Itrigga module NetHelper module_function # gets a list of engines available (each is a different class in this module that implements 'get' method) def available_http_engines Itrigga::NetHelper.constants end def default_http_engine "RestClient" end # wrapper method for backwards compat. def do_get(url, timeout=5, retries_on_timeout=5, max_redirects = 3) get :url => url, :timeout => timeout, :retries_on_timeout => retries_on_timeout, :max_redirects => max_redirects end def get(options = {}) opts = { :timeout => 5, :retries_on_timeout => 5, :max_redirects => 3, :headers => {} }.merge(options) raise ArgumentError.new(":url is required" ) unless opts[:url] engine_klass = get_engine opts with_timeout(opts) { engine_klass.get(opts) } end def with_timeout(opts, &block) retrycount = 0 resp = begin if defined?(SystemTimer) SystemTimer.timeout_after(opts[:timeout]) do yield end else timeout( opts[:timeout] ) do yield end end rescue ::TimeoutError, ::RestClient::RequestTimeout if(retrycount < opts[:retries_on_timeout]) retrycount+=1 retry else raise IOError.new( "HTTP request timed out #{retrycount} times" ) end end resp end def get_engine(opts = {}) # default to rest_client if not given an engine opts[:http_engine] ||= ( defined?(ITNH_HTTP_ENGINE) ? ITNH_HTTP_ENGINE : default_http_engine ) # only use typhoeus if it is actually been required opts[:http_engine] = default_http_engine if opts[:http_engine] == "Typhoeus" && defined?(::Typhoeus) == nil if available_http_engines.include?(opts[:http_engine]) Itrigga::NetHelper.const_get opts[:http_engine] else puts "[NetHelper] Could not find http_engine '#{opts[:http_engine]}'. Available engines are: #{available_http_engines.join(',')}. Defaulting to Net::HTTP" Itrigga::NetHelper.const_get default_http_engine end end def query_string( h, opts={:encode_values=>false, :skip_empty=>false} ) params = [] h.each{ |k,v| params << "#{k.to_s}=#{ opts[:encode_values] ? url_encode(v) : v }" unless v.to_s.empty? && opts[:skip_empty] } params.sort.join('&') end def url_encode( s ) URI.escape( s.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]") ) end def discover_favicon_url(root_url, time_out = 5) favicon_urls = ['favicon.ico'] + discover_icon_hrefs(root_url) favicon_urls.compact.each do |url| abs_url = URI.join(root_url, url).to_s return abs_url if get_with_timeout( abs_url, time_out ).is_a?(Net::HTTPSuccess) end nil end def discover_icon_hrefs(url) pd = open(url) { |f| parse_with_hpricot(f) } links = pd.search("//link[@rel='shortcut icon']") + pd.search("//link[@rel='icon']") links.map{ |link| link.attributes['href'] }.compact.to_a end # Have difficulty mocking/stubbing the top-level Hpricot call # This wrapper makes it testable def parse_with_hpricot(html) Hpricot.parse(html) end def ip_of(server) IPSocket::getaddress(server) end # uses the url shortener service defined for this site # by default will use bit.ly def shorten_url(opts={}) opts[:attempts] ||= 0 return opts[:url] if opts[:attempts] > 3 api_url = self.format_url_shortener_api_url(opts[:url], opts[:config]) begin response = NetHelper.get :url => api_url rescue Exception => e sleep 10 unless defined?(RAILS_ENV) && RAILS_ENV == "test" return shorten_url(:url=>opts[:url], :attempts=>opts[:attempts]+1) end data = JSON.parse(response).recursive_symbolize_keys! if data[:status_code] == 200 return data[:data][:url] else # theres been a problem, try again if we have tries left sleep 10 unless defined?(RAILS_ENV) && RAILS_ENV == "test" return shorten_url(:url=>opts[:url], :attempts=>opts[:attempts]+1) end end # parses the :url template for the url shortener service def format_url_shortener_api_url(raw_url, config_hash) escaped_url = CGI::escape(raw_url) config_hash[:url].gsub("{{username}}",config_hash[:username] || "").gsub("{{api_key}}",config_hash[:api_key] || "").gsub("{{raw_url}}",escaped_url) end # # Transfer a file using scp between servers # # Options: # :host - the target server name. Hash. Must have keys: # => :port - the port on which to connect # => :ssh_key_path - the absolute path to the ssh key file which it will use to connect # => :user - the user to connect as # => :host - the hostname to connect to # :target_path - the absolute path on the target server to put the file # :file - the file to transfer. Must be an absolute path to the file # def transfer_file_scp(opts = {}) raise ArgumentError.new("File '#{opts[:file]}' does not exist!") unless opts[:file] && File.exist?(opts[:file]) raise ArgumentError.new("host is required") unless opts[:host] raise ArgumentError.new("No target_path defined") unless opts[:target_path] #host = TRIGGA_CONFIG.hosts.detect{|host| host[:display_name].to_s == opts[:target].to_s} # now that everything checks out make sure the destination dir path exists. If not then create it command = "ssh -p#{opts[:host][:port]} -i #{opts[:host][:ssh_key_path]} #{opts[:host][:user]}@#{opts[:host][:host]} 'mkdir -p #{File.dirname(opts[:target_path])}'" `#{command}` # now transfer the file across command = "scp -P#{opts[:host][:port]} -i #{opts[:host][:ssh_key_path]} #{opts[:file]} #{opts[:host][:user]}@#{opts[:host][:host]}:#{opts[:target_path]}" `#{command}` end end end