Contents

require 'httparty'
require 'nokogiri'

module Jamnagar
  module Utilities
    class UrlExpander
      def initialize(client=nil,cache=nil,parser=nil)
        @client = client || HTTParty
        @cache = cache || Jamnagar::Storage::InMemoryCache.new
        @parser = parser || Nokogiri
      end
      def expand(url)
        cache_hit = nil

        cache_hit = expanded = check_cache(url) if @cache
        expanded = lookup(url) unless expanded

        cache_expanded(url, expanded) unless cache_hit

        expanded
      end
    private
      def check_cache(url)
        result = @cache.get(url)
        return url if result == ""
        result
      end

      def cache_expanded(key, value)
        @cache.set(key, value)
      end

      def lookup(url)
        response = @client.get url, :limit => 10
        {"final_url" => response.request.last_uri.to_s, "final_url_host" => response.request.last_uri.host.to_s, "body" => "null" }
      rescue URI::InvalidURIError => error
        matches = error.message.match(/^bad\sURI\(is\snot\sURI\?\)\:\s(.*)$/)
        return url if matches.nil?
        bad_uri = matches[1]
        good_uri = URI.encode bad_uri
        expand good_uri
      rescue OpenSSL::SSL::SSLError, Net::ReadTimeout, SocketError, EOFError, HTTParty::RedirectionTooDeep, Errno::ETIMEDOUT, Zlib::DataError => error
        {"final_url" => url, "body" => ""}
      end
    end
  end
end

Version data entries

3 entries across 3 versions & 1 rubygems

Version	Path
jamnagar-1.3.9.1	lib/jamnagar/utilities/url_expander.rb
jamnagar-1.3.9	lib/jamnagar/utilities/url_expander.rb
jamnagar-1.3.8	lib/jamnagar/utilities/url_expander.rb