require 'net/https' require 'cgi' module S33r include Net # The client performs operations over the network, # using the core to build request headers and content; # only client-specific headers are managed here: other headers # can be handled by the core. #-- # TODO: need to wrap XML returned into object representation. # TODO: use customisable thread pool for requests. #-- class Client include S33r attr_accessor :aws_access_key, :aws_secret_access_key # Size of data chunk to be sent per request when putting data. attr_accessor :chunk_size # Headers which should be sent with every request by default (unless overridden). attr_accessor :client_headers # Configure either an SSL-enabled or plain HTTP client. # (If using SSL, no verification of server certificate is performed.) # # +options+: hash of optional client config.: # * :use_ssl => false: only use plain HTTP for connections # * :dump_requests => true: dump each request's initial line and headers to STDOUT def initialize(aws_access_key, aws_secret_access_key, options={}) if false == options[:use_ssl] @client = HTTP.new(HOST, NON_SSL_PORT) @client.use_ssl = false else @client = HTTP.new(HOST, PORT) # turn off SSL certificate verification @client.verify_mode = OpenSSL::SSL::VERIFY_NONE @client.use_ssl = true end @dump_requests = (true == options[:dump_requests]) # set default chunk size for streaming request body @chunk_size = DEFAULT_CHUNK_SIZE # Amazon S3 developer keys @aws_access_key = aws_access_key @aws_secret_access_key = aws_secret_access_key # headers sent with every request made by this client @client_headers = {} end # Initialise client from YAML configuration file # (see load_config method for details of acceptable format). def Client.init(config_file) aws_access_key, aws_secret_access_key, options, _ = load_config(config_file) Client.new(aws_access_key, aws_secret_access_key, options) end # Load YAML config. file for a client. The config. file looks like this: # # :include: test/files/namedbucket_config.yml # # The +options+ section contains settings specific to Client and NamedClient instances; +custom+ # contains extra settings specific to your application. # +options+ and +custom+ sections can be omitted, but settings for AWS keys are required. # # Returns an array [aws_access_key, aws_secret_access_key, options, custom], where +options+ # and +custom+ are hashes. def Client.load_config(config_file) require 'yaml' config = YAML::load_file(config_file) aws_access_key = config['aws_access_key'] aws_secret_access_key = config['aws_secret_access_key'] options = {} options = S33r.keys_to_symbols(config['options']) if config['options'] custom = {} custom = S33r.keys_to_symbols(config['custom']) if config['custom'] [aws_access_key, aws_secret_access_key, options, custom] end # Wrapper round embedded client +use_ssl+ accessor. def use_ssl? @client.use_ssl end # Send a request over the wire. # # This method streams +data+ if it responds to the +stat+ method # (as files do). # #-- TODO: set timeout on requests in case S3 is unresponsive def do_request(method, path, data=nil, headers={}) req = get_requester(method, path) req.chunk_size = @chunk_size # add the S3 headers which are always required headers = add_default_headers(headers) # add any client-specific default headers headers = add_client_headers(headers) headers['Authorization'] = generate_auth_header_value(method, path, headers, @aws_access_key, @aws_secret_access_key) headers.each do |key, value| req[key] = value end if req.request_body_permitted? # for streaming files; NB Content-Length will be set by Net::HTTP # for character-based body content if data.respond_to?(:stat) req.body_stream = data req['Content-Length'] = data.stat.size.to_s data = nil end else data = nil end if @dump_requests puts req.to_s end @client.start do return @client.request(req, data) end end # Return an instance of an appropriate request class. def get_requester(method, path) raise S33rException::UnsupportedHTTPMethod, "The #{method} HTTP method is not supported" if !(METHOD_VERBS.include?(method)) eval("HTTP::" + method[0,1].upcase + method[1..-1].downcase + ".new('#{path}')") end # List all buckets. def list_buckets do_get('/') end # List entries in a bucket. # # +query_params+: hash of options on the bucket listing request, passed as querystring parameters to S3 # (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/). # * :prefix => 'some_string': restrict results to keys beginning with 'some_string' # * :marker => 'some_string': restict results to keys occurring lexicographically after 'some_string' # * :max_keys => 1000: return at most this number of keys (maximum possible value is 1000) # * :delimiter => 'some_string': keys containing the same string between prefix and the delimiter # are rolled up into a CommonPrefixes element inside the response def list_bucket(bucket_name, query_params={}) if query_params[:max_keys] max_keys = query_params[:max_keys].to_i raise S33rException::BucketListingMaxKeysError, "max_keys option to list bucket cannot be > #{BUCKET_LIST_MAX_MAX_KEYS}" \ if max_keys > BUCKET_LIST_MAX_MAX_KEYS # take out the max_keys parameter and move it to max-keys query_params['max-keys'] = query_params.delete(:max_keys) end resp = do_get("/#{bucket_name}" + generate_querystring(query_params)) bucket_listing = BucketListing.new(resp.body) [resp, bucket_listing] end # Create a bucket. def create_bucket(bucket_name, headers={}) do_put("/#{bucket_name}", nil, headers) end # Delete a bucket. # # +options+ hash can contain the following: # * :force => true: delete all keys within the bucket then delete the bucket itself #-- TODO: maybe delete keys matching a partial path def delete_bucket(bucket_name, headers={}, options={}) if true == options[:force] _, bucket_listing = list_bucket(bucket_name) bucket_listing.contents.each_value do |obj| delete_resource(bucket_name, obj.key) end end do_delete("/#{bucket_name}", headers) end # Returns true if bucket exists. def bucket_exists?(bucket_name) do_head("/#{bucket_name}").ok? end # Fetch a resource and return a fleshed-out S3Object instance. def get_resource(bucket_name, resource_key, headers={}) do_get("/#{bucket_name}/#{resource_key}", headers) end # Put some generic resource onto S3. def put_resource(bucket_name, resource_key, data, headers={}) do_put("/#{bucket_name}/" + "#{CGI::escape(resource_key)}", data, headers) end # Put a string onto S3. def put_text(string, bucket_name, resource_key, headers={}) headers["Content-Type"] = "text/plain" put_resource(bucket_name, resource_key, string, headers) end # Put a file onto S3. # # If +resource_key+ is nil, the filename is used as the key instead. # # +headers+ sets some headers with the request; useful if you have an odd file type # not recognised by the mimetypes library, and want to explicitly set the Content-Type header. # # +options+ hash simplifies setting some headers with specific meaning to S3: # * :render_as_attachment => true: set the Content-Disposition for this file to "attachment" and set # the default filename for saving the file (when accessed by a web browser) to +filename+; this # turns the file into a download when opened in a browser, rather than trying to render it inline. # # Note that this method uses a handle to the file, so it can be streamed in chunks to S3. def put_file(filename, bucket_name, resource_key=nil, headers={}, options={}) # default to the file path as the resource key if none explicitly set if resource_key.nil? resource_key = filename end # set Content-Disposition header if options[:render_as_attachment] headers['Content-Disposition'] = "attachment; filename=#{File.basename(filename)}" end # content type is explicitly set in the headers, so apply to request if headers[:content_type] # use the first MIME type corresponding to this content type string # (MIME::Types returns an array of possible MIME types) mime_type = MIME::Types[headers[:content_type]][0] else mime_type = guess_mime_type(filename) end content_type = mime_type.simplified headers['Content-Type'] = content_type headers['Content-Transfer-Encoding'] = 'binary' if mime_type.binary? # the data we want to put (handle to file, so we can stream from it) File.open(filename) do |data| # send the put request put_resource(bucket_name, resource_key, data, headers) end end # Delete a resource from S3. def delete_resource(bucket_name, resource_key, headers={}) do_delete("/#{bucket_name}/#{resource_key}", headers) end # Add any default headers which should be sent with every request from the client. # # +headers+ is a hash of headers already set up. Any headers passed in here # override the defaults in +client_headers+. # # Returns +headers+ with the content of +client_headers+ merged in. def add_client_headers(headers) headers.merge!(client_headers) { |key, arg, default| arg } end protected def do_get(path='/', headers={}) do_request('GET', path, headers) end def do_head(path='/', headers={}) do_request('HEAD', path, nil, headers) end def do_post(path='/', data=nil, headers={}) do_request('POST', path, data, headers) end def do_put(path='/', data=nil, headers={}) do_request('PUT', path, data, headers) end def do_delete(path, headers={}) do_request('DELETE', path, nil, headers) end end end