require 'net/https'
require 'cgi'
module S33r
include Net
# The client performs operations over the network,
# using the core to build request headers and content;
# only client-specific headers are managed here: other headers
# can be handled by the core.
#--
# TODO: need to wrap XML returned into object representation.
# TODO: use customisable thread pool for requests.
#--
class Client
include S33r
attr_accessor :aws_access_key, :aws_secret_access_key
# Size of data chunk to be sent per request when putting data.
attr_accessor :chunk_size
# Headers which should be sent with every request by default (unless overridden).
attr_accessor :client_headers
# Configure either an SSL-enabled or plain HTTP client.
# (If using SSL, no verification of server certificate is performed.)
#
# +options+: hash of optional client config.:
# * :use_ssl => false: only use plain HTTP for connections
# * :dump_requests => true: dump each request's initial line and headers to STDOUT
def initialize(aws_access_key, aws_secret_access_key, options={})
if false == options[:use_ssl]
@client = HTTP.new(HOST, NON_SSL_PORT)
@client.use_ssl = false
else
@client = HTTP.new(HOST, PORT)
# turn off SSL certificate verification
@client.verify_mode = OpenSSL::SSL::VERIFY_NONE
@client.use_ssl = true
end
@dump_requests = (true == options[:dump_requests])
# set default chunk size for streaming request body
@chunk_size = DEFAULT_CHUNK_SIZE
# Amazon S3 developer keys
@aws_access_key = aws_access_key
@aws_secret_access_key = aws_secret_access_key
# headers sent with every request made by this client
@client_headers = {}
end
# Initialise client from YAML configuration file
# (see load_config method for details of acceptable format).
def Client.init(config_file)
aws_access_key, aws_secret_access_key, options, _ = load_config(config_file)
Client.new(aws_access_key, aws_secret_access_key, options)
end
# Load YAML config. file for a client. The config. file looks like this:
#
# :include: test/files/namedbucket_config.yml
#
# The +options+ section contains settings specific to Client and NamedClient instances; +custom+
# contains extra settings specific to your application.
# +options+ and +custom+ sections can be omitted, but settings for AWS keys are required.
#
# Returns an array [aws_access_key, aws_secret_access_key, options, custom], where +options+
# and +custom+ are hashes.
def Client.load_config(config_file)
require 'yaml'
config = YAML::load_file(config_file)
aws_access_key = config['aws_access_key']
aws_secret_access_key = config['aws_secret_access_key']
options = {}
options = S33r.keys_to_symbols(config['options']) if config['options']
custom = {}
custom = S33r.keys_to_symbols(config['custom']) if config['custom']
[aws_access_key, aws_secret_access_key, options, custom]
end
# Wrapper round embedded client +use_ssl+ accessor.
def use_ssl?
@client.use_ssl
end
# Send a request over the wire.
#
# This method streams +data+ if it responds to the +stat+ method
# (as files do).
#
#-- TODO: set timeout on requests in case S3 is unresponsive
def do_request(method, path, data=nil, headers={})
req = get_requester(method, path)
req.chunk_size = @chunk_size
# add the S3 headers which are always required
headers = add_default_headers(headers)
# add any client-specific default headers
headers = add_client_headers(headers)
headers['Authorization'] = generate_auth_header_value(method, path, headers,
@aws_access_key, @aws_secret_access_key)
headers.each do |key, value|
req[key] = value
end
if req.request_body_permitted?
# for streaming files; NB Content-Length will be set by Net::HTTP
# for character-based body content
if data.respond_to?(:stat)
req.body_stream = data
req['Content-Length'] = data.stat.size.to_s
data = nil
end
else
data = nil
end
if @dump_requests
puts req.to_s
end
@client.start do
return @client.request(req, data)
end
end
# Return an instance of an appropriate request class.
def get_requester(method, path)
raise S33rException::UnsupportedHTTPMethod, "The #{method} HTTP method is not supported" if !(METHOD_VERBS.include?(method))
eval("HTTP::" + method[0,1].upcase + method[1..-1].downcase + ".new('#{path}')")
end
# List all buckets.
def list_buckets
do_get('/')
end
# List entries in a bucket.
#
# +query_params+: hash of options on the bucket listing request, passed as querystring parameters to S3
# (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/).
# * :prefix => 'some_string': restrict results to keys beginning with 'some_string'
# * :marker => 'some_string': restict results to keys occurring lexicographically after 'some_string'
# * :max_keys => 1000: return at most this number of keys (maximum possible value is 1000)
# * :delimiter => 'some_string': keys containing the same string between prefix and the delimiter
# are rolled up into a CommonPrefixes element inside the response
def list_bucket(bucket_name, query_params={})
if query_params[:max_keys]
max_keys = query_params[:max_keys].to_i
raise S33rException::BucketListingMaxKeysError, "max_keys option to list bucket cannot be > #{BUCKET_LIST_MAX_MAX_KEYS}" \
if max_keys > BUCKET_LIST_MAX_MAX_KEYS
# take out the max_keys parameter and move it to max-keys
query_params['max-keys'] = query_params.delete(:max_keys)
end
resp = do_get("/#{bucket_name}" + generate_querystring(query_params))
bucket_listing = BucketListing.new(resp.body)
[resp, bucket_listing]
end
# Create a bucket.
def create_bucket(bucket_name, headers={})
do_put("/#{bucket_name}", nil, headers)
end
# Delete a bucket.
#
# +options+ hash can contain the following:
# * :force => true: delete all keys within the bucket then delete the bucket itself
#-- TODO: maybe delete keys matching a partial path
def delete_bucket(bucket_name, headers={}, options={})
if true == options[:force]
_, bucket_listing = list_bucket(bucket_name)
bucket_listing.contents.each_value do |obj|
delete_resource(bucket_name, obj.key)
end
end
do_delete("/#{bucket_name}", headers)
end
# Returns true if bucket exists.
def bucket_exists?(bucket_name)
do_head("/#{bucket_name}").ok?
end
# Fetch a resource and return a fleshed-out S3Object instance.
def get_resource(bucket_name, resource_key, headers={})
do_get("/#{bucket_name}/#{resource_key}", headers)
end
# Put some generic resource onto S3.
def put_resource(bucket_name, resource_key, data, headers={})
do_put("/#{bucket_name}/" + "#{CGI::escape(resource_key)}", data, headers)
end
# Put a string onto S3.
def put_text(string, bucket_name, resource_key, headers={})
headers["Content-Type"] = "text/plain"
put_resource(bucket_name, resource_key, string, headers)
end
# Put a file onto S3.
#
# If +resource_key+ is nil, the filename is used as the key instead.
#
# +headers+ sets some headers with the request; useful if you have an odd file type
# not recognised by the mimetypes library, and want to explicitly set the Content-Type header.
#
# +options+ hash simplifies setting some headers with specific meaning to S3:
# * :render_as_attachment => true: set the Content-Disposition for this file to "attachment" and set
# the default filename for saving the file (when accessed by a web browser) to +filename+; this
# turns the file into a download when opened in a browser, rather than trying to render it inline.
#
# Note that this method uses a handle to the file, so it can be streamed in chunks to S3.
def put_file(filename, bucket_name, resource_key=nil, headers={}, options={})
# default to the file path as the resource key if none explicitly set
if resource_key.nil?
resource_key = filename
end
# set Content-Disposition header
if options[:render_as_attachment]
headers['Content-Disposition'] = "attachment; filename=#{File.basename(filename)}"
end
# content type is explicitly set in the headers, so apply to request
if headers[:content_type]
# use the first MIME type corresponding to this content type string
# (MIME::Types returns an array of possible MIME types)
mime_type = MIME::Types[headers[:content_type]][0]
else
mime_type = guess_mime_type(filename)
end
content_type = mime_type.simplified
headers['Content-Type'] = content_type
headers['Content-Transfer-Encoding'] = 'binary' if mime_type.binary?
# the data we want to put (handle to file, so we can stream from it)
File.open(filename) do |data|
# send the put request
put_resource(bucket_name, resource_key, data, headers)
end
end
# Delete a resource from S3.
def delete_resource(bucket_name, resource_key, headers={})
do_delete("/#{bucket_name}/#{resource_key}", headers)
end
# Add any default headers which should be sent with every request from the client.
#
# +headers+ is a hash of headers already set up. Any headers passed in here
# override the defaults in +client_headers+.
#
# Returns +headers+ with the content of +client_headers+ merged in.
def add_client_headers(headers)
headers.merge!(client_headers) { |key, arg, default| arg }
end
protected
def do_get(path='/', headers={})
do_request('GET', path, headers)
end
def do_head(path='/', headers={})
do_request('HEAD', path, nil, headers)
end
def do_post(path='/', data=nil, headers={})
do_request('POST', path, data, headers)
end
def do_put(path='/', data=nil, headers={})
do_request('PUT', path, data, headers)
end
def do_delete(path, headers={})
do_request('DELETE', path, nil, headers)
end
end
end