require 'fileutils' require 'time' require 'fakes3/s3_object' require 'fakes3/bucket' require 'fakes3/rate_limitable_file' require 'digest/md5' require 'yaml' module FakeS3 class FileStore FAKE_S3_METADATA_DIR = ".fakes3_metadataFFF" # S3 clients with overly strict date parsing fails to parse ISO 8601 dates # without any sub second precision (e.g. jets3t v0.7.2), and the examples # given in the official AWS S3 documentation specify three (3) decimals for # sub second precision. SUBSECOND_PRECISION = 3 def initialize(root) @root = root @buckets = [] @bucket_hash = {} Dir[File.join(root,"*")].each do |bucket| bucket_name = File.basename(bucket) bucket_obj = Bucket.new(bucket_name,Time.now,[]) @buckets << bucket_obj @bucket_hash[bucket_name] = bucket_obj end end # Pass a rate limit in bytes per second def rate_limit=(rate_limit) if rate_limit.is_a?(String) if rate_limit =~ /^(\d+)$/ RateLimitableFile.rate_limit = rate_limit.to_i elsif rate_limit =~ /^(.*)K$/ RateLimitableFile.rate_limit = $1.to_f * 1000 elsif rate_limit =~ /^(.*)M$/ RateLimitableFile.rate_limit = $1.to_f * 1000000 elsif rate_limit =~ /^(.*)G$/ RateLimitableFile.rate_limit = $1.to_f * 1000000000 else raise "Invalid Rate Limit Format: Valid values include (1000,10K,1.1M)" end else RateLimitableFile.rate_limit = nil end end def buckets @buckets end def get_bucket_folder(bucket) File.join(@root, bucket.name) end def get_bucket(bucket) @bucket_hash[bucket] end def create_bucket(bucket) FileUtils.mkdir_p(File.join(@root, bucket)) bucket_obj = Bucket.new(bucket, Time.now, []) if !@bucket_hash[bucket] @buckets << bucket_obj @bucket_hash[bucket] = bucket_obj end bucket_obj end def delete_bucket(bucket_name) bucket = get_bucket(bucket_name) raise NoSuchBucket if !bucket raise BucketNotEmpty if bucket.objects.count > 0 FileUtils.rm_r(get_bucket_folder(bucket)) @bucket_hash.delete(bucket_name) end def get_object(bucket, object_name, request) begin real_obj = S3Object.new obj_root = File.join(@root,bucket,object_name,FAKE_S3_METADATA_DIR) metadata = File.open(File.join(obj_root, "metadata")) { |file| YAML::load(file) } real_obj.name = object_name real_obj.md5 = metadata[:md5] real_obj.content_type = metadata.fetch(:content_type) { "application/octet-stream" } real_obj.content_encoding = metadata.fetch(:content_encoding) real_obj.io = RateLimitableFile.open(File.join(obj_root, "content"), 'rb') real_obj.size = metadata.fetch(:size) { 0 } real_obj.creation_date = File.ctime(obj_root).utc.iso8601(SUBSECOND_PRECISION) real_obj.modified_date = metadata.fetch(:modified_date) do File.mtime(File.join(obj_root, "content")).utc.iso8601(SUBSECOND_PRECISION) end real_obj.custom_metadata = metadata.fetch(:custom_metadata) { {} } return real_obj rescue puts $! $!.backtrace.each { |line| puts line } return nil end end def object_metadata(bucket, object) end def copy_object(src_bucket_name, src_name, dst_bucket_name, dst_name, request) begin src_root = File.join(@root,src_bucket_name,src_name,FAKE_S3_METADATA_DIR) src_metadata_filename = File.join(src_root, "metadata") src_metadata = File.open(src_metadata_filename) { |file| YAML::load(file) } src_content_filename = File.join(src_root, "content") dst_filename= File.join(@root,dst_bucket_name,dst_name) FileUtils.mkdir_p(dst_filename) metadata_dir = File.join(dst_filename,FAKE_S3_METADATA_DIR) FileUtils.mkdir_p(metadata_dir) content = File.join(metadata_dir, "content") metadata = File.join(metadata_dir, "metadata") if src_bucket_name != dst_bucket_name || src_name != dst_name File.open(content, 'wb') do |f| File.open(src_content_filename, 'rb') do |input| f << input.read end end File.open(metadata,'w') do |f| File.open(src_metadata_filename,'r') do |input| f << input.read end end end metadata_directive = request.header["x-amz-metadata-directive"].first if metadata_directive == "REPLACE" metadata_struct = create_metadata(content,request) File.open(metadata,'w') do |f| f << YAML::dump(metadata_struct) end end src_bucket = get_bucket(src_bucket_name) || create_bucket(src_bucket_name) dst_bucket = get_bucket(dst_bucket_name) || create_bucket(dst_bucket_name) obj = S3Object.new obj.name = dst_name obj.md5 = src_metadata[:md5] obj.content_type = src_metadata[:content_type] obj.content_encoding = src_metadata[:content_encoding] obj.size = src_metadata[:size] obj.modified_date = src_metadata[:modified_date] src_bucket.find(src_name) dst_bucket.add(obj) return obj rescue puts $! $!.backtrace.each { |line| puts line } return nil end end def store_object(bucket, object_name, request) filedata = "" # TODO put a tmpfile here first and mv it over at the end content_type = request.content_type || "" match = content_type.match(/^multipart\/form-data; boundary=(.+)/) boundary = match[1] if match if boundary boundary = WEBrick::HTTPUtils::dequote(boundary) form_data = WEBrick::HTTPUtils::parse_form_data(request.body, boundary) if form_data['file'] == nil || form_data['file'] == "" raise WEBrick::HTTPStatus::BadRequest end filedata = form_data['file'] else request.body { |chunk| filedata << chunk } end do_store_object(bucket, object_name, filedata, request) end def do_store_object(bucket, object_name, filedata, request) begin filename = File.join(@root, bucket.name, object_name) FileUtils.mkdir_p(filename) metadata_dir = File.join(filename, FAKE_S3_METADATA_DIR) FileUtils.mkdir_p(metadata_dir) content = File.join(filename, FAKE_S3_METADATA_DIR, "content") metadata = File.join(filename, FAKE_S3_METADATA_DIR, "metadata") File.open(content,'wb') { |f| f << filedata } metadata_struct = create_metadata(content, request) File.open(metadata,'w') do |f| f << YAML::dump(metadata_struct) end obj = S3Object.new obj.name = object_name obj.md5 = metadata_struct[:md5] obj.content_type = metadata_struct[:content_type] obj.content_encoding = metadata_struct[:content_encoding] obj.size = metadata_struct[:size] obj.modified_date = metadata_struct[:modified_date] bucket.add(obj) return obj rescue puts $! $!.backtrace.each { |line| puts line } return nil end end def combine_object_parts(bucket, upload_id, object_name, parts, request) upload_path = File.join(@root, bucket.name) base_path = File.join(upload_path, "#{upload_id}_#{object_name}") complete_file = "" chunk = "" part_paths = [] parts.sort_by { |part| part[:number] }.each do |part| part_path = "#{base_path}_part#{part[:number]}" content_path = File.join(part_path, FAKE_S3_METADATA_DIR, 'content') File.open(content_path, 'rb') { |f| chunk = f.read } etag = Digest::MD5.hexdigest(chunk) if part[:etag] == "\"#{etag}\"" && part[:etag] == etag raise "invalid file chunk: p='#{part[:etag]}', comp='#{etag}'" end complete_file << chunk part_paths << part_path end object = do_store_object(bucket, object_name, complete_file, request) # clean up parts part_paths.each do |path| FileUtils.remove_dir(path) end object end def delete_object(bucket,object_name) begin filename = File.join(@root,bucket.name,object_name) FileUtils.rm_rf(filename) object = bucket.find(object_name) bucket.remove(object) rescue puts $! $!.backtrace.each { |line| puts line } return nil end end # TODO: abstract getting meta data from request. def create_metadata(content, request) metadata = {} metadata[:md5] = Digest::MD5.file(content).hexdigest metadata[:content_type] = request.header["content-type"].first metadata[:content_encoding] = request.header["content-encoding"].first metadata[:size] = File.size(content) metadata[:modified_date] = File.mtime(content).utc.iso8601(SUBSECOND_PRECISION) metadata[:amazon_metadata] = {} metadata[:custom_metadata] = {} # Add custom metadata from the request header request.header.each do |key, value| match = /^x-amz-([^-]+)-(.*)$/.match(key) next unless match if match[1].eql?('meta') && (match_key = match[2]) metadata[:custom_metadata][match_key] = value.join(', ') next end metadata[:amazon_metadata][key.gsub(/^x-amz-/, '')] = value.join(', ') end return metadata end end end