=begin = mys3.rb *Copyright*:: (C) 2013 by Novu, LLC *Author(s)*:: Tamara Temple *Since*:: 2013-05-01 *License*:: MIT *Version*:: 0.0.1 == Description Upload the raw file and the clean file up to AWS S3 =end require 'methadone' module NewBackup class MyS3 include Methadone::CLILogging # Maximum number of attempts to try to upload file MAX_TRIES = 3 # Initialize the MyS3 object # # options:: pass in the program options def initialize(options={}) @options = options end # Connect to S3, cache the connection, and yield self def connect(&block) options = { :aws_access_key_id => @options[:aws][:access_key], :aws_secret_access_key => @options[:aws][:secret_key], :region => @options[:s3][:region], :provider => 'AWS', :scheme => 'https'} connection = Fog::Storage.new(options) yield connection end # Connect to specific S3 bucket # # == inputs # *connection*:: S3 connection to use # *bucket_name*:: name of the bucket to use in S3 # *&block*:: block passed to evaluate in context of bucket # (If no block given simply return the bucket pointer.) def connect_bucket(connection, bucket_name, &block) debug "#{self.class}##{__method__}:#{File.basename(__FILE__)}:#{__LINE__}: connection: #{connection.inspect}" info "Connecting to #{bucket_name}" buckets = connection.directories raise "No buckets!" if buckets.nil? || buckets.empty? bucket = connection.directories.get(bucket_name) raise "No bucket #{bucket_name}" if bucket.nil? if block_given? yield bucket else bucket end end # Do the heavy lifing to put the file in the appropriate bucket # # bucket:: directory where to put the file # fn:: name of file to upload (gzipped) def put_file(bucket,fn) info "Saving #{fn} to #{bucket.key}" s3_fn = File.join(@options[:s3][:prefix], File.basename(fn)) options = { :key => s3_fn, :body => File.open(fn,'r'), :acl => 'authenticated-read', :encryption => 'AES256', :content_type => 'application/x-gzip'} bucket.files.create(options) # Verify that file was uploaded files = bucket.files.all.select {|f| f.key == s3_fn } raise "#{fn} was not uploaded to #{s3_fn}!" unless files.count > 0 "s3://#{bucket.key}/#{s3_fn}" end # Prune extra files def prune(bucket, keep=0) if keep > 0 files = bucket.files.all('prefix' => @options[:s3][:prefix]) return if files.nil? || files.empty? if files.count > keep info "Pruning down to #{keep} files" files.sort {|x,y| x.last_modified <=> y.last_modified}. take(files.count - keep). map(&:destroy) end end files = bucket.files.all('prefix' => @options[:s3][:prefix]) files.count end end end