#!/usr/bin/env ruby # This software code is made available "AS IS" without warranties of any # kind. You may copy, display, modify and redistribute the software # code either by itself or as incorporated into your code; provided that # you do not remove any proprietary notices. Your use of this software # code is at your own risk and you waive any claim against the author # with respect to your use of this software code. # (c) 2007 s3sync.net # module S3sync $AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"] $AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"] $AWS_S3_HOST = (ENV["AWS_S3_HOST"] or "s3.amazonaws.com") $HTTP_PROXY_HOST = ENV["HTTP_PROXY_HOST"] $HTTP_PROXY_PORT = ENV["HTTP_PROXY_PORT"] $HTTP_PROXY_USER = ENV["HTTP_PROXY_USER"] $HTTP_PROXY_PASSWORD = ENV["HTTP_PROXY_PASSWORD"] $SSL_CERT_DIR = ENV["SSL_CERT_DIR"] $SSL_CERT_FILE = ENV["SSL_CERT_FILE"] $S3SYNC_RETRIES = (ENV["S3SYNC_RETRIES"] or 100).to_i # number of errors to tolerate $S3SYNC_WAITONERROR = (ENV["S3SYNC_WAITONERROR"] or 30).to_i # seconds $S3SYNC_NATIVE_CHARSET = (ENV["S3SYNC_NATIVE_CHARSET"] or "ISO-8859-1") $AWS_CALLING_FORMAT = (ENV["AWS_CALLING_FORMAT"] or "REGULAR") require 'S3' require 'HTTPStreaming' require 'S3encoder' CGI::exemptSlashesInEscape = true CGI::usePercent20InEscape = true CGI::useUTF8InEscape = true CGI::nativeCharacterEncoding = $S3SYNC_NATIVE_CHARSET require 'S3_s3sync_mod' $S3syncRetriesLeft = $S3SYNC_RETRIES.to_i def S3sync.s3trySetup # ---------- CONNECT ---------- # $S3syncConnection = S3::AWSAuthConnection.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST) $S3syncConnection.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT) if $S3syncOptions['--ssl'] if $SSL_CERT_DIR $S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER $S3syncConnection.ca_path = $SSL_CERT_DIR elsif $SSL_CERT_FILE $S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER $S3syncConnection.ca_file = $SSL_CERT_FILE end end end def S3sync.s3urlSetup $S3syncGenerator = S3::QueryStringAuthGenerator.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST) $S3syncGenerator.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT) $S3syncGenerator.expires_in = $S3syncOptions['--expires-in'] end def S3sync.S3tryConnect(bucket, host='') $S3syncHttp = $S3syncConnection.make_http(bucket, host, $HTTP_PROXY_HOST, $HTTP_PROXY_PORT, $HTTP_PROXY_USER, $HTTP_PROXY_PASSWORD) end def S3sync.S3try(command, bucket, *args) if(not $S3syncHttp or (bucket != $S3syncLastBucket)) $stderr.puts "Creating new connection" if $S3syncOptions['--debug'] $S3syncLastBucket = bucket S3sync.S3tryConnect(bucket) end result = nil delim = $, $,=' ' while $S3syncRetriesLeft > 0 do $stderr.puts "Trying command #{command} #{bucket} #{args} with #{$S3syncRetriesLeft} retries left" if $S3syncOptions['--debug'] forceRetry = false now = false hush = false begin result = $S3syncConnection.send(command, bucket, *args) rescue Errno::EPIPE => e forceRetry = true $stderr.puts "Broken pipe: #{e}" rescue Errno::ECONNRESET => e forceRetry = true $stderr.puts "Connection reset: #{e}" rescue Errno::ECONNABORTED => e forceRetry = true $stderr.puts "Connection aborted: #{e}" rescue Errno::ETIMEDOUT => e forceRetry = true $stderr.puts "Connection timed out: #{e}" rescue Timeout::Error => e forceRetry = true $stderr.puts "Connection timed out: #{e}" rescue EOFError => e # i THINK this is happening like a connection reset forceRetry = true $stderr.puts "EOF error: #{e}" rescue OpenSSL::SSL::SSLError => e forceRetry = true $stderr.puts "SSL Error: #{e}" rescue NoMethodError # we get this when using --progress, and the local item is something unreadable $stderr.puts "Null stream error: #{e}" break end if forceRetry # kill and reset connection when we receive a non-50x yet retry-able error S3sync.S3tryConnect(bucket) end begin debug("Response code: #{result.http_response.code}") break if ((200...300).include? result.http_response.code.to_i) and (not forceRetry) if result.http_response.code.to_i == 301 $stderr.puts "Permanent redirect received. Try setting AWS_CALLING_FORMAT to SUBDOMAIN" elsif result.http_response.code.to_i == 307 # move to the other host host = %r{https?://([^/]+)}.match(result.http_response['Location'])[1] $stderr.puts("Temporary Redirect to: " + host) debug("Host: " + host) S3sync.S3tryConnect(bucket, host) $S3syncRetriesLeft = $S3syncRetriesLeft+1 # don't use one up below forceRetry = true now = true hush = true else $stderr.puts "S3 command failed:\n#{command} #{args}" $stderr.puts "With result #{result.http_response.code} #{result.http_response.message}\n" debug(result.http_response.body) end # only retry 500's, per amazon break unless ((500...600).include? result.http_response.code.to_i) or forceRetry rescue NoMethodError debug("No result available") end $S3syncRetriesLeft -= 1 $stderr.puts "#{$S3syncRetriesLeft} retries left" unless hush Kernel.sleep $S3SYNC_WAITONERROR unless now end if $S3syncRetriesLeft <= 0 $stderr.puts "Ran out of retries; operations did not complete!" end $, = delim result end def S3sync.S3url(command, bucket, *args) S3sync.s3urlSetup() unless $S3syncGenerator result = nil delim = $, $,=' ' $stderr.puts "Calling command #{command} #{bucket} #{args}" if $S3syncOptions['--debug'] result = $S3syncGenerator.send(command, bucket, *args) $, = delim result end end #module