lib/s3backup/manager.rb in s3backup-0.7.3 vs lib/s3backup/manager.rb in s3backup-0.8.1
- old
+ new
@@ -1,7 +1,7 @@
require 'cgi'
-require 'tempfile'
+require 'time'
require 'fileutils'
require 's3backup/s3log'
require 's3backup/tree_info'
require 's3backup/crypt'
module S3backup
@@ -20,10 +20,11 @@
return a.join;
end
def initialize(target,config)
@target = target
@resume = false
+ @temporary = "/tmp"
set_config(config)
end
def set_config(config)
if config["password"] and config["password"] != ""
unless config["salt"]
@@ -47,39 +48,55 @@
exit(-1)
end
else
@buf_size = DEFAULT_BUF_READ_SIZE
end
- if config["buffer_size"]
- if config["buffer_size"].class == String
- @buf_size = config["buffer_size"].to_i
- else
- @buf_size = config["buffer_size"]
- end
- if @buf_size > 1000*1000*1000*5
- S3log.error("buffer_size must be less than 5G\n")
- exit(-1)
- end
- else
- @buf_size = DEFAULT_BUF_READ_SIZE
+ if config["temporary"]
+ @temporary = config["temporary"]
end
if config["resume"] == true
@resume = true
end
end
+ def to_gz(file,remain=false)
+ if remain
+ cmd = "(cd #{shell_name(File.dirname(file))};gzip -c #{shell_name(file)} > #{shell_name(file)}.gz " +
+ "2>/dev/null)"
+ else
+ cmd = "(cd #{shell_name(File.dirname(file))};gzip #{shell_name(file)} > /dev/null 2>&1)"
+ end
+ S3log.debug(cmd)
+ system(cmd)
+ unless $?.success?
+ S3log.error("feiled #{cmd} execute. #{$?.inspect}")
+ exit(-1)
+ end
+ end
+ def from_gz(file)
+ cmd = "(cd #{shell_name(File.dirname(file))};gunzip #{shell_name(file)} > /dev/null 2>&1)"
+ S3log.debug(cmd)
+ system(cmd)
+ unless $?.success?
+ S3log.error("feiled #{cmd} execute. #{$?.inspect}")
+ exit(-1)
+ end
+ end
#指定されたディレクトリをtar gzip形式で圧縮する
def to_tgz(path,dir)
#サブディレクトリを圧縮の対象外にする。
sub_dir = []
Dir.foreach(dir) do |file|
next if /^\.+$/ =~ file
sub_dir.push(file) if File.directory?(dir+"/"+file)
end
exclude = ""
- exclude = exclude + " --exclude=" + sub_dir.map{|d| shell_name(d)}.join(" --exclude=") if sub_dir.length != 0
- cmd = "(cd #{shell_name(File.dirname(dir))};tar -czvf #{shell_name(path)} #{exclude} -- #{shell_name(File.basename(dir))}" +
- " > /dev/null 2>&1)"
+ if sub_dir.length != 0
+ exclude = " --exclude=#{shell_name(File.basename(dir))}/" + sub_dir.map{|d| shell_name(d)}.join(
+ " --exclude=#{shell_name(File.basename(dir))}/")
+ end
+ cmd = "(cd #{shell_name(File.dirname(dir))};tar -czvf #{shell_name(path)} #{exclude} -- " +
+ "#{shell_name(File.basename(dir))} > /dev/null 2>&1)"
S3log.debug(cmd)
system(cmd)
unless $?.success?
S3log.error("feiled #{cmd} execute. #{$?.inspect}")
exit(-1)
@@ -92,42 +109,37 @@
unless $?.success?
S3log.error("feiled #{cmd} execute. #{$?.inspect}")
exit(-1)
end
end
- def get_chain(key)
+ def get_chain(key,path)
data = nil
- data_set = nil
i=1
if @aes
key = @aes.encrypt(key)
end
- while 1
- key_name = i.to_s()+"_"+key
- data = @target.get(key_name)
- if data == nil
- break
+ File.open(path,"w") do |f|
+ while 1
+ key_name = i.to_s()+"_"+key
+ data = @target.get(key_name)
+ if data == nil
+ break
+ end
+ if @aes
+ data = @aes.decrypt(data)
+ end
+ f.write(data)
+ i+=1
end
- if i==1
- data_set = ''
- end
- if @aes
- data = @aes.decrypt(data)
- end
- data_set += data
- i+=1
end
- return data_set
end
def get_directory(dir,out_dir)
- data = get_chain(dir)
- tmp = Tempfile.open("s3backup")
- tmp.write(data)
- tmp.close
+ file_name = @temporary + "/"+ CGI.escape(dir)
+ get_chain(dir,file_name + ".tgz")
#tgzのファイルをcur_dirに展開
- from_tgz(tmp.path,out_dir)
- tmp.close(true)
+ from_tgz(file_name + ".tgz",out_dir)
+ #File.unlink(file_name)
end
def get_directories(dirs,prefix,output_dir)
prefix_len = prefix.length
dirs.each do |dir|
parent = File.dirname(dir)
@@ -136,14 +148,13 @@
cur_dir = output_dir + relative_path
get_directory(dir,cur_dir)
end
end
def store_directory(dir)
- tmp = Tempfile.open("s3backup")
- tmp.close
+ tmp_file = @temporary + "/bk_" + CGI.escape(dir)
#tgzのファイルをtmp.pathに作成
- to_tgz(tmp.path,dir)
+ to_tgz(tmp_file,dir)
#S3にディレクトリの絶対パスをキーにして、圧縮したデータをストア
i=1
key = nil
if @aes
key = @aes.encrypt(dir)
@@ -154,24 +165,25 @@
cnt = 1
while @target.exists?(cnt.to_s() + "_" + key)
@target.delete(cnt.to_s() + "_" + key)
cnt+=1
end
- f = File.open(tmp.path,"r")
- begin
- while 1
- key_name = i.to_s()+"_"+key
- data = f.readpartial(@buf_size)
- if @aes
- data = @aes.encrypt(data)
+ File.open(tmp_file,"r") do |f|
+ begin
+ while 1
+ key_name = i.to_s()+"_"+key
+ data = f.readpartial(@buf_size)
+ if @aes
+ data = @aes.encrypt(data)
+ end
+ @target.post(key_name,data)
+ i+=1
end
- @target.post(key_name,data)
- i+=1
+ rescue EOFError
end
- rescue EOFError
end
- tmp.close(true)
+ File.unlink(tmp_file)
end
def delete_direcory(dir)
if @aes
dir = @aes.encrypt(dir)
end
@@ -180,45 +192,60 @@
i+=1
end
end
def differential_copy(dir)
#現在のファイル・ツリーを比較
- tree_info = TreeInfo.new(dir)
-
- target_tree_name = "tree_"+dir+".yml"
- tree_data = nil
+ tree_info = TreeInfo.new({:format=>:directory,:directory=>dir,:db=>@temporary + "/new_" +
+ Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"})
+ target_db_name = dir+".gz"
#前回のファイル・ツリーを取得
- old_tree = TreeInfo.new(@target.get(target_tree_name))
-
- #前回と今回のファイル・ツリーを比較
- diff_info = tree_info.diff(old_tree)
- S3log.debug("diff_info=#{diff_info.inspect}")
- dir_map = nil
- if @resume
- new_dir_map = tree_info.make_dir_map
- old_dir_map = old_tree.make_dir_map
+ data = @target.get(target_db_name)
+ old_tree = nil
+ if data
+ db_name = @temporary + "/old_" + Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"
+ File.open(db_name + ".gz","w") do |f|
+ f.write(data)
+ end
+ from_gz(db_name + ".gz")
+ old_tree = TreeInfo.new({:format=>:database,:db=>db_name})
else
- #メモリ節約のため開放
- old_tree = nil
+ target_tree_name = "tree_"+dir+".yml"
+ #以前のフォーマットだった場合は変換
+ data = @target.get(target_tree_name)
+ if data
+ old_tree = TreeInfo.new({:format=>:yaml,:data=>data,:db=>@temporary + "/old_" +
+ Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"})
+ else
+ old_tree = TreeInfo.new({:db=>@temporary + "/old_" +
+ Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"})
+ end
end
- update_dir = diff_info[:directory][:add] + diff_info[:directory][:modify]
- #更新されたディレクトリをアップロード
- update_dir.each do |udir|
- GC.start
- store_directory(udir)
- if @resume
- #前回のファイル・ツリー情報のうち、今回アップデートしたディレクトリ情報ファイル情報を更新
- old_dir_map = old_tree.update_dir(udir,old_dir_map,new_dir_map[udir])
+ data = nil;
+ GC.start
+ cnt=0
+ #前回と今回のファイル・ツリーを比較
+ tree_info.modify(old_tree) do |dir_info|
+ cnt+=1
+ S3log.debug("diff_info=#{dir_info[:name]}")
+ #更新されたディレクトリをアップロード
+ store_directory(dir_info[:name])
+ #前回のファイル・ツリー情報のうち、今回アップデートしたディレクトリ情報ファイル情報を更新
+ old_dir_map = old_tree.update_dir(dir_info)
+ if cnt != 0 and cnt % 10 == 0
#更新したファイル・ツリー情報をアップロード(途中で失敗しても、resumeできるようにするため。)
- @target.post(target_tree_name,old_tree.dump_yaml)
+ to_gz(old_tree.db_name,true)
+ @target.post(target_db_name,File.read(old_tree.db_name + ".gz"))
end
end
- diff_info[:directory][:remove].each do |rm_dir|
- delete_direcory(rm_dir)
+ tree_info.remove(old_tree) do |dir_info|
+ delete_direcory(dir_info[:name])
end
#今回のファイル・ツリーをAWS S3に登録
- @target.post(target_tree_name,tree_info.dump_yaml)
+ to_gz(tree_info.db_name)
+ @target.post(target_db_name,File.read(tree_info.db_name + ".gz"))
+ tree_info.close(true)
+ old_tree.close(true)
end
def get_target_tree(dir)
base_dir = dir
tree_data = nil
before_base=""
@@ -226,31 +253,27 @@
while 1
base = base_dir
if base == before_base
break
end
- tree_file_name = "tree_"+base+".yml"
- tree_data = @target.get(tree_file_name)
+ tree_db_name = base+".gz"
+ tree_data = @target.get(tree_db_name)
if tree_data
break
end
before_base = base
base_dir = File.dirname(base_dir)
end
unless tree_data
return nil
end
- return TreeInfo.new(tree_data)
- end
- def get_target_bases
- files = @target.find(/^tree_.*\.yml/)
- dirs = files.map do |d|
- m=/tree_(.*)\.yml/.match(d)
- next nil unless m
- m[1]
+ db_name = @temporary + "/" + Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"
+ File.open(db_name + ".gz","w") do |f|
+ f.write(tree_data)
end
- return dirs.compact
+ from_gz(db_name + ".gz")
+ return TreeInfo.new({:format=>:database,:db=>db_name})
end
def expand_tree(dir,tree_info,output_dir)
now = Time.new
tree = tree_info.hierarchie(dir)
top = tree[0].keys[0]
@@ -265,20 +288,20 @@
(tree.length - 1).downto(0){|n|
tree[n].each do |k,v|
dir_len = k.length
relative_path = k.slice(top_dir_len,dir_len - top_dir_len)
dir = output_dir + relative_path
- File.utime(now,v[:mtime],dir)
+ File.utime(now,Time.parse(v[:mtime]),dir)
end
}
end
def restore(dir,output_dir)
tree = get_target_tree(dir)
unless tree
S3log.warn("#{dir} isn't find in AWS S3. ignore")
return
end
expand_tree(dir,tree,output_dir)
- S3log.debug("expand_tree=#{tree.inspect}")
+ tree.close(true)
end
end
end