Sha256: 72cdd656c526d18ffb6b411993ccf3ffe8cf71afffb312a8bf0ea87d0b4410c7
Contents?: true
Size: 1.92 KB
Versions: 1
Compression:
Stored size: 1.92 KB
Contents
require 'fileutils' require 'digest' module MLS::CLI::Documents def self.partition(value, depth: 5) split = value.scan(/.{1,4}/) split.shift(depth).join("/") + split.join("") end def self.calculate_digest(file) md5_digest = Digest::MD5.new sha1_digest = Digest::SHA1.new sha256_digest = Digest::SHA256.new buf = "" file.rewind while file.read(16384, buf) md5_digest << buf sha1_digest << buf sha256_digest << buf end { md5: md5_digest.hexdigest, sha1: sha1_digest.hexdigest, sha256: sha256_digest.hexdigest } end def self.backup(dir) last_sync_file = File.join(dir, '.last_sync') query = if File.exist?(last_sync_file) from_timestamp = Time.iso8601(File.read(last_sync_file)) Document.where(Document.arel_table['created_at'].gteq(from_timestamp)) else Document.all end query.find_each do |document| if document.sha256 && File.exists?(File.join(dir, partition(document.sha256))) puts "Downloaded #{document.id}" next end if document.provider.nil? || document.provider.include?('s3/hash_key') storage_engine = StandardStorage::S3.new(MLS::CLI.options[:s3]) key = 'hash_key' else raise 'unkown storage engine' end print "Downloading #{document.id.to_s.ljust(7)} " storage_engine.copy_to_tempfile(document.send(key)) do |file| digests = calculate_digest(file) puts partition(document.sha256) raise 'MD5 does not match' if digests[:md5] != document.md5 document.update!(digests.merge({provider: ['s3/hash_key']})) FileUtils.mkdir_p(File.dirname(File.join(dir, partition(document.sha256)))) FileUtils.mv(file.path, File.join(dir, partition(document.sha256))) end File.write(last_sync_file, document.created_at.iso8601(6)) end end end
Version data entries
1 entries across 1 versions & 1 rubygems
Version | Path |
---|---|
mls-1.8.0 | lib/mls/cli/documents.rb |