require 'json' require 'net/http' require 'fileutils' require 'digest/md5' module Filbunke class Client attr_reader :repository UPDATES_ACTION = 'updates' FILES_ACTION = 'files' URL_KEY = 'url' FROM_CHECKPOINT_KEY = 'from_checkpoint' HASH_KEY = 'hash' def initialize(repository, logger, callbacks = []) @repository = repository @logger = logger @callbacks = callbacks end def with_updated_files(last_checkpoint) updates = get_updated_file_list(last_checkpoint) updated_files = updates["files"] || [] failure = false new_checkpoint = update["checkpoint"] @logger.log "Updating repository: #{repository.name}: #{updated_files.size} files. Checkpoint: #{last_checkpoint} ==> #{new_checkpoint}" if updated_files.size > 0 updated_files.each do |raw_file| file = File.new(raw_file) local_file_path = ::File.join(repository.local_path, file.path) if file_needs_update?(file, local_file_path) if file.url =~ /^http:\/\// if update_http_file!(file, local_file_path) then @callbacks.each do |callback| callback.on_update(file) end yield file else @logger.log "Unable to get file #{file.url} ==> #{file.path}!" failure = true end else raise "Unsupported protocol for file: #{file.inspect}" end end end failure ? last_checkpoint : (new_checkpoint || last_checkpoint) end def update_files!(last_checkpoint) with_updated_files(last_checkpoint) {} end def register_updated_file!(path, url, hash = nil) register_http = Net::HTTP.new(@repository.host, @repository.port) register_http.start do |http| register_path = "/#{FILES_ACTION}/#{@repository.name}/#{path}?#{URL_KEY}=#{url}" register_path += "&#{HASH_KEY}=#{hash}" if hash request = Net::HTTP::Put.new(register_path) response = http.request(request) if response.code.to_i != 204 raise "Failed to register updated file: #{path}" end end end def register_deleted_file!(path) register_http = Net::HTTP.new(@repository.host, @repository.port) register_http.start do |http| register_path = "/#{FILES_ACTION}/#{@repository.name}/#{path}" request = Net::HTTP::Delete.new(register_path) response = http.request(request) if response.code.to_i != 204 raise "Failed to register deleted file: #{path}" end end end private def file_needs_update?(file, local_file_path) return true if file.hash.nil? || file.hash == "" return true unless File.exists?(local_file_path) local_hash = Digest::MD5.hexdigest(File.read(local_file_path)) local_hash != file.hash end def get_updated_file_list(last_checkpoint) begin updates_http = Net::HTTP.new(@repository.host, @repository.port) updates_http.start do |http| begin updates_path = "/#{UPDATES_ACTION}/#{@repository.name}?#{FROM_CHECKPOINT_KEY}=#{last_checkpoint}" request = Net::HTTP::Get.new(updates_path) response = http.request(request) if response.code.to_i == 200 JSON.parse(response.body) else @logger.log "Failed to download updates for #{@repository.name}, error code = #{response.code}" {} end rescue StandardError => e @logger.log "Error getting file list: #{e.message}! Retrying later.." {} end end rescue StandardError => e @logger.log "Unable to create HTTP connection to #{@repository.host}:#{@repository.port} (#{e.message})!" return {} end end def update_http_file!(file, local_file_path) begin if file.state == "DELETED" then delete_file!(local_file_path) return true end uri = URI.parse(file.url) file_http=Net::HTTP.new(uri.host, uri.port) file_http.start do |http| request = Net::HTTP::Get.new(uri.path) request.basic_auth @repository.user, @repository.pass if @repository.user response = http.request(request) if response.code.to_i == 200 write_file!(local_file_path, response.body) elsif response.code.to_i == 404 delete_file!(local_file_path) else @logger.log "Failed to update file #{uri}, error code = #{response.code}" end return true end rescue StandardError => e @logger.log "Failed to update file #{uri}: #{e.message}" return false end end def write_file!(file_path, contents) @logger.log("Writing: #{file_path}") ::FileUtils.mkdir_p(::File.dirname(file_path)) ::File.open(file_path, 'w') do |file| file.write(contents); file.close end end def delete_file!(file_path) if ::File.exists?(file_path) then @logger.log("Deleting: #{file_path}") ::File.delete(file_path) end end end end