lib/fig/repository.rb in fig-0.1.62 vs lib/fig/repository.rb in fig-0.1.64

- old
+ new

@@ -1,363 +1,608 @@ +require 'set' require 'socket' require 'sys/admin' +require 'tmpdir' +require 'fig/atexit' +require 'fig/command' require 'fig/logging' require 'fig/notfounderror' require 'fig/packagecache' require 'fig/packagedescriptor' require 'fig/parser' require 'fig/repositoryerror' require 'fig/statement/archive' require 'fig/statement/resource' require 'fig/urlaccesserror' -module Fig - # Overall management of a repository. Handles local operations itself; - # defers remote operations to others. - class Repository - METADATA_SUBDIRECTORY = '_meta' +module Fig; end - def self.is_url?(url) - not (/ftp:\/\/|http:\/\/|file:\/\/|ssh:\/\// =~ url).nil? - end +# Overall management of a repository. Handles local operations itself; +# defers remote operations to others. +class Fig::Repository + METADATA_SUBDIRECTORY = '_meta' + RESOURCES_FILE = 'resources.tar.gz' + VERSION_FILE_NAME = 'repository-format-version' + VERSION_SUPPORTED = 1 - def initialize( - os, - local_repository_dir, - remote_repository_url, - application_config, - remote_repository_user, - update, - update_if_missing, - check_include_versions - ) - @operating_system = os - @local_repository_dir = local_repository_dir - @remote_repository_url = remote_repository_url - @remote_repository_user = remote_repository_user - @update = update - @update_if_missing = update_if_missing + def self.is_url?(url) + not (/ftp:\/\/|http:\/\/|file:\/\/|ssh:\/\// =~ url).nil? + end - @parser = Parser.new(application_config, check_include_versions) + def initialize( + os, + local_repository_directory, + application_config, + remote_repository_user, + update, + update_if_missing, + check_include_versions + ) + @operating_system = os + @local_repository_directory = local_repository_directory + @application_config = application_config + @remote_repository_user = remote_repository_user + @update = update + @update_if_missing = update_if_missing - reset_cached_data() - end + @parser = Fig::Parser.new(application_config, check_include_versions) - def reset_cached_data() - @packages = PackageCache.new() - end + initialize_local_repository() + reset_cached_data() + end - def list_packages - results = [] - if File.exist?(@local_repository_dir) - @operating_system.list(@local_repository_dir).each do |name| - @operating_system.list(File.join(@local_repository_dir, name)).each do |version| - results << PackageDescriptor.format(name, version, nil) - end + def reset_cached_data() + @package_cache = Fig::PackageCache.new() + end + + def list_packages + check_local_repository_format() + + results = [] + if File.exist?(local_package_directory()) + @operating_system.list(local_package_directory()).each do |name| + @operating_system.list(File.join(local_package_directory(), name)).each do + |version| + results << Fig::PackageDescriptor.format(name, version, nil) end end - - return results end - def list_remote_packages - paths = @operating_system.download_list(@remote_repository_url) + return results + end - return paths.reject { |path| path =~ %r< ^ #{METADATA_SUBDIRECTORY} / >xs } - end + def list_remote_packages + check_remote_repository_format() - def get_package( - descriptor, - allow_any_version = false - ) - if not descriptor.version - if allow_any_version - package = @packages.get_any_version_of_package(descriptor.name) - if package - Logging.warn( - "Picked version #{package.version} of #{package.name} at random." - ) - return package - end - end + paths = @operating_system.download_list(remote_repository_url()) - raise RepositoryError.new( - %Q<Cannot retrieve "#{descriptor.name}" without a version.> - ) + return paths.reject { |path| path =~ %r< ^ #{METADATA_SUBDIRECTORY} / >xs } + end + + def get_package( + descriptor, + allow_any_version = false + ) + check_local_repository_format() + + if not descriptor.version + if allow_any_version + package = @package_cache.get_any_version_of_package(descriptor.name) + if package + Fig::Logging.warn( + "Picked version #{package.version} of #{package.name} at random." + ) + return package + end end - package = @packages.get_package(descriptor.name, descriptor.version) - return package if package + raise Fig::RepositoryError.new( + %Q<Cannot retrieve "#{descriptor.name}" without a version.> + ) + end - Logging.debug \ - "Considering #{PackageDescriptor.format(descriptor.name, descriptor.version, nil)}." + package = @package_cache.get_package(descriptor.name, descriptor.version) + return package if package - if should_update?(descriptor) - update_package(descriptor) - end + Fig::Logging.debug \ + "Considering #{Fig::PackageDescriptor.format(descriptor.name, descriptor.version, nil)}." - return read_local_package(descriptor) + if should_update?(descriptor) + check_remote_repository_format() + + update_package(descriptor) end - def clean(descriptor) - @packages.remove_package(descriptor.name, descriptor.version) + return read_local_package(descriptor) + end - dir = File.join(@local_repository_dir, descriptor.name) - dir = File.join(dir, descriptor.version) if descriptor.version + def clean(descriptor) + check_local_repository_format() - FileUtils.rm_rf(dir) + @package_cache.remove_package(descriptor.name, descriptor.version) - return + FileUtils.rm_rf(local_dir_for_package(descriptor)) + + return + end + + def publish_package(package_statements, descriptor, local_only) + check_local_repository_format() + if not local_only + check_remote_repository_format() end - def publish_package(package_statements, descriptor, local_only) - temp_dir = temp_dir_for_package(descriptor) - @operating_system.clear_directory(temp_dir) - local_dir = local_dir_for_package(descriptor) - @operating_system.clear_directory(local_dir) - fig_file = File.join(temp_dir, '.fig') - content = derive_package_content( - package_statements, descriptor, local_dir, local_only - ) - @operating_system.write(fig_file, content.join("\n").strip) + validate_asset_names(package_statements) + + temp_dir = publish_temp_dir() + @operating_system.delete_and_recreate_directory(temp_dir) + local_dir = local_dir_for_package(descriptor) + @operating_system.delete_and_recreate_directory(local_dir) + fig_file = File.join(temp_dir, PACKAGE_FILE_IN_REPO) + content = publish_package_content_and_derive_dot_fig_contents( + package_statements, descriptor, local_dir, local_only + ) + @operating_system.write(fig_file, content) + + if not local_only @operating_system.upload( fig_file, remote_fig_file_for_package(descriptor), @remote_repository_user - ) unless local_only - @operating_system.copy( - fig_file, local_fig_file_for_package(descriptor) ) + end + @operating_system.copy( + fig_file, local_fig_file_for_package(descriptor) + ) - FileUtils.rm_rf(temp_dir) + FileUtils.rm_rf(temp_dir) + + return true + end + + def updating? + return @update || @update_if_missing + end + + private + + PACKAGE_FILE_IN_REPO = '.fig' + + def initialize_local_repository() + if not File.exist?(@local_repository_directory) + Dir.mkdir(@local_repository_directory) end - def updating? - return @update || @update_if_missing + version_file = local_version_file() + if not File.exist?(version_file) + File.open(version_file, 'w') { |handle| handle.write(VERSION_SUPPORTED) } end - private + return + end - def should_update?(descriptor) - return true if @update + def check_local_repository_format() + check_repository_format('Local', local_repository_version()) - return @update_if_missing && package_missing?(descriptor) - end + return + end - def read_local_package(descriptor) - directory = local_dir_for_package(descriptor) - return read_package_from_directory(directory, descriptor) + def check_remote_repository_format() + check_repository_format('Remote', remote_repository_version()) + + return + end + + def check_repository_format(name, version) + if version != VERSION_SUPPORTED + Fig::Logging.fatal \ + "#{name} repository is in version #{version} format. This version of fig can only deal with repositories in version #{VERSION_SUPPORTED} format." + raise Fig::RepositoryError.new end - def bundle_resources(package_statements) - resources = [] - new_package_statements = package_statements.reject do |statement| - if ( - statement.is_a?(Statement::Resource) && - ! Repository.is_url?(statement.url) - ) - resources << statement.url - true - else - false - end - end + return + end - if resources.size > 0 - resources = expand_globs_from(resources) - file = 'resources.tar.gz' - @operating_system.create_archive(file, resources) - new_package_statements.unshift(Statement::Archive.new(nil, file)) - at_exit { File.delete(file) } - end + def local_repository_version() + if @local_repository_version.nil? + version_file = local_version_file() - return new_package_statements + @local_repository_version = + parse_repository_version(version_file, version_file) end - def install_package(descriptor) - temp_dir = nil + return @local_repository_version + end + def local_version_file() + return File.join(@local_repository_directory, VERSION_FILE_NAME) + end + + def local_package_directory() + return File.expand_path(File.join(@local_repository_directory, 'repos')) + end + + def remote_repository_version() + if @remote_repository_version.nil? + temp_dir = base_temp_dir() + @operating_system.delete_and_recreate_directory(temp_dir) + remote_version_file = "#{remote_repository_url()}/#{VERSION_FILE_NAME}" + local_version_file = File.join(temp_dir, "remote-#{VERSION_FILE_NAME}") begin - package = read_local_package(descriptor) - temp_dir = temp_dir_for_package(descriptor) - @operating_system.clear_directory(temp_dir) - package.archive_urls.each do |archive_url| - if not Repository.is_url?(archive_url) - archive_url = remote_dir_for_package(descriptor) + '/' + archive_url - end - @operating_system.download_archive(archive_url, File.join(temp_dir)) + @operating_system.download(remote_version_file, local_version_file) + rescue Fig::NotFoundError + # The download may create an empty file, so get rid of it. + if File.exist?(local_version_file) + File.unlink(local_version_file) end - package.resource_urls.each do |resource_url| - if not Repository.is_url?(resource_url) - resource_url = - remote_dir_for_package(descriptor) + '/' + resource_url - end - @operating_system.download_resource(resource_url, File.join(temp_dir)) - end - local_dir = local_dir_for_package(descriptor) - @operating_system.clear_directory(local_dir) - # some packages contain no files, only a fig file. - if not (package.archive_urls.empty? && package.resource_urls.empty?) - FileUtils.mv(Dir.glob(File.join(temp_dir, '*')), local_dir) - end - write_local_package(descriptor, package) - rescue - Logging.fatal 'Install failed, cleaning up.' - delete_local_package(descriptor) - raise RepositoryError.new - ensure - if temp_dir - FileUtils.rm_rf(temp_dir) - end end + + @remote_repository_version = + parse_repository_version(local_version_file, remote_version_file) end - def update_package(descriptor) - remote_fig_file = remote_fig_file_for_package(descriptor) - local_fig_file = local_fig_file_for_package(descriptor) - begin - if @operating_system.download(remote_fig_file, local_fig_file) - install_package(descriptor) - end - rescue NotFoundError - Logging.fatal "Package not found in remote repository: #{descriptor.to_string()}" - delete_local_package(descriptor) - raise RepositoryError.new - end + return @remote_repository_version + end + + def parse_repository_version(version_file, description) + if not File.exist?(version_file) + return 1 # Since there was no version file early in Fig development. end - # 'resources' is an Array of fileglob patterns: ['tmp/foo/file1', - # 'tmp/foo/*.jar'] - def expand_globs_from(resources) - expanded_files = [] - resources.each {|f| expanded_files.concat(Dir.glob(f))} - expanded_files + version_string = IO.read(version_file) + version_string.strip!() + if version_string !~ / \A \d+ \z /x + Fig::Logging.fatal \ + %Q<Could not parse the contents of "#{description}" ("#{version_string}") as a version.> + raise Fig::RepositoryError.new end - def read_package_from_directory(dir, descriptor) - file = nil - dot_fig_file = File.join(dir, '.fig') - if File.exist?(dot_fig_file) - file = dot_fig_file - else - package_dot_fig_file = File.join(dir, 'package.fig') - if not File.exist?(package_dot_fig_file) - Logging.fatal %Q<Fig file not found for package "#{descriptor.name || '<unnamed>'}". Looked for "#{dot_fig_file}" and "#{package_dot_fig_file}" and found neither.> - raise RepositoryError.new + return version_string.to_i() + end + + def validate_asset_names(package_statements) + asset_statements = package_statements.select { |s| s.is_asset? } + + asset_names = Set.new() + asset_statements.each do + |statement| + + asset_name = statement.asset_name() + if not asset_name.nil? + if asset_name == RESOURCES_FILE + Fig::Logging.fatal \ + %Q<You cannot have an asset with the name "#{RESOURCES_FILE}"#{statement.position_string()} due to Fig implementation details.> end - file = package_dot_fig_file + if asset_names.include?(asset_name) + Fig::Logging.fatal \ + %Q<Found multiple archives with the name "#{asset_name}"#{statement.position_string()}. If these were allowed, archives would overwrite each other.> + raise Fig::RepositoryError.new + else + asset_names.add(asset_name) + end end - - return read_package_from_file(file, descriptor) end + end - def read_package_from_file(file_name, descriptor) - if not File.exist?(file_name) - Logging.fatal "Package not found: #{descriptor.to_string()}" - raise RepositoryError.new - end - content = File.read(file_name) + def remote_repository_url() + return @application_config.remote_repository_url() + end - package = @parser.parse_package( - descriptor, File.dirname(file_name), content - ) + def should_update?(descriptor) + return true if @update - @packages.add_package(package) + return @update_if_missing && package_missing?(descriptor) + end - return package - end + def read_local_package(descriptor) + directory = local_dir_for_package(descriptor) + return read_package_from_directory(directory, descriptor) + end - def delete_local_package(descriptor) - FileUtils.rm_rf(local_dir_for_package(descriptor)) - end + def update_package(descriptor) + temp_dir = package_download_temp_dir(descriptor) + begin + install_package(descriptor, temp_dir) + rescue Fig::NotFoundError + Fig::Logging.fatal \ + "Package not found in remote repository: #{descriptor.to_string()}" - def write_local_package(descriptor, package) - file = local_fig_file_for_package(descriptor) - @operating_system.write(file, package.unparse) - end + delete_local_package(descriptor) - def remote_fig_file_for_package(descriptor) - "#{@remote_repository_url}/#{descriptor.name}/#{descriptor.version}/.fig" - end + raise Fig::RepositoryError.new + rescue StandardError => exception + Fig::Logging.debug exception + Fig::Logging.fatal 'Install failed, cleaning up.' - def local_fig_file_for_package(descriptor) - File.join(local_dir_for_package(descriptor), '.fig') + delete_local_package(descriptor) + + raise Fig::RepositoryError.new + ensure + FileUtils.rm_rf(temp_dir) end - def remote_dir_for_package(descriptor) - "#{@remote_repository_url}/#{descriptor.name}/#{descriptor.version}" + return + end + + def install_package(descriptor, temp_dir) + @operating_system.delete_and_recreate_directory(temp_dir) + + remote_fig_file = remote_fig_file_for_package(descriptor) + local_fig_file = fig_file_for_package_download(temp_dir) + + return if not @operating_system.download(remote_fig_file, local_fig_file) + + package = read_package_from_directory(temp_dir, descriptor) + + package.archive_urls.each do |archive_url| + if not Fig::Repository.is_url?(archive_url) + archive_url = remote_dir_for_package(descriptor) + '/' + archive_url + end + @operating_system.download_and_unpack_archive(archive_url, temp_dir) end + package.resource_urls.each do |resource_url| + if not Fig::Repository.is_url?(resource_url) + resource_url = + remote_dir_for_package(descriptor) + '/' + resource_url + end + @operating_system.download_resource(resource_url, temp_dir) + end - def local_dir_for_package(descriptor) - return File.join( - @local_repository_dir, descriptor.name, descriptor.version - ) + local_dir = local_dir_for_package(descriptor) + FileUtils.rm_rf(local_dir) + FileUtils.mkdir_p( File.dirname(local_dir) ) + FileUtils.mv(temp_dir, local_dir) + + return + end + + # 'resources' is an Array of fileglob patterns: ['tmp/foo/file1', + # 'tmp/foo/*.jar'] + def expand_globs_from(resources) + expanded_files = [] + + resources.each do + |path| + + globbed_files = Dir.glob(path) + if globbed_files.empty? + expanded_files << path + else + expanded_files.concat(globbed_files) + end end - def temp_dir_for_package(descriptor) - File.join(@local_repository_dir, 'tmp') + return expanded_files + end + + def read_package_from_directory(directory, descriptor) + dot_fig_file = File.join(directory, PACKAGE_FILE_IN_REPO) + if not File.exist?(dot_fig_file) + Fig::Logging.fatal %Q<Fig file not found for package "#{descriptor.name || '<unnamed>'}". There is nothing in "#{dot_fig_file}".> + raise Fig::RepositoryError.new end - def package_missing?(descriptor) - not File.exist?(local_fig_file_for_package(descriptor)) + return read_package_from_file(dot_fig_file, descriptor) + end + + def read_package_from_file(file_name, descriptor) + if not File.exist?(file_name) + Fig::Logging.fatal "Package not found: #{descriptor.to_string()}" + raise Fig::RepositoryError.new end + content = File.read(file_name) - def derive_package_content( + package = @parser.parse_package( + descriptor, File.dirname(file_name), descriptor.to_string(), content + ) + + @package_cache.add_package(package) + + return package + end + + def delete_local_package(descriptor) + FileUtils.rm_rf(local_dir_for_package(descriptor)) + end + + def remote_fig_file_for_package(descriptor) + "#{remote_dir_for_package(descriptor)}/#{PACKAGE_FILE_IN_REPO}" + end + + def local_fig_file_for_package(descriptor) + File.join(local_dir_for_package(descriptor), PACKAGE_FILE_IN_REPO) + end + + def fig_file_for_package_download(package_download_dir) + File.join(package_download_dir, PACKAGE_FILE_IN_REPO) + end + + def remote_dir_for_package(descriptor) + "#{remote_repository_url()}/#{descriptor.name}/#{descriptor.version}" + end + + def local_dir_for_package(descriptor) + return File.join( + local_package_directory(), descriptor.name, descriptor.version + ) + end + + def base_temp_dir() + File.join(@local_repository_directory, 'tmp') + end + + def publish_temp_dir() + File.join(base_temp_dir(), 'publish') + end + + def package_download_temp_dir(descriptor) + base_directory = File.join(base_temp_dir(), 'package-download') + FileUtils.mkdir_p(base_directory) + + return Dir.mktmpdir( + "#{descriptor.name}.version.#{descriptor.version}+", base_directory + ) + end + + def package_missing?(descriptor) + not File.exist?(local_fig_file_for_package(descriptor)) + end + + def publish_package_content_and_derive_dot_fig_contents( + package_statements, descriptor, local_dir, local_only + ) + header_strings = derive_package_metadata_comments( + package_statements, descriptor + ) + deparsed_statement_strings = publish_package_content( package_statements, descriptor, local_dir, local_only ) - header_strings = derive_package_metadata_comments(descriptor) - resource_statement_strings = derive_package_resources( - package_statements, descriptor, local_dir, local_only - ) - return [header_strings, resource_statement_strings].flatten() - end + statement_strings = [header_strings, deparsed_statement_strings].flatten() + return statement_strings.join("\n").gsub(/\n{3,}/, "\n\n").strip() + "\n" + end - def derive_package_metadata_comments(descriptor) - now = Time.now() + def derive_package_metadata_comments(package_statements, descriptor) + now = Time.now() - return [ - %Q<# Publishing information for #{descriptor.to_string()}:>, + asset_statements = + package_statements.select { |statement| statement.is_asset? } + asset_strings = + asset_statements.collect { |statement| statement.unparse('# ') } + asset_summary = nil + + if asset_strings.empty? + asset_summary = [ %q<#>, - %Q<# Time: #{now} (epoch: #{now.to_i()})>, - %Q<# User: #{Sys::Admin.get_login()}>, - %Q<# Host: #{Socket.gethostname()}>, - %Q<# Args: "#{ARGV.join %q[", "]}">, - %q<#> + %q<# There were no asset statements in the unpublished package definition.> ] + else + asset_summary = [ + %q<#>, + %q<# Original asset statements: >, + %q<#>, + asset_strings + ] end - def derive_package_resources( - package_statements, descriptor, local_dir, local_only - ) - return bundle_resources(package_statements).map do |statement| - if statement.is_a?(Statement::Publish) - nil - elsif statement.is_a?(Statement::Archive) || statement.is_a?(Statement::Resource) - if statement.is_a?(Statement::Resource) && ! Repository.is_url?(statement.url) - archive_name = statement.url - archive_remote = "#{remote_dir_for_package(descriptor)}/#{statement.url}" - else - archive_name = statement.url.split('/').last - archive_remote = "#{remote_dir_for_package(descriptor)}/#{archive_name}" + return [ + %Q<# Publishing information for #{descriptor.to_string()}:>, + %q<#>, + %Q<# Time: #{now} (epoch: #{now.to_i()})>, + %Q<# User: #{Sys::Admin.get_login()}>, + %Q<# Host: #{Socket.gethostname()}>, + %Q<# Args: "#{ARGV.join %q[", "]}">, + %Q<# Fig: v#{Fig::Command.get_version()}>, + asset_summary, + %Q<\n>, + ].flatten() + end + + # Deals with Archive and Resource statements. It downloads any remote + # files (those where the statement references a URL as opposed to a local + # file) and then copies all files into the local repository and the remote + # repository (if not a local-only publish). + # + # Returns the deparsed strings for the resource statements with URLs + # replaced with in-package paths. + def publish_package_content( + package_statements, descriptor, local_dir, local_only + ) + return create_resource_archive(package_statements).map do |statement| + if statement.is_asset? + asset_name = statement.asset_name() + asset_remote = "#{remote_dir_for_package(descriptor)}/#{asset_name}" + + if Fig::Repository.is_url?(statement.url) + asset_local = File.join(publish_temp_dir(), asset_name) + + begin + @operating_system.download(statement.url, asset_local) + rescue Fig::NotFoundError + Fig::Logging.fatal "Could not download #{statement.url}." + raise Fig::RepositoryError.new end - if Repository.is_url?(statement.url) - archive_local = File.join(temp_dir, archive_name) - @operating_system.download(statement.url, archive_local) - else - archive_local = statement.url - end - @operating_system.upload(archive_local, archive_remote, @remote_repository_user) unless local_only - @operating_system.copy(archive_local, local_dir + '/' + archive_name) - if statement.is_a?(Statement::Archive) - @operating_system.unpack_archive(local_dir, archive_name) - end - statement.class.new(nil, archive_name).unparse('') else - statement.unparse('') + asset_local = statement.url + check_asset_path(asset_local) end - end.select { |s| not s.nil? } + + if not local_only + @operating_system.upload( + asset_local, asset_remote, @remote_repository_user + ) + end + + @operating_system.copy(asset_local, local_dir + '/' + asset_name) + if statement.is_a?(Fig::Statement::Archive) + @operating_system.unpack_archive(local_dir, asset_name) + end + + statement.class.new(nil, nil, asset_name).unparse('') + else + statement.unparse('') + end end + end + + # Grabs all of the Resource statements that don't reference URLs, creates a + # "resources.tar.gz" file containing all the referenced files, strips the + # Resource statements out of the statements, replacing them with a single + # Archive statement. Thus the caller should substitute its set of + # statements with the return value. + def create_resource_archive(package_statements) + asset_paths = [] + new_package_statements = package_statements.reject do |statement| + if ( + statement.is_a?(Fig::Statement::Resource) && + ! Fig::Repository.is_url?(statement.url) + ) + asset_paths << statement.url + true + else + false + end + end + + if asset_paths.size > 0 + asset_paths = expand_globs_from(asset_paths) + check_asset_paths(asset_paths) + + file = RESOURCES_FILE + @operating_system.create_archive(file, asset_paths) + new_package_statements.unshift( + Fig::Statement::Archive.new(nil, nil, file) + ) + Fig::AtExit.add { File.delete(file) } + end + + return new_package_statements + end + + def check_asset_path(asset_path) + if not File.exist?(asset_path) + Fig::Logging.fatal "Could not find file #{asset_path}." + raise Fig::RepositoryError.new + end + + return + end + + def check_asset_paths(asset_paths) + non_existing_paths = + asset_paths.select {|path| ! File.exist?(path) && ! File.symlink?(path) } + + if not non_existing_paths.empty? + if non_existing_paths.size > 1 + Fig::Logging.fatal "Could not find files: #{ non_existing_paths.join(', ') }" + else + Fig::Logging.fatal "Could not find file #{non_existing_paths[0]}." + end + + raise Fig::RepositoryError.new + end + + return end end