bin/autoproj_bootstrap in autoproj-2.0.0.rc3 vs bin/autoproj_bootstrap in autoproj-2.0.0.rc4

- old
+ new

@@ -3,3212 +3,221 @@ if RUBY_VERSION < "1.9.2" STDERR.puts "autoproj requires Ruby >= 1.9.2" exit 1 end -require 'rbconfig' -module Autobuild - @windows = RbConfig::CONFIG["host_os"] =~ %r!(msdos|mswin|djgpp|mingw|[Ww]indows)! - def self.windows? - @windows - end - - @macos = RbConfig::CONFIG["host_os"] =~ %r!([Dd]arwin)! - def self.macos? - @macos - end -end - +require 'optparse' +require 'fileutils' require 'yaml' -require 'set' module Autoproj - class ConfigError < RuntimeError; end - class << self - attr_reader :verbose - end - - def self.color(string, *args) - string - end - - def self.warn(str, *args) - STDERR.puts "WARN #{str}" - end - def self.message(str) - STDERR.puts " #{str}" - end -end - -module Autobuild - class Exception < RuntimeError; end - - def self.do_update - true - end - def self.message(str) - STDERR.puts " #{str}" - end - def self.progress(key, str) - STDERR.puts " #{str}" - end - def self.progress_done(key) - end - def self.message(str) - STDERR.puts " #{str}" - end - - class << self - attr_reader :programs - end - @programs = Hash.new - def self.tool(name) - # Let the ability to set programs[name] to nil to make sure we don't use - # that program. This is used later on in this file to make sure we - # aren't using the wrong rubygems binary - if programs.has_key?(name) - programs[name] - else - name - end - end - - module Subprocess - def self.run(name, phase, *cmd) - if cmd.last.kind_of?(Hash) - options = cmd.pop - (options[:env] || Hash.new).each do |k, v| - ENV[k] = v - end - end - - output = `#{cmd.join(" ")}` - if $?.exitstatus != 0 - STDERR.puts "ERROR: failed to run #{cmd.join(" ")}" - STDERR.puts "ERROR: command output is: #{output}" - exit 1 - end - end - end -end - -module Autoproj - # Definition of an autoproj option as defined by - # {Configuration#declare} - class BuildOption - attr_reader :name - attr_reader :type - attr_reader :options - - attr_reader :validator - - TRUE_STRINGS = %w{on yes y true} - FALSE_STRINGS = %w{off no n false} - def initialize(name, type, options, validator) - @name, @type, @options = name.to_str, type.to_str, options.to_hash - @validator = validator.to_proc if validator - if !BuildOption.respond_to?("validate_#{type}") - raise ConfigError.new, "invalid option type #{type}" - end - end - - def short_doc - if short_doc = options[:short_doc] - short_doc - elsif doc = options[:doc] - if doc.respond_to?(:to_ary) then doc.first - else doc - end - else "#{name} (no documentation for this option)" - end - end - - def doc - doc = (options[:doc] || "#{name} (no documentation for this option)") - if doc.respond_to?(:to_ary) # multi-line - first_line = doc[0] - remaining = doc[1..-1] - if remaining.empty? - first_line - else - remaining = remaining.join("\n").split("\n").join("\n ") - Autoproj.color(first_line, :bold) + "\n " + remaining - end - else - doc - end - end - - def ask(current_value, doc = nil) - default_value = - if !current_value.nil? then current_value.to_s - elsif options[:default] then options[:default].to_str - else '' - end - - STDOUT.print " #{doc || self.doc} [#{default_value}] " - STDOUT.flush - answer = STDIN.readline.chomp - if answer == '' - answer = default_value - end - validate(answer) - - rescue InputError => e - Autoproj.message("invalid value: #{e.message}", :red) - retry - end - - def validate(value) - value = BuildOption.send("validate_#{type}", value, options) - if validator - value = validator[value] - end - value - end - - def self.validate_boolean(value, options) - if TRUE_STRINGS.include?(value.downcase) - true - elsif FALSE_STRINGS.include?(value.downcase) - false - else - raise InputError, "invalid boolean value '#{value}', accepted values are '#{TRUE_STRINGS.join(", ")}' for true, and '#{FALSE_STRINGS.join(", ")} for false" - end - end - - def self.validate_string(value, options) - if possible_values = options[:possible_values] - if options[:lowercase] - value = value.downcase - elsif options[:uppercase] - value = value.upcase - end - - if !possible_values.include?(value) - raise InputError, "invalid value '#{value}', accepted values are '#{possible_values.join("', '")}' (without the quotes)" - end - end - value - end - end -end - - -module Autoproj - # Class that does the handling of configuration options as well as - # loading/saving on disk - class Configuration - # Set of currently known options + module Ops + # This class contains the functionality necessary to install autoproj in a + # clean root # - # These are the values that are going to be saved on disk. Use - # {override} to change a value without changing the saved configuration - # file. - attr_reader :config - # Set of overriden option values that won't get written to file - attr_reader :overrides - # Set of options that have been declared with {declare} - attr_reader :declared_options - # The options that have already been shown to the user - attr_reader :displayed_options - # The path to the underlying configuration file - attr_reader :path + # It can be required standalone (i.e. does not depend on anything else than + # ruby and the ruby standard library) + class Install + # The directory in which to install autoproj + attr_reader :root_dir + # Content of the Gemfile generated to install autoproj itself + attr_accessor :gemfile - def initialize(path = nil) - @config = Hash.new - @overrides = Hash.new - @declared_options = Hash.new - @displayed_options = Hash.new - @path = path - end - - # Deletes the current value for an option - # - # The user will be asked for a new value next time the option is needed - # - # @param [String] the option name - # @return the deleted value - def reset(name) - config.delete(name) - end - - # Sets a configuration option - # - # @param [String] key the option name - # @param [Object] value the option value - # @param [Boolean] user_validated if true, autoproj will not ask the - # user about this value next time it is needed. Otherwise, it will be - # asked about it, the new value being used as default - def set(key, value, user_validated = false) - config[key] = [value, user_validated] - end - - # Override a known option value - # - # The new value will not be saved to disk, unlike with {set} - def override(option_name, value) - overrides[option_name] = value - end - - # Tests whether a value is set for the given option name - # - # @return [Boolean] - def has_value_for?(name) - config.has_key?(name) || overrides.has_key?(name) - end - - # Get the value for a given option - def get(key, *default_value) - if overrides.has_key?(key) - return overrides[key] + def initialize(root_dir) + @root_dir = root_dir + @gemfile = default_gemfile_contents + @private_bundler = false + @private_autoproj = false + @private_gems = false end - value, validated = config[key] - if value.nil? && !declared?(key) && !default_value.empty? - default_value.first - elsif value.nil? || (declared?(key) && !validated) - value = configure(key) - else - if declared?(key) && (displayed_options[key] != value) - doc = declared_options[key].short_doc - if doc[-1, 1] != "?" - doc = "#{doc}:" - end - displayed_options[key] = value - end - value - end - end + def dot_autoproj; File.join(root_dir, '.autoproj') end + def bin_dir; File.join(dot_autoproj, 'bin') end + def bundler_install_dir; File.join(dot_autoproj, 'bundler') end + def autoproj_install_dir; File.join(dot_autoproj, 'autoproj') end + # The path to the gemfile used to install autoproj + def autoproj_gemfile_path; File.join(autoproj_install_dir, 'Gemfile') end + def autoproj_config_path; File.join(dot_autoproj, 'config.yml') end - # Returns the option's name-value pairs for the options that do not - # require user input - def validated_values - config.inject(Hash.new) do |h, (k, v)| - h[k] = - if overrides.has_key?(k) then overrides[k] - elsif v.last || !declared?(k) then v.first - end - h - end - end + # Whether bundler should be installed locally in {#dot_autoproj} + def private_bundler?; @private_bundler end + # Whether autoproj should be installed locally in {#dot_autoproj} + def private_autoproj?; @private_autoproj end + # Whether bundler should be installed locally in the workspace + # prefix directory + def private_gems?; @private_gems end - # Declare an option - # - # This declares a given option, thus allowing to ask the user about it - # - # @param [String] name the option name - # @param [String] type the option type (can be 'boolean' or 'string') - # @option options [String] :short_doc the one-line documentation string - # that is displayed when the user does not have to be queried. It - # defaults to the first line of :doc if not given - # @option options [String] :doc the full option documentation. It is - # displayed to the user when he is explicitly asked about the option's - # value - # @option options [Object] :default the default value this option should - # take - # @option options [Array] :possible_values list of possible values (only - # if the option type is 'string') - # @option options [Boolean] :lowercase (false) whether the user's input - # should be converted to lowercase before it gets validated / saved. - # @option options [Boolean] :uppercase (false) whether the user's input - # should be converted to uppercase before it gets validated / saved. - def declare(name, type, options, &validator) - declared_options[name] = BuildOption.new(name, type, options, validator) - end - - # Checks if an option exists - # @return [Boolean] - def declared?(name) - declared_options.has_key?(name) - end - - # Configures a given option by asking the user about its desired value - # - # @return [Object] the new option value - # @raise ConfigError if the option is not declared - def configure(option_name) - if opt = declared_options[option_name] - if current_value = config[option_name] - current_value = current_value.first - end - value = opt.ask(current_value) - config[option_name] = [value, true] - displayed_options[option_name] = value - value - else - raise ConfigError.new, "undeclared option '#{option_name}'" - end - end - - def load(options = Hash.new) - options = validate_options options, - path: self.path, - reconfigure: false - - if h = YAML.load(File.read(options[:path])) - h.each do |key, value| - set(key, value, !options[:reconfigure]) - end - end - end - - def reconfigure! - new_config = Hash.new - config.each do |key, (value, user_validated)| - new_config[key] = [value, false] - end - @config = new_config - end - - def save(path = self.path) - File.open(path, "w") do |io| - h = Hash.new - config.each do |key, value| - h[key] = value.first - end - - io.write YAML.dump(h) - end - end - - def each_reused_autoproj_installation - if has_value_for?('reused_autoproj_installations') - get('reused_autoproj_installations').each(&proc) - else [].each(&proc) - end - end - - def import_log_enabled? - get('import_log_enabled', true) - end - - def import_log_enabled=(value) - set('import_log_enabled', !!value) - end - - def parallel_build_level - get('parallel_build_level', nil) || Autobuild.parallel_build_level - end - - def parallel_build_level=(level) - set('parallel_build_level', level) - Autobuild.parallel_build_level = level - end - - def parallel_import_level - get('parallel_import_level', 10) - end - - def parallel_import_level=(level) - set('parallel_import_level', level) - end - - def ruby_executable - if path = get('ruby_executable', nil) - path - else - path = OSDependencies.autodetect_ruby_program - set('ruby_executable', path, true) - path - end - end - - def validate_ruby_executable - if has_value_for?('ruby_executable') - expected = get('ruby_executable') - if expected != ruby_executable - raise ConfigError.new, "this autoproj installation was bootstrapped using #{expected}, but you are currently running under #{ruby_executable}. This is usually caused by calling a wrong gem program (for instance, gem1.8 instead of gem1.9.1)" - end - end - set('ruby_executable', ruby_executable, true) - end - - def use_prerelease? - use_prerelease = - if env_flag = ENV['AUTOPROJ_USE_PRERELEASE'] - env_flag == '1' - elsif has_value_for?('autoproj_use_prerelease') - get('autoproj_use_prerelease') - end - set "autoproj_use_prerelease", (use_prerelease ? true : false), true - use_prerelease - end - - def shell_helpers? - get 'shell_helpers', true - end - - def shell_helpers=(flag) - set 'shell_helpers', flag, true - end - - def apply_autobuild_configuration - if has_value_for?('autobuild') - params = get('autobuild') - if params.kind_of?(Hash) - params.each do |k, v| - Autobuild.send("#{k}=", v) - end - end - end - end - - # The directory in which packages will be installed. - # - # If it is a relative path, it is relative to the root dir of the - # installation. - # - # The default is "install" - # - # @return [String] - def prefix_dir - get('prefix', 'install') - end - - # Defines the temporary area in which packages should put their build - # files - # - # If absolute, it is handled as {#prefix_dir}: the package name will be - # appended to it. If relative, it is relative to the package's source - # directory - # - # The default is "build" - # - # @return [String] - def build_dir - get('build', 'build') - end - - # Returns true if there should be one prefix per package - # - # The default is false (disabled) - # - # @return [Boolean] - def separate_prefixes? - get('separate_prefixes', false) - end - - # Controls whether there should be one prefix per package - # - # @see separate_prefixes? - def separate_prefixes=(flag) - set('separate_prefixes', flag, true) - end - - # Returns true if packages and prefixes should be auto-generated, based - # on the SHA of the package names. This is meant to be used for build - # services that want to check that dependencies are properly set - # - # The default is false (disabled) - # - # @return [Boolean] - def randomize_layout? - get('randomize_layout', false) - end - - # Sets whether the layout should be randomized - # - # @return [Boolean] - # @see randomize_layout? - def randomize_layout=(value) - set('randomize_layout', value, true) - end - - DEFAULT_UTILITY_SETUP = Hash[ - 'doc' => true, - 'test' => false] - - # The configuration key that should be used to store the utility - # enable/disable information - # - # @param [String] the utility name - # @return [String] the config key - def utility_key(utility) - "autoproj_#{utility}_utility" - end - - # Returns whether a given utility is enabled for the package - # - # If there is no specific configuration for the package, uses the global - # default set with utility_enable_all or utility_disable_all. If none of - # these methods has been called, uses the default in - # {DEFAULT_UTILITY_SETUP} - # - # @param [String] utility the utility name (e.g. 'doc' or 'test') - # @param [String] package the package name - # @return [Boolean] true if the utility should be enabled for the - # requested package and false otherwise - def utility_enabled_for?(utility, package) - utility_config = get(utility_key(utility), Hash.new) - if utility_config.has_key?(package) - utility_config[package] - else get("#{utility_key(utility)}_default", DEFAULT_UTILITY_SETUP[utility]) - end - end - - # Enables a utility for all packages - # - # This both sets the default value for all packages and resets all - # package-specific values set with {utility_enable_for} and - # {utility_disable_for} - # - # @param [String] utility the utility name (e.g. 'doc' or 'test') - # @return [void] - def utility_enable_all(utility) - reset(utility_key(utility)) - set("#{utility_key(utility)}_default", true) - end - - # Enables a utility for a set of packages - # - # @param [String] utility the utility name (e.g. 'doc' or 'test') - # @param [String] packages the package names - # @return [void] - def utility_enable(utility, *packages) - utility_config = get(utility_key(utility), Hash.new) - packages.each do |pkg_name| - utility_config[pkg_name] = true - end - set(utility_key(utility), utility_config) - end - - # Disables a utility for all packages - # - # This both sets the default value for all packages and resets all - # package-specific values set with {utility_enable_for} and - # {utility_disable_for} - # - # @param [String] utility the utility name (e.g. 'doc' or 'test') - # @return [void] - def utility_disable_all(utility) - reset(utility_key(utility)) - set("#{utility_key(utility)}_default", false) - end - - # Disables a utility for a specific package - # - # Note that if the default for this utility is to be disabled, this is - # essentially a no-op. - # - # @param [String] utility the utility name (e.g. 'doc' or 'test') - # @param [String] packages the package names - # @return [void] - def utility_disable(utility, *packages) - utility_config = get(utility_key(utility), Hash.new) - packages.each do |pkg_name| - utility_config[pkg_name] = false - end - set(utility_key(utility), utility_config) - end - - def merge(conf) - config.merge!(conf.config) - end - end -end - - -module Autoproj - def self.config - @config ||= Configuration.new - end -end - -require 'tempfile' -require 'json' -module Autoproj - # Module that contains the package manager implementations for the - # OSDependencies class - module PackageManagers - # Base class for all package managers. Subclasses must add the - # #install(packages) method and may add the - # #filter_uptodate_packages(packages) method - # - # Package managers must be registered in PACKAGE_HANDLERS and - # (if applicable) OS_PACKAGE_HANDLERS. - class Manager - # @return [Array<String>] the various names this package manager is - # known about - attr_reader :names - - attr_writer :enabled - def enabled?; !!@enabled end - - attr_writer :silent - def silent?; !!@silent end - - # Create a package manager registered with various names - # - # @param [Array<String>] names the package manager names. It MUST be - # different from the OS names that autoproj uses. See the comment - # for OS_PACKAGE_HANDLERS for an explanation - def initialize(names = []) - @names = names.dup - @enabled = true - @silent = true - end - - # The primary name for this package manager - def name - names.first - end - - # Overload to perform initialization of environment variables in - # order to have a properly functioning package manager - # - # This is e.g. needed for python pip or rubygems - def self.initialize_environment(_env = nil, _manifest = nil, _root_dir = Autoproj.root_dir) - end - end - - # Dummy package manager used for unknown OSes. It simply displays a - # message to the user when packages are needed - class UnknownOSManager < Manager - def initialize - super(['unknown']) - @installed_osdeps = Set.new - end - - def osdeps_interaction_unknown_os(osdeps) - puts <<-EOMSG - #{Autoproj.color("The build process requires some other software packages to be installed on our operating system", :bold)} - #{Autoproj.color("If they are already installed, simply ignore this message", :red)} - - #{osdeps.to_a.sort.join("\n ")} - - EOMSG - print Autoproj.color("Press ENTER to continue", :bold) - STDOUT.flush - STDIN.readline - puts - nil - end - - def install(osdeps) - if silent? - return false - else - osdeps = osdeps.to_set - osdeps -= @installed_osdeps - if !osdeps.empty? - result = osdeps_interaction_unknown_os(osdeps) - end - @installed_osdeps |= osdeps - return result - end - end - end - - # Base class for all package managers that simply require the call of a - # shell script to install packages (e.g. yum, apt, ...) - class ShellScriptManager < Manager - def self.execute(script, with_locking, with_root) - if with_locking - File.open('/tmp/autoproj_osdeps_lock', 'w') do |lock_io| - begin - while !lock_io.flock(File::LOCK_EX | File::LOCK_NB) - Autoproj.message " waiting for other autoproj instances to finish their osdeps installation" - sleep 5 - end - return execute(script, false,with_root) - ensure - lock_io.flock(File::LOCK_UN) - end - end - end - - sudo = Autobuild.tool_in_path('sudo') - Tempfile.open('osdeps_sh') do |io| - io.puts "#! /bin/bash" - io.puts GAIN_ROOT_ACCESS % [sudo] if with_root - io.write script - io.flush - Autobuild::Subprocess.run 'autoproj', 'osdeps', '/bin/bash', io.path - end - end - - GAIN_ROOT_ACCESS = <<-EOSCRIPT -# Gain root access using sudo -if test `id -u` != "0"; then - exec %s /bin/bash $0 "$@" - -fi - EOSCRIPT - - # Overrides the {#needs_locking?} flag - attr_writer :needs_locking - # Whether two autoproj instances can run this package manager at the - # same time - # - # This declares if this package manager cannot be used concurrently. - # If it is the case, autoproj will ensure that there is no two - # autoproj instances running this package manager at the same time - # - # @return [Boolean] - # @see needs_locking= - def needs_locking?; !!@needs_locking end - - # Overrides the {#needs_root?} flag - attr_writer :needs_root - # Whether this package manager needs root access. - # - # This declares if the command line(s) for this package manager - # should be started as root. Root access is provided using sudo - # - # @return [Boolean] - # @see needs_root= - def needs_root?; !!@needs_root end - - # Command line used by autoproj to install packages - # - # Since it is to be used for automated install by autoproj, it - # should not require any interaction with the user. When generating - # the command line, the %s slot is replaced by the quoted package - # name(s). - # - # @return [String] a command line pattern that allows to install - # packages without user interaction. It is used when a package - # should be installed by autoproj automatically - attr_reader :auto_install_cmd - # Command line displayed to the user to install packages - # - # When generating the command line, the %s slot is replaced by the - # quoted package name(s). - # - # @return [String] a command line pattern that allows to install - # packages with user interaction. It is displayed to the - # user when it chose to not let autoproj install packages for this - # package manager automatically - attr_reader :user_install_cmd - - # @param [Array<String>] names the package managers names, see - # {#names} - # @param [Boolean] needs_locking whether this package manager can be - # started by two separate autoproj instances at the same time. See - # {#needs_locking?} - # @param [String] user_install_cmd the user-visible command line. See - # {#user_install_cmd} - # @param [String] auto_install_cmd the command line used by autoproj - # itself, see {#auto_install_cmd}. - # @param [Boolean] needs_root if the command lines should be started - # as root or not. See {#needs_root?} - def initialize(names, needs_locking, user_install_cmd, auto_install_cmd,needs_root=true) - super(names) - @needs_locking, @user_install_cmd, @auto_install_cmd,@needs_root = - needs_locking, user_install_cmd, auto_install_cmd, needs_root - end - - # Generate the shell script that would allow the user to install - # the given packages - # - # @param [Array<String>] os_packages the name of the packages to be - # installed - # @option options [String] :user_install_cmd (#user_install_cmd) the - # command-line pattern that should be used to generate the script. - # If given, it overrides the default value stored in - # {#user_install_cmd] - def generate_user_os_script(os_packages, options = Hash.new) - user_install_cmd = options[:user_install_cmd] || self.user_install_cmd - if user_install_cmd - (user_install_cmd % [os_packages.join("' '")]) - else generate_auto_os_script(os_packages) - end - end - - # Generate the shell script that should be executed by autoproj to - # install the given packages - # - # @param [Array<String>] os_packages the name of the packages to be - # installed - # @option options [String] :auto_install_cmd (#auto_install_cmd) the - # command-line pattern that should be used to generate the script. - # If given, it overrides the default value stored in - # {#auto_install_cmd] - def generate_auto_os_script(os_packages, options = Hash.new) - auto_install_cmd = options[:auto_install_cmd] || self.auto_install_cmd - (auto_install_cmd % [os_packages.join("' '")]) - end - - # Handles interaction with the user - # - # This method will verify whether the user required autoproj to - # install packages from this package manager automatically. It - # displays a relevant message if it is not the case. - # - # @return [Boolean] true if the packages should be installed - # automatically, false otherwise - def osdeps_interaction(os_packages, shell_script) - if OSDependencies.force_osdeps - return true - elsif enabled? - return true - elsif silent? - return false - end - - # We're asked to not install the OS packages but to display them - # anyway, do so now - puts <<-EOMSG - - #{Autoproj.color("The build process and/or the packages require some other software to be installed", :bold)} - #{Autoproj.color("and you required autoproj to not install them itself", :bold)} - #{Autoproj.color("\nIf these packages are already installed, simply ignore this message\n", :red) if !respond_to?(:filter_uptodate_packages)} - The following packages are available as OS dependencies, i.e. as prebuilt - packages provided by your distribution / operating system. You will have to - install them manually if they are not already installed - - #{os_packages.sort.join("\n ")} - - the following command line(s) can be run as root to install them: - - #{shell_script.split("\n").join("\n| ")} - - EOMSG - print " #{Autoproj.color("Press ENTER to continue ", :bold)}" - STDOUT.flush - STDIN.readline - puts - false - end - - # Install packages using this package manager - # - # @param [Array<String>] packages the name of the packages that - # should be installed - # @option options [String] :user_install_cmd (#user_install_cmd) the - # command line that should be displayed to the user to install said - # packages. See the option in {#generate_user_os_script} - # @option options [String] :auto_install_cmd (#auto_install_cmd) the - # command line that should be used by autoproj to install said - # packages. See the option in {#generate_auto_os_script} - # @return [Boolean] true if packages got installed, false otherwise - def install(packages, options = Hash.new) - handled_os = OSDependencies.supported_operating_system? - if handled_os - shell_script = generate_auto_os_script(packages, options) - user_shell_script = generate_user_os_script(packages, options) - end - if osdeps_interaction(packages, user_shell_script) - Autoproj.message " installing OS packages: #{packages.sort.join(", ")}" - - if Autoproj.verbose - Autoproj.message "Generating installation script for non-ruby OS dependencies" - Autoproj.message shell_script - end - ShellScriptManager.execute(shell_script, needs_locking?,needs_root?) - return true - end - false - end - end - - # Package manager interface for systems that use port (i.e. MacPorts/Darwin) as - # their package manager - class PortManager < ShellScriptManager - def initialize - super(['macports'], true, - "port install '%s'", - "port install '%s'") - end - end - - # Package manager interface for Mac OS using homebrew as - # its package manager - class HomebrewManager < ShellScriptManager - def initialize - super(['brew'], true, - "brew install '%s'", - "brew install '%s'", - false) - end - - def filter_uptodate_packages(packages, options = Hash.new) - # TODO there might be duplicates in packages which should be fixed - # somewhere else - packages = packages.uniq - result = `brew info --json=v1 '#{packages.join("' '")}'` - result = begin - JSON.parse(result) - rescue JSON::ParserError - if result && !result.empty? - Autoproj.warn "Error while parsing result of brew info --json=v1" - else - # one of the packages is unknown fallback to install all - # packaes which will complain about it - end - return packages - end - # fall back if something else went wrong - if packages.size != result.size - Autoproj.warn "brew info returns less or more packages when requested. Falling back to install all packages" - return packages - end - - new_packages = [] - result.each do |pkg| - new_packages << pkg["name"] if pkg["installed"].empty? - end - new_packages - end - end - - # Package manager interface for systems that use pacman (i.e. arch) as - # their package manager - class PacmanManager < ShellScriptManager - def initialize - super(['pacman'], true, - "pacman -Sy --needed '%s'", - "pacman -Sy --needed --noconfirm '%s'") - end - end - - # Package manager interface for systems that use emerge (i.e. gentoo) as - # their package manager - class EmergeManager < ShellScriptManager - def initialize - super(['emerge'], true, - "emerge '%s'", - "emerge --noreplace '%s'") - end - end - # Package manager interface for systems that use pkg (i.e. FreeBSD) as - # their package manager - class PkgManager < ShellScriptManager - def initialize - super(['pkg'], true, - "pkg install -y '%s'", - "pkg install -y '%s'") - end - end - - #Package manger for OpenSuse and Suse (untested) - class ZypperManager < ShellScriptManager - def initialize - super(['zypper'], true, - "zypper install '%s'", - "zypper -n install '%s'") - end - - def filter_uptodate_packages(packages, options = Hash.new) - result = `LANG=C rpm -q --whatprovides '#{packages.join("' '")}'` - has_all_pkgs = $?.success? - - if !has_all_pkgs - return packages # let zypper filter, we need root now anyways - else - return [] - end - end - - def install(packages) - patterns, packages = packages.partition { |pkg| pkg =~ /^@/ } - patterns = patterns.map { |str| str[1..-1] } - result = false - if !patterns.empty? - result |= super(patterns, - :auto_install_cmd => "zypper --non-interactive install --type pattern '%s'", - :user_install_cmd => "zypper install --type pattern '%s'") - end - if !packages.empty? - result |= super(packages) - end - if result - # Invalidate caching of installed packages, as we just - # installed new packages ! - @installed_packages = nil - end - end - end - - # Package manager interface for systems that use yum - class YumManager < ShellScriptManager - def initialize - super(['yum'], true, - "yum install '%s'", - "yum install -y '%s'") - end - - def filter_uptodate_packages(packages, options = Hash.new) - result = `LANG=C rpm -q --queryformat "%{NAME}\n" '#{packages.join("' '")}'` - - installed_packages = [] - new_packages = [] - result.split("\n").each_with_index do |line, index| - line = line.strip - if line =~ /package (.*) is not installed/ - package_name = $1 - if !packages.include?(package_name) # something is wrong, fallback to installing everything - return packages - end - new_packages << package_name - else - package_name = line.strip - if !packages.include?(package_name) # something is wrong, fallback to installing everything - return packages - end - installed_packages << package_name - end - end - new_packages - end - - def install(packages) - patterns, packages = packages.partition { |pkg| pkg =~ /^@/ } - patterns = patterns.map { |str| str[1..-1] } - result = false - if !patterns.empty? - result |= super(patterns, - :auto_install_cmd => "yum groupinstall -y '%s'", - :user_install_cmd => "yum groupinstall '%s'") - end - if !packages.empty? - result |= super(packages) - end - if result - # Invalidate caching of installed packages, as we just - # installed new packages ! - @installed_packages = nil - end - end - end - - # Package manager interface for systems that use APT and dpkg for - # package management - class AptDpkgManager < ShellScriptManager - attr_accessor :status_file - - def initialize(status_file = "/var/lib/dpkg/status") - @status_file = status_file - super(['apt-dpkg'], true, - "apt-get install '%s'", - "export DEBIAN_FRONTEND=noninteractive; apt-get install -y '%s'") - end - - # On a dpkg-enabled system, checks if the provided package is installed - # and returns true if it is the case - def installed?(package_name) - if !@installed_packages - @installed_packages = Set.new - dpkg_status = File.readlines(status_file) - dpkg_status << "" - - current_packages = [] - is_installed = false - dpkg_status.each do |line| - line = line.chomp - line = line.encode( "UTF-8", "binary", :invalid => :replace, :undef => :replace) - if line == "" - if is_installed - current_packages.each do |pkg| - @installed_packages << pkg - end - is_installed = false - end - current_packages.clear - elsif line =~ /Package: (.*)$/ - current_packages << $1 - elsif line =~ /Provides: (.*)$/ - current_packages.concat($1.split(',').map(&:strip)) - elsif line == "Status: install ok installed" - is_installed = true - end - end - end - - if package_name =~ /^(\w[a-z0-9+-.]+)/ - @installed_packages.include?($1) - else - Autoproj.warn "#{package_name} is not a valid Debian package name" - false - end - end - - def install(packages) - if super - # Invalidate caching of installed packages, as we just - # installed new packages ! - @installed_packages = nil - end - end - - def filter_uptodate_packages(packages, options = Hash.new) - packages.find_all do |package_name| - !installed?(package_name) - end - end - end - - # Package manager interface for the RubyGems system - class GemManager < Manager - class << self - attr_writer :with_prerelease - attr_accessor :with_doc - end - @with_prerelease = false - @with_doc = false - - def self.with_prerelease(*value) - if value.empty? - @with_prerelease - else - begin - saved_flag = @with_prerelease - @with_prerelease = value.first - yield - ensure - @with_prerelease = saved_flag - end - end - end - - # Filters all paths that come from other autoproj installations out - # of GEM_PATH - def self.initialize_environment(env = Autobuild.env, manifest = Autoproj.manifest, root_dir = Autoproj.root_dir) - env.original_env['GEM_PATH'] = - (env['GEM_PATH'] || "").split(File::PATH_SEPARATOR).find_all do |p| - !Autoproj.in_autoproj_installation?(p) - end.join(File::PATH_SEPARATOR) - env.inherit 'GEM_PATH' - env.init_from_env 'GEM_PATH' - - orig_gem_path = env.original_env['GEM_PATH'].split(File::PATH_SEPARATOR) - env.system_env['GEM_PATH'] = Gem.default_path - env.original_env['GEM_PATH'] = orig_gem_path.join(File::PATH_SEPARATOR) - - manifest.each_reused_autoproj_installation do |p| - p_gems = File.join(p, '.gems') - if File.directory?(p_gems) - env.push_path 'GEM_PATH', p_gems - env.push_path 'PATH', File.join(p_gems, 'bin') - end - end - - @gem_home = (ENV['AUTOPROJ_GEM_HOME'] || File.join(root_dir, ".gems")) - env.push_path 'GEM_PATH', gem_home - env.set 'GEM_HOME', gem_home - env.push_path 'PATH', "#{gem_home}/bin" - - # Now, reset the directories in our own RubyGems instance - Gem.paths = env.resolved_env - - use_cache_dir - end - - # Override the gem home detected by {initialize_environment}, or set - # it in cases where calling {initialize_environment} is not possible - def self.gem_home=(gem_home) - @gem_home = gem_home - end - - # A global cache directory that should be used to avoid - # re-downloading gems - def self.cache_dir - if dir = ENV['AUTOBUILD_CACHE_DIR'] - dir = File.join(dir, 'gems') - FileUtils.mkdir_p dir - dir - end - end - - def self.use_cache_dir - # If there is a cache directory, make sure .gems/cache points to - # it (there are no programmatic ways to override this) - if cache = cache_dir - gem_cache_dir = File.join(gem_home, 'cache') - if !File.symlink?(gem_cache_dir) || File.readlink(gem_cache_dir) != cache - FileUtils.mkdir_p gem_home - FileUtils.rm_rf gem_cache_dir - Autoproj.create_symlink(cache, gem_cache_dir) - end - end - end - - # Return the directory in which RubyGems package should be installed - def self.gem_home - @gem_home - end - - # Returns the set of default options that are added to gem - # - # By default, we add --no-user-install to un-break distributions - # like Arch that set --user-install by default (thus disabling the - # role of GEM_HOME) - def self.default_install_options - @default_install_options ||= ['--no-user-install', '--no-format-executable'] - end - - def initialize - super(['gem']) - @installed_gems = Set.new - end - - # Used to override the Gem::SpecFetcher object used by this gem - # manager. Useful mainly for testing - attr_writer :gem_fetcher - - # The set of gems installed during this autoproj session - attr_reader :installed_gems - - def gem_fetcher - if !@gem_fetcher - Autoproj.message " looking for RubyGems updates" - @gem_fetcher = Gem::SpecFetcher.fetcher - end - @gem_fetcher - end - def guess_gem_program - if Autobuild.programs['gem'] - return Autobuild.programs['gem'] - end - ruby_bin = RbConfig::CONFIG['RUBY_INSTALL_NAME'] ruby_bindir = RbConfig::CONFIG['bindir'] candidates = ['gem'] if ruby_bin =~ /^ruby(.+)$/ candidates << "gem#{$1}" end candidates.each do |gem_name| if File.file?(gem_full_path = File.join(ruby_bindir, gem_name)) - Autobuild.programs['gem'] = gem_full_path - return + return gem_full_path end end - raise ArgumentError, "cannot find a gem program (tried #{candidates.sort.join(", ")} in #{ruby_bindir})" end - def build_gem_cmdlines(gems) - with_version, without_version = gems.partition { |name, v| v } - - cmdlines = [] - if !without_version.empty? - cmdlines << without_version.flatten - end - with_version.each do |name, v| - cmdlines << [name, "-v", v] - end - cmdlines + # The content of the default {#gemfile} + # + # @param [String] autoproj_version a constraint on the autoproj version + # that should be used + # @return [String] + def default_gemfile_contents(autoproj_version = ">= 0") + ["source \"https://rubygems.org\"", + "gem \"autoproj\", \"#{autoproj_version}\""].join("\n") end - def pristine(gems) - guess_gem_program - base_cmdline = [Autobuild.tool_in_path('ruby'), '-S', Autobuild.tool('gem')] - cmdlines = [ - [*base_cmdline, 'clean'], - ] - cmdlines += build_gem_cmdlines(gems).map do |line| - base_cmdline + ["pristine", "--extensions"] + line - end - if gems_interaction(gems, cmdlines) - Autoproj.message " restoring RubyGems: #{gems.map { |g| g.join(" ") }.sort.join(", ")}" - cmdlines.each do |c| - Autobuild::Subprocess.run 'autoproj', 'osdeps', *c + # Parse the provided command line options and returns the non-options + def parse_options(args = ARGV) + options = OptionParser.new do |opt| + opt.on '--private-bundler', 'install bundler locally in the workspace' do + @private_bundler = true end - end - end - - def install(gems) - guess_gem_program - - base_cmdline = [Autobuild.tool_in_path('ruby'), '-S', Autobuild.tool('gem'), 'install', *GemManager.default_install_options] - if !GemManager.with_doc - base_cmdline << '--no-rdoc' << '--no-ri' - end - - if GemManager.with_prerelease - base_cmdline << "--prerelease" - end - - cmdlines = build_gem_cmdlines(gems).map do |line| - base_cmdline + line - end - if gems_interaction(gems, cmdlines) - Autoproj.message " installing/updating RubyGems dependencies: #{gems.map { |g| g.join(" ") }.sort.join(", ")}" - - cmdlines.each do |c| - Autobuild::Subprocess.run 'autoproj', 'osdeps', *c, - env: Hash['GEM_HOME' => Gem.paths.home, - 'GEM_PATH' => Gem.paths.path.join(":")] + opt.on '--private-autoproj', 'install autoproj locally in the workspace' do + @private_autoproj = true end - gems.each do |name, v| - installed_gems << name + opt.on '--private-gems', 'install gems locally in the prefix directory' do + @private_gems = true end - true - end - end - - # Returns the set of RubyGem packages in +packages+ that are not already - # installed, or that can be upgraded - def filter_uptodate_packages(gems, options = Hash.new) - options = validate_options options, - install_only: !Autobuild.do_update - - # Don't install gems that are already there ... - gems = gems.dup - gems.delete_if do |name, version| - next(true) if installed_gems.include?(name) - - version_requirements = Gem::Requirement.new(version || '>= 0') - installed = - if Gem::Specification.respond_to?(:find_by_name) - begin - [Gem::Specification.find_by_name(name, version_requirements)] - rescue Gem::LoadError - [] - end - else - Gem.source_index.find_name(name, version_requirements) - end - - if !installed.empty? && !options[:install_only] - # Look if we can update the package ... - dep = Gem::Dependency.new(name, version_requirements) - available = - if gem_fetcher.respond_to?(:find_matching) - non_prerelease = gem_fetcher.find_matching(dep, true, true).map(&:first) - if GemManager.with_prerelease - prerelease = gem_fetcher.find_matching(dep, false, true, true).map(&:first) - else prerelease = Array.new - end - (non_prerelease + prerelease). - map { |n, v, _| [n, v] } - - else # Post RubyGems-2.0 - type = if GemManager.with_prerelease then :complete - else :released - end - - gem_fetcher.detect(type) do |tuple| - tuple.name == name && dep.match?(tuple) - end.map { |tuple, _| [tuple.name, tuple.version] } - end - installed_version = installed.map(&:version).max - available_version = available.map { |_, v| v }.max - if !available_version - if version - raise ConfigError.new, "cannot find any gem with the name '#{name}' and version #{version}" - else - raise ConfigError.new, "cannot find any gem with the name '#{name}'" - end - end - needs_update = (available_version > installed_version) - !needs_update - else - !installed.empty? + opt.on '--private', 'whether bundler, autoproj and the workspace gems should be installed locally in the workspace' do + @private_bundler = true + @private_autoproj = true + @private_gems = true end - end - gems - end - - def parse_package_entry(entry) - if entry =~ /^([^><=~]*)([><=~]+.*)$/ - [$1.strip, $2.strip] - else - [entry] - end - end - - def gems_interaction(gems, cmdlines) - if OSDependencies.force_osdeps - return true - elsif enabled? - return true - elsif silent? - return false - end - - # We're not supposed to install rubygem packages but silent is not - # set, so display information about them anyway - puts <<-EOMSG - #{Autoproj.color("The build process and/or the packages require some Ruby Gems to be installed", :bold)} - #{Autoproj.color("and you required autoproj to not do it itself", :bold)} - You can use the --all or --ruby options to autoproj osdeps to install these - packages anyway, and/or change to the osdeps handling mode by running an - autoproj operation with the --reconfigure option as for instance - autoproj build --reconfigure - - The following command line can be used to install them manually - - #{cmdlines.map { |c| c.join(" ") }.join("\n ")} - - Autoproj expects these Gems to be installed in #{GemManager.gem_home} This can - be overridden by setting the AUTOPROJ_GEM_HOME environment variable manually - - EOMSG - print " #{Autoproj.color("Press ENTER to continue ", :bold)}" - - STDOUT.flush - STDIN.readline - puts - false - end - end - - # Using pip to install python packages - class PipManager < Manager - - attr_reader :installed_gems - - def self.initialize_environment(env = Autobuild.env, _manifest = nil, root_dir = Autoproj.root_dir) - env.set 'PYTHONUSERBASE', pip_home(env, root_dir) - end - - # Return the directory where python packages are installed to. - # The actual path is pip_home/lib/pythonx.y/site-packages. - def self.pip_home(env = Autobuild.env, root_dir = Autobuild.root_dir) - env['AUTOPROJ_PYTHONUSERBASE'] || File.join(root_dir,".pip") - end - - - def initialize - super(['pip']) - @installed_pips = Set.new - end - - def guess_pip_program - if Autobuild.programs['pip'] - return Autobuild.programs['pip'] - end - - Autobuild.programs['pip'] = "pip" - end - - def install(pips) - guess_pip_program - if pips.is_a?(String) - pips = [pips] - end - - base_cmdline = [Autobuild.tool('pip'), 'install','--user'] - - cmdlines = [base_cmdline + pips] - - if pips_interaction(pips, cmdlines) - Autoproj.message " installing/updating Python dependencies: "+ - "#{pips.sort.join(", ")}" - - cmdlines.each do |c| - Autobuild::Subprocess.run 'autoproj', 'osdeps', *c + opt.on '--version=VERSION_CONSTRAINT', String, 'use the provided string as a version constraint for autoproj' do |version| + @gemfile = default_gemfile_contents(version) end - - pips.each do |p| - @installed_pips << p + opt.on '--gemfile=PATH', String, 'use the given Gemfile to install autoproj instead of the default' do |path| + @gemfile = File.read(path) end end + options.parse(ARGV) end - - def pips_interaction(pips, cmdlines) - if OSDependencies.force_osdeps - return true - elsif enabled? - return true - elsif silent? - return false - end - # We're not supposed to install rubygem packages but silent is not - # set, so display information about them anyway - puts <<-EOMSG - #{Autoproj.color("The build process and/or the packages require some Python packages to be installed", :bold)} - #{Autoproj.color("and you required autoproj to not do it itself", :bold)} - The following command line can be used to install them manually - - #{cmdlines.map { |c| c.join(" ") }.join("\n ")} - - Autoproj expects these Python packages to be installed in #{PipManager.pip_home} This can - be overridden by setting the AUTOPROJ_PYTHONUSERBASE environment variable manually + def install_bundler + gem_program = guess_gem_program + puts "Detected 'gem' to be #{gem_program}" - EOMSG - print " #{Autoproj.color("Press ENTER to continue ", :bold)}" + result = system( + Hash['GEM_PATH' => nil, + 'GEM_HOME' => bundler_install_dir], + gem_program, 'install', '--no-document', '--no-user-install', '--no-format-executable', + "--bindir=#{File.join(bundler_install_dir, 'bin')}", 'bundler') - STDOUT.flush - STDIN.readline - puts - false - end - end - - end - - - # Manager for packages provided by external package managers - class OSDependencies - class << self - # When requested to load a file called '$FILE', the osdeps code will - # also look for files called '$FILE-suffix', where 'suffix' is an - # element in +suffixes+ - # - # A usage of this functionality is to make loading conditional to - # the available version of certain tools, namely Ruby. Autoproj for - # instance adds ruby18 when started on Ruby 1.8 and ruby19 when - # started on Ruby 1.9 - attr_reader :suffixes - end - @suffixes = [] - - def self.load(file) - if !File.file?(file) - raise ArgumentError, "no such file or directory #{file}" - end - - candidates = [file] - candidates.concat(suffixes.map { |s| "#{file}-#{s}" }) - - error_t = if defined? Psych::SyntaxError then [ArgumentError, Psych::SyntaxError] - else ArgumentError - end - - result = OSDependencies.new - candidates.each do |file| - next if !File.file?(file) - file = File.expand_path(file) - begin - data = YAML.load(File.read(file)) || Hash.new - verify_definitions(data) - rescue *error_t => e - raise ConfigError.new, "error in #{file}: #{e.message}", e.backtrace + if !result + STDERR.puts "FATAL: failed to install bundler in #{dot_autoproj}" + exit 1 end - - result.merge(OSDependencies.new(data, file)) - end - result - end - - class << self - attr_reader :aliases - attr_accessor :force_osdeps - end - @aliases = Hash.new - - attr_writer :silent - def silent?; @silent end - - def self.alias(old_name, new_name) - @aliases[new_name] = old_name - end - - def self.ruby_version_keyword - "ruby#{RUBY_VERSION.split('.')[0, 2].join("")}" - end - - def self.autodetect_ruby_program - ruby = RbConfig::CONFIG['RUBY_INSTALL_NAME'] - ruby_bindir = RbConfig::CONFIG['bindir'] - ruby_executable = File.join(ruby_bindir, ruby) - Autobuild.programs['ruby'] = ruby_executable - ruby_executable - end - - def self.autodetect_ruby - self.alias(ruby_version_keyword, "ruby") - end - self.suffixes << ruby_version_keyword - autodetect_ruby - - AUTOPROJ_OSDEPS = File.join(File.expand_path(File.dirname(__FILE__)), 'default.osdeps') - def self.load_default - file = ENV['AUTOPROJ_DEFAULT_OSDEPS'] || AUTOPROJ_OSDEPS - if !File.file?(file) - Autoproj.warn "#{file} (from AUTOPROJ_DEFAULT_OSDEPS) is not a file, falling back to #{AUTOPROJ_OSDEPS}" - file = AUTOPROJ_OSDEPS + File.join(bin_dir, 'bundler') end - OSDependencies.load(file) - end - def load_default - merge(self.class.load_default) - end - - PACKAGE_HANDLERS = [PackageManagers::AptDpkgManager, - PackageManagers::GemManager, - PackageManagers::EmergeManager, - PackageManagers::PacmanManager, - PackageManagers::HomebrewManager, - PackageManagers::YumManager, - PackageManagers::PortManager, - PackageManagers::ZypperManager, - PackageManagers::PipManager , - PackageManagers::PkgManager] - - # Mapping from OS name to package manager name - # - # Package handlers and OSes MUST have different names. The former are - # used to resolve packages and the latter to resolve OSes in the osdeps. - # Since one can force the use of a package manager in any OS by adding a - # package manager entry, as e.g. - # - # ubuntu: - # homebrew: package - # - # we need to be able to separate between OS and package manager names. - OS_PACKAGE_HANDLERS = { - 'debian' => 'apt-dpkg', - 'gentoo' => 'emerge', - 'arch' => 'pacman', - 'fedora' => 'yum', - 'macos-port' => 'macports', - 'macos-brew' => 'brew', - 'opensuse' => 'zypper', - 'freebsd' => 'pkg' - } - - # The information contained in the OSdeps files, as a hash - attr_reader :definitions - # All the information contained in all the OSdeps files, as a mapping - # from the OSdeps package name to [osdeps_file, definition] pairs - attr_reader :all_definitions - # The information as to from which osdeps file the current package - # information in +definitions+ originates. It is a mapping from the - # package name to the osdeps file' full path - attr_reader :sources - - # Use to override the autodetected OS-specific package handler - attr_writer :os_package_handler - - # Returns the package manager object for the current OS - def os_package_handler - if @os_package_handler.nil? - os_names, _ = OSDependencies.operating_system - if os_names && (key = os_names.find { |name| OS_PACKAGE_HANDLERS[name] }) - @os_package_handler = package_handlers[OS_PACKAGE_HANDLERS[key]] - if !@os_package_handler - raise ArgumentError, "found #{OS_PACKAGE_HANDLERS[name]} as the required package handler for #{os_names.join(", ")}, but it is not registered" - end - else - @os_package_handler = PackageManagers::UnknownOSManager.new + def save_env_sh + env = Autobuild::Environment.new + path = [] + if private_bundler? + env.push_path 'PATH', File.join(bundler_install_dir, 'bin') + env.push_path 'GEM_PATH', bundler_install_dir end - end - return @os_package_handler - end - - # Returns the set of package managers - def package_handlers - if !@package_handlers - @package_handlers = Hash.new - PACKAGE_HANDLERS.each do |klass| - obj = klass.new - obj.names.each do |n| - @package_handlers[n] = obj - end + env.push_path 'PATH', File.join(autoproj_install_dir, 'bin') + env.inherit 'PATH' + if private_autoproj? + env.push_path 'GEM_PATH', autoproj_install_dir end - end - @package_handlers - end - # The Gem::SpecFetcher object that should be used to query RubyGems, and - # install RubyGems packages - def initialize(defs = Hash.new, file = nil) - @definitions = defs.to_hash - @all_definitions = Hash.new { |h, k| h[k] = Array.new } - - @sources = Hash.new - @installed_packages = Set.new - if file - defs.each_key do |package_name| - sources[package_name] = file - all_definitions[package_name] << [[file], defs[package_name]] + # Generate environment files right now, we can at least use bundler + File.open(File.join(dot_autoproj, 'env.sh'), 'w') do |io| + env.export_env_sh(io) end - end - @silent = true - @filter_uptodate_packages = true - end - # Returns the name of all known OS packages - # - # It includes even the packages for which there are no definitions on - # this OS - def all_package_names - all_definitions.keys - end - - # Returns the full path to the osdeps file from which the package - # definition for +package_name+ has been taken - def source_of(package_name) - sources[package_name] - end - - # Merges the osdeps information of +info+ into +self+. If packages are - # defined in both OSDependencies objects, the information in +info+ - # takes precedence - def merge(info) - root_dir = nil - @definitions = definitions.merge(info.definitions) do |h, v1, v2| - if v1 != v2 - root_dir ||= "#{Autoproj.root_dir}/" - old = source_of(h).gsub(root_dir, '') - new = info.source_of(h).gsub(root_dir, '') - - # Warn if the new osdep definition resolves to a different - # set of packages than the old one - old_resolved = resolve_package(h).inject(Hash.new) do |osdep_h, (handler, status, list)| - osdep_h[handler.name] = [status, list] - osdep_h - end - new_resolved = info.resolve_package(h).inject(Hash.new) do |osdep_h, (handler, status, list)| - osdep_h[handler.name] = [status, list] - osdep_h - end - if old_resolved != new_resolved - Autoproj.warn("osdeps definition for #{h}, previously defined in #{old} overridden by #{new}") - end + File.open(File.join(root_dir, 'env.sh'), 'w') do |io| + io.write <<-EOSHELL +source "#{File.join(dot_autoproj, 'env.sh')}" +export AUTOPROJ_CURRENT_ROOT=#{root_dir} + EOSHELL end - v2 end - @sources = sources.merge(info.sources) - @all_definitions = all_definitions.merge(info.all_definitions) do |package_name, all_defs, new_all_defs| - all_defs = all_defs.dup - new_all_defs = new_all_defs.dup - new_all_defs.delete_if do |files, data| - if entry = all_defs.find { |_, d| d == data } - entry[0] |= files - end - end - all_defs.concat(new_all_defs) - end - end - # Perform some sanity checks on the given osdeps definitions - def self.verify_definitions(hash, path = []) - hash.each do |key, value| - if value && !key.kind_of?(String) - raise ArgumentError, "invalid osdeps definition: found an #{key.class} as a key in #{path.join("/")}. Don't forget to put quotes around numbers" - elsif !value && (key.kind_of?(Hash) || key.kind_of?(Array)) - verify_definitions(key) + def save_gemfile + FileUtils.mkdir_p File.dirname(autoproj_gemfile_path) + File.open(autoproj_gemfile_path, 'w') do |io| + io.write gemfile end - next if !value - - if value.kind_of?(Array) || value.kind_of?(Hash) - verify_definitions(value, (path + [key])) - else - if !value.kind_of?(String) - raise ArgumentError, "invalid osdeps definition: found an #{value.class} as a value in #{path.join("/")}. Don't forget to put quotes around numbers" - end - end end - end - - # Returns true if it is possible to install packages for the operating - # system on which we are installed - def self.supported_operating_system? - if @supported_operating_system.nil? - os_names, _ = operating_system - @supported_operating_system = - if !os_names then false - else - os_names.any? { |os_name| OS_PACKAGE_HANDLERS.has_key?(os_name) } - end - end - return @supported_operating_system - end - - # Used mainly during testing to bypass the operating system - # autodetection - def self.operating_system=(values) - @supported_operating_system = nil - @operating_system = values - end - - def self.guess_operating_system - if File.exists?('/etc/debian_version') - versions = [File.read('/etc/debian_version').strip] - if versions.first =~ /sid/ - versions = ["unstable", "sid"] + + def install_autoproj(bundler) + # Force bundler to update. If the user does not want this, let him specify a + # Gemfile with tighter version constraints + lockfile = File.join(File.dirname(autoproj_gemfile_path), 'Gemfile.lock') + if File.exist?(lockfile) + FileUtils.rm lockfile end - [['debian'], versions] - elsif File.exists?('/etc/redhat-release') - release_string = File.read('/etc/redhat-release').strip - release_string =~ /(.*) release ([\d.]+)/ - name = $1.downcase - version = $2 - if name =~ /Red Hat Entreprise/ - name = 'rhel' - end - [[name], [version]] - elsif File.exists?('/etc/gentoo-release') - release_string = File.read('/etc/gentoo-release').strip - release_string =~ /^.*([^\s]+)$/ - version = $1 - [['gentoo'], [version]] - elsif File.exists?('/etc/arch-release') - [['arch'], []] - elsif Autobuild.macos? - version=`sw_vers | head -2 | tail -1`.split(":")[1] - manager = - if ENV['AUTOPROJ_MACOSX_PACKAGE_MANAGER'] - ENV['AUTOPROJ_MACOSX_PACKAGE_MANAGER'] - else 'macos-brew' - end - if !OS_PACKAGE_HANDLERS.include?(manager) - known_managers = OS_PACKAGE_HANDLERS.keys.grep(/^macos/) - raise ArgumentError, "#{manager} is not a known MacOSX package manager. Known package managers are #{known_managers.join(", ")}" - end - managers = - if manager == 'macos-port' - [manager, 'port'] - else [manager] - end - [[*managers, 'darwin'], [version.strip]] - elsif Autobuild.windows? - [['windows'], []] - elsif File.exists?('/etc/SuSE-release') - version = File.read('/etc/SuSE-release').strip - version =~/.*VERSION\s+=\s+([^\s]+)/ - version = $1 - [['opensuse'], [version.strip]] - elsif Autobuild.freebsd? - version = `uname -r`.strip.split("-")[0] - [['freebsd'],[version]] - end - end + env = Hash['BUNDLE_GEMFILE' => nil, 'RUBYLIB' => nil] + opts = Array.new - def self.ensure_derivatives_refer_to_their_parents(names) - names = names.dup - version_files = Hash[ - '/etc/debian_version' => 'debian', - '/etc/redhat-release' => 'fedora', - '/etc/gentoo-release' => 'gentoo', - '/etc/arch-release' => 'arch', - '/etc/SuSE-release' => 'opensuse'] - version_files.each do |file, name| - if File.exists?(file) && !names.include?(name) - names << name + if private_autoproj? + env = Hash['GEM_PATH' => bundler_install_dir, + 'GEM_HOME' => nil] + opts << "--clean" << "--path=#{autoproj_install_dir}" end - end - names - end - - def self.normalize_os_representation(names, versions) - # Normalize the names to lowercase - names = names.map(&:downcase) - versions = versions.map(&:downcase) - if !versions.include?('default') - versions += ['default'] - end - return names, versions - end - # Autodetects the operating system name and version - # - # +osname+ is the operating system name, all in lowercase (e.g. ubuntu, - # arch, gentoo, debian) - # - # +versions+ is a set of names that describe the OS version. It includes - # both the version number (as a string) and/or the codename if there is - # one. - # - # Examples: ['debian', ['sid', 'unstable']] or ['ubuntu', ['lucid lynx', '10.04']] - def self.operating_system(options = Hash.new) - # Validate the options. We check on the availability of - # validate_options as to not break autoproj_bootstrap (in which - # validate_options is not available) - options = validate_options options, force: false, config: Autoproj.config - config = options.fetch(:config) - - if user_os = ENV['AUTOPROJ_OS'] - @operating_system = - if user_os.empty? then false - else - names, versions = user_os.split(':') - normalize_os_representation(names.split(','), versions.split(',')) - end - return @operating_system - end - - - if options[:force] - @operating_system = nil - elsif !@operating_system.nil? # @operating_system can be set to false to simulate an unknown OS - return @operating_system - elsif config.has_value_for?('operating_system') - os = config.get('operating_system') - if os.respond_to?(:to_ary) - if os[0].respond_to?(:to_ary) && os[0].all? { |s| s.respond_to?(:to_str) } && - os[1].respond_to?(:to_ary) && os[1].all? { |s| s.respond_to?(:to_str) } - @operating_system = os - return os - end - end - @operating_system = nil # Invalid OS format in the configuration file - end - - Autobuild.progress :operating_system_autodetection, "autodetecting the operating system" - names, versions = os_from_os_release - - if !names - names, versions = guess_operating_system - end - - # on Debian, they refuse to put enough information to detect - # 'unstable' reliably. So, we use the heuristic method for it - if names[0] == "debian" - # check if we actually got a debian with the "unstable" (sid) - # flavour. it seems that "/etc/debian_version" does not contain - # "sid" (but "8.0" for example) during the feature freeze - # phase... - if File.exists?('/etc/debian_version') - debian_versions = [File.read('/etc/debian_version').strip] - if debian_versions.first =~ /sid/ - versions = ["unstable", "sid"] - end - end - # otherwise "versions" contains the result it previously had - end - return if !names - - names = ensure_derivatives_refer_to_their_parents(names) - names, versions = normalize_os_representation(names, versions) - - @operating_system = [names, versions] - config.set('operating_system', @operating_system, true) - Autobuild.progress :operating_system_autodetection, "operating system: #{(names - ['default']).join(",")} - #{(versions - ['default']).join(",")}" - @operating_system - ensure - Autobuild.progress_done :operating_system_autodetection - end - - def self.os_from_os_release(filename = '/etc/os-release') - return if !File.exists?(filename) - - fields = Hash.new - File.readlines(filename).each do |line| - line = line.strip - if line.strip =~ /^(\w+)=(?:["'])?([^"']+)(?:["'])?$/ - fields[$1] = $2 - elsif !line.empty? - Autoproj.warn "could not parse line '#{line.inspect}' in /etc/os-release" - end - end - - names = [] - versions = [] - names << fields['ID'] << fields['ID_LIKE'] - versions << fields['VERSION_ID'] - version = fields['VERSION'] || '' - versions.concat(version.gsub(/[^\w.]/, ' ').split(' ')) - return names.compact.uniq, versions.compact.uniq - end - - def self.os_from_lsb - if !Autobuild.find_in_path('lsb_release') - return - end - - distributor = [`lsb_release -i -s`.strip.downcase] - codename = `lsb_release -c -s`.strip.downcase - version = `lsb_release -r -s`.strip.downcase - - return [distributor, [codename, version]] - end - - class InvalidRecursiveStatement < Autobuild::Exception; end - - # Return the path to the osdeps name for a given package name while - # accounting for package aliases - # - # returns an array contain the path starting with name and - # ending at the resolved name - def self.resolve_name(name) - path = [ name ] - while OSDependencies.aliases.has_key?(name) - name = OSDependencies.aliases[name] - path << name - end - path - end - - # Return the list of packages that should be installed for +name+ - # - # The following two simple return values are possible: - # - # nil:: +name+ has no definition - # []:: +name+ has no definition on this OS and/or for this specific OS - # version - # - # In all other cases, the method returns an array of triples: - # - # [package_handler, status, package_list] - # - # where status is FOUND_PACKAGES if +package_list+ is the list of - # packages that should be installed with +package_handler+ for +name+, - # and FOUND_NONEXISTENT if the nonexistent keyword is used for this OS - # name and version. The package list might be empty even if status == - # FOUND_PACKAGES, for instance if the ignore keyword is used. - def resolve_package(name) - path = OSDependencies.resolve_name(name) - name = path.last - - os_names, os_versions = OSDependencies.operating_system - os_names = os_names.dup - os_names << 'default' - - dep_def = definitions[name] - if !dep_def - return nil - end - - # Partition the found definition in all entries that are interesting - # for us: toplevel os-independent package managers, os-dependent - # package managers and os-independent package managers selected by - # OS or version - if !os_names - os_names = ['default'] - os_versions = ['default'] - end - - package_handler_names = package_handlers.keys - - result = [] - found, pkg = partition_osdep_entry(name, dep_def, nil, (package_handler_names - os_package_handler.names), os_names, os_versions) - if found - result << [os_package_handler, found, pkg] - end - - # NOTE: package_handlers might contain the same handler multiple - # times (when a package manager has multiple names). That's why we - # do a to_set.each - package_handlers.each_value.to_set.each do |handler| - found, pkg = partition_osdep_entry(name, dep_def, handler.names, [], os_names, os_versions) - if found - result << [handler, found, pkg] - end - end - - # Recursive resolutions - found, pkg = partition_osdep_entry(name, dep_def, ['osdep'], [], os_names, os_versions) - if found - pkg.each do |pkg_name| - resolved = resolve_package(pkg_name) - if !resolved - raise InvalidRecursiveStatement, "osdep #{pkg_name} does not exist. It is referred to by #{name}." - end - result.concat(resolved) - end - end - - result.map do |handler, status, entries| - if handler.respond_to?(:parse_package_entry) - [handler, status, entries.map { |s| handler.parse_package_entry(s) }] - else - [handler, status, entries] - end - end - end - - # Value returned by #resolve_package and #partition_osdep_entry in - # the status field. See the documentation of these methods for more - # information - FOUND_PACKAGES = 0 - # Value returned by #resolve_package and #partition_osdep_entry in - # the status field. See the documentation of these methods for more - # information - FOUND_NONEXISTENT = 1 - - # Helper method that parses the osdep definition to split between the - # parts needed for this OS and specific package handlers. - # - # +osdep_name+ is the name of the osdep. It is used to resolve explicit - # mentions of a package handler, i.e. so that: - # - # pkg: gem - # - # is resolved as the 'pkg' package to be installed by the 'gem' handler - # - # +dep_def+ is the content to parse. It can be a string, array or hash - # - # +handler_names+ is a list of entries that we are looking for. If it is - # not nil, only entries that explicitely refer to +handler_names+ will - # be browsed, i.e. in: - # - # pkg: - # - test: 1 - # - [a, list, of, packages] - # - # partition_osdep_entry('osdep_name', data, ['test'], []) - # - # will ignore the toplevel list of packages, while - # - # partition_osdep_entry('osdep_name', data, nil, []) - # - # will return it. - # - # +excluded+ is a list of branches that should be ignored during - # parsing. It is used to e.g. ignore 'gem' when browsing for the main OS - # package list. For instance, in - # - # pkg: - # - test - # - [a, list, of, packages] - # - # partition_osdep_entry('osdep_name', data, nil, ['test']) - # - # the returned value will only include the list of packages (and not - # 'test') - # - # The rest of the arguments are array of strings that contain list of - # keys to browse for (usually, the OS names and version) - # - # The return value is either nil if no packages were found, or a pair - # [status, package_list] where status is FOUND_NONEXISTENT if the - # nonexistent keyword was found, and FOUND_PACKAGES if either packages - # or the ignore keyword were found. - # - def partition_osdep_entry(osdep_name, dep_def, handler_names, excluded, *keys) - keys, *additional_keys = *keys - keys ||= [] - found = false - nonexistent = false - result = [] - found_keys = Hash.new - Array(dep_def).each do |names, values| - if !values - # Raw array of packages. Possible only if we are not at toplevel - # (i.e. if we already have a handler) - if names == 'ignore' - found = true if !handler_names - elsif names == 'nonexistent' - nonexistent = true if !handler_names - elsif !handler_names && names.kind_of?(Array) - result.concat(result) - found = true - elsif names.respond_to?(:to_str) - if excluded.include?(names) - elsif handler_names && handler_names.include?(names) - result << osdep_name - found = true - elsif !handler_names - result << names - found = true - end - elsif names.respond_to?(:to_hash) - rec_found, rec_result = partition_osdep_entry(osdep_name, names, handler_names, excluded, keys, *additional_keys) - if rec_found == FOUND_NONEXISTENT then nonexistent = true - elsif rec_found == FOUND_PACKAGES then found = true - end - result.concat(rec_result) - end - else - if names.respond_to?(:to_str) # names could be an array already - names = names.split(',') - end - - if handler_names - if matching_name = handler_names.find { |k| names.any? { |name_tag| k == name_tag.downcase } } - rec_found, rec_result = partition_osdep_entry(osdep_name, values, nil, excluded, keys, *additional_keys) - if rec_found == FOUND_NONEXISTENT then nonexistent = true - elsif rec_found == FOUND_PACKAGES then found = true - end - result.concat(rec_result) - end - end - - matching_name = keys.find { |k| names.any? { |name_tag| k == name_tag.downcase } } - if matching_name - rec_found, rec_result = partition_osdep_entry(osdep_name, values, handler_names, excluded, *additional_keys) - # We only consider the first highest-priority entry, - # regardless of whether it has some packages for us or - # not - idx = keys.index(matching_name) - if !rec_found - if !found_keys.has_key?(idx) - found_keys[idx] = nil - end - else - found_keys[idx] ||= [0, []] - found_keys[idx][0] += rec_found - found_keys[idx][1].concat(rec_result) - end - end - end - end - first_entry = found_keys.keys.sort.first - found_keys = found_keys[first_entry] - if found_keys - if found_keys[0] > 0 - nonexistent = true - else - found = true - end - result.concat(found_keys[1]) - end - - found = - if nonexistent then FOUND_NONEXISTENT - elsif found then FOUND_PACKAGES - else false - end - - return found, result - end - - # Resolves the given OS dependencies into the actual packages that need - # to be installed on this particular OS. - # - # @param [Array<String>] dependencies the list of osdep names that should be resolved - # @return [Array<#install,Array<String>>] the set of packages, grouped - # by the package handlers that should be used to install them - # - # @raise MissingOSDep if some packages can't be found or if the - # nonexistent keyword was found for some of them - def resolve_os_dependencies(dependencies) - all_packages = [] - dependencies.each do |name| - result = resolve_package(name) + result = system(env, + bundler, 'install', + "--gemfile=#{autoproj_gemfile_path}", + "--binstubs=#{File.join(autoproj_install_dir, 'bin')}", + *opts) if !result - path = OSDependencies.resolve_name(name) - raise MissingOSDep.new, "there is no osdeps definition for #{path.last} (search tree: #{path.join("->")})" + STDERR.puts "FATAL: failed to install autoproj in #{dot_autoproj}" + exit 1 end - - if result.empty? - if OSDependencies.supported_operating_system? - os_names, os_versions = OSDependencies.operating_system - raise MissingOSDep.new, "there is an osdeps definition for #{name}, but not for this operating system and version (resp. #{os_names.join(", ")} and #{os_versions.join(", ")})" - end - result = [[os_package_handler, FOUND_PACKAGES, [name]]] - end - - result.each do |handler, status, packages| - if status == FOUND_NONEXISTENT - raise MissingOSDep.new, "there is an osdep definition for #{name}, and it explicitely states that this package does not exist on your OS" - end - if entry = all_packages.find { |h, _| h == handler } - entry[1].concat(packages) - else - all_packages << [handler, packages] - end - end end - all_packages.delete_if do |handler, pkg| - pkg.empty? - end - return all_packages - end - - - # Returns true if +name+ is an acceptable OS package for this OS and - # version - def has?(name) - status = availability_of(name) - status == AVAILABLE || status == IGNORE - end - - # Value returned by #availability_of if the required package has no - # definition - NO_PACKAGE = 0 - # Value returned by #availability_of if the required package has - # definitions, but not for this OS name or version - WRONG_OS = 1 - # Value returned by #availability_of if the required package has - # definitions, but the local OS is unknown - UNKNOWN_OS = 2 - # Value returned by #availability_of if the required package has - # definitions, but the nonexistent keyword was used for this OS - NONEXISTENT = 3 - # Value returned by #availability_of if the required package is - # available - AVAILABLE = 4 - # Value returned by #availability_of if the required package is - # available, but no package needs to be installed to have it - IGNORE = 5 - - # If +name+ is an osdeps that is available for this operating system, - # returns AVAILABLE. Otherwise, returns one of: - # - # NO_PACKAGE:: the package has no definitions - # WRONG_OS:: the package has a definition, but not for this OS - # UNKNOWN_OS:: the package has a definition, but the local OS is unknown - # NONEXISTENT:: the package has a definition, but the 'nonexistent' - # keyword was found for this OS - # AVAILABLE:: the package is available for this OS - # IGNORE:: the package is available for this OS, but no packages need to - # be installed for it - def availability_of(name) - resolved = resolve_package(name) - if !resolved - return NO_PACKAGE - end - - if resolved.empty? - if !OSDependencies.operating_system - return UNKNOWN_OS - elsif !OSDependencies.supported_operating_system? - return AVAILABLE - else return WRONG_OS - end - end - - resolved = resolved.delete_if { |_, status, list| status == FOUND_PACKAGES && list.empty? } - failed = resolved.find_all do |handler, status, list| - status == FOUND_NONEXISTENT - end - if failed.empty? - if resolved.empty? - return IGNORE + def update_configuration + if File.exist?(autoproj_config_path) + config = YAML.load(File.read(autoproj_config_path)) || Hash.new else - return AVAILABLE + config = Hash.new end - else - return NONEXISTENT - end - end - - HANDLE_ALL = 'all' - HANDLE_RUBY = 'ruby' - HANDLE_OS = 'os' - HANDLE_NONE = 'none' - - def self.osdeps_mode_option_unsupported_os(config = Autoproj.config) - long_doc =<<-EOT -The software packages that autoproj will have to build may require other -prepackaged softwares (a.k.a. OS dependencies) to be installed (RubyGems -packages, packages from your operating system/distribution, ...). Autoproj is -usually able to install those automatically, but unfortunately your operating -system is not (yet) supported by autoproj's osdeps mechanism, it can only offer -you some limited support. - -Some package handlers are cross-platform, and are therefore supported. However, -you will have to install the kind of OS dependencies (so-called OS packages) - -This option is meant to allow you to control autoproj's behaviour while handling -OS dependencies. - -* if you say "all", all OS-independent packages are going to be installed. -* if you say "gem", the RubyGem packages will be installed. -* if you say "pip", the Pythin PIP packages will be installed. -* if you say "none", autoproj will not do anything related to the OS - dependencies. - -As any configuration value, the mode can be changed anytime by calling - autoproj reconfigure - -Finally, the "autoproj osdeps" command will give you the necessary information -about the OS packages that you will need to install manually. - -So, what do you want ? (all, none or a comma-separated list of: gem pip) - EOT - message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically (all, none or a comma-separated list of: gem pip) ?", long_doc.strip ] - - config.declare 'osdeps_mode', 'string', - :default => 'ruby', - :doc => message, - :lowercase => true - end - - def self.osdeps_mode_option_supported_os(config = Autoproj.config) - long_doc =<<-EOT -The software packages that autoproj will have to build may require other -prepackaged softwares (a.k.a. OS dependencies) to be installed (RubyGems -packages, packages from your operating system/distribution, ...). Autoproj -is able to install those automatically for you. - -Advanced users may want to control this behaviour. Additionally, the -installation of some packages require administration rights, which you may -not have. This option is meant to allow you to control autoproj's behaviour -while handling OS dependencies. - -* if you say "all", it will install all packages automatically. - This requires root access thru 'sudo' -* if you say "pip", only the Ruby packages will be installed. - Installing these packages does not require root access. -* if you say "gem", only the Ruby packages will be installed. - Installing these packages does not require root access. -* if you say "os", only the OS-provided packages will be installed. - Installing these packages requires root access. -* if you say "none", autoproj will not do anything related to the - OS dependencies. - -Finally, you can provide a comma-separated list of pip gem and os. - -As any configuration value, the mode can be changed anytime by calling - autoproj reconfigure - -Finally, the "autoproj osdeps" command will give you the necessary information -about the OS packages that you will need to install manually. - -So, what do you want ? (all, none or a comma-separated list of: os gem pip) - EOT - message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically (all, none or a comma-separated list of: os gem pip) ?", long_doc.strip ] - - config.declare 'osdeps_mode', 'string', - :default => 'all', - :doc => message, - :lowercase => true - end - - def self.define_osdeps_mode_option(config = Autoproj.config) - if supported_operating_system? - osdeps_mode_option_supported_os(config) - else - osdeps_mode_option_unsupported_os(config) - end - end - - def self.osdeps_mode_string_to_value(string) - string = string.to_s.downcase.split(',') - modes = [] - string.map do |str| - case str - when 'all' then modes.concat(['os', 'gem', 'pip']) - when 'ruby' then modes << 'gem' - when 'gem' then modes << 'gem' - when 'pip' then modes << 'pip' - when 'os' then modes << 'os' - when 'none' then - else raise ArgumentError, "#{str} is not a known package handler" + config['private_bundler'] = private_bundler? + config['private_autoproj'] = private_autoproj? + config['private_gems'] = private_gems? + File.open(autoproj_config_path, 'w') do |io| + YAML.dump(config, io) end end - modes - end - # If set to true (the default), #install will try to remove the list of - # already uptodate packages from the installed packages. Set to false to - # install all packages regardless of their status - attr_writer :filter_uptodate_packages - - # If set to true (the default), #install will try to remove the list of - # already uptodate packages from the installed packages. Use - # #filter_uptodate_packages= to set it to false to install all packages - # regardless of their status - def filter_uptodate_packages? - !!@filter_uptodate_packages - end - - # Override the osdeps mode - def osdeps_mode=(value) - @osdeps_mode = OSDependencies.osdeps_mode_string_to_value(value) - end - - # Returns the osdeps mode chosen by the user - def osdeps_mode - # This has two uses. It caches the value extracted from the - # AUTOPROJ_OSDEPS_MODE and/or configuration file. Moreover, it - # allows to override the osdeps mode by using - # OSDependencies#osdeps_mode= - if @osdeps_mode - return @osdeps_mode - end - - @osdeps_mode = OSDependencies.osdeps_mode - end - - def self.osdeps_mode(config = Autoproj.config) - while true - mode = - if !config.has_value_for?('osdeps_mode') && - mode_name = ENV['AUTOPROJ_OSDEPS_MODE'] - begin OSDependencies.osdeps_mode_string_to_value(mode_name) - rescue ArgumentError - Autoproj.warn "invalid osdeps mode given through AUTOPROJ_OSDEPS_MODE (#{mode})" - nil - end - else - mode_name = config.get('osdeps_mode') - begin OSDependencies.osdeps_mode_string_to_value(mode_name) - rescue ArgumentError - Autoproj.warn "invalid osdeps mode stored in configuration file" - nil - end - end - - if mode - @osdeps_mode = mode - config.set('osdeps_mode', mode_name, true) - return mode + def install + if private_bundler? + puts "Installing bundler in #{bundler_install_dir}" + bundler = install_bundler end - - # Invalid configuration values. Retry - config.reset('osdeps_mode') - ENV['AUTOPROJ_OSDEPS_MODE'] = nil + save_gemfile + puts "Installing autoproj in #{dot_autoproj}" + install_autoproj(bundler || 'bundler') end - end - # The set of packages that have already been installed - attr_reader :installed_packages - - # Set up the registered package handlers according to the specified osdeps mode - # - # It enables/disables package handlers based on either the value - # returned by {#osdeps_mode} or the value passed as option (the latter - # takes precedence). Moreover, sets the handler's silent flag using - # {#silent?} - # - # @option options [Array<String>] the package handlers that should be - # enabled. The default value is returned by {#osdeps_mode} - # @return [Array<PackageManagers::Manager>] the set of enabled package - # managers - def setup_package_handlers(options = Hash.new) - options = - if Kernel.respond_to?(:validate_options) - Kernel.validate_options options, - osdeps_mode: osdeps_mode - else - options = options.dup - options[:osdeps_mode] ||= osdeps_mode - options - end - - os_package_handler.enabled = false - package_handlers.each_value do |handler| - handler.enabled = false + # Actually perform the install + def run + install + ENV['BUNDLE_GEMFILE'] = autoproj_gemfile_path + require 'bundler' + Bundler.setup + require 'autobuild' + save_env_sh + update_configuration end - options[:osdeps_mode].each do |m| - if m == 'os' - os_package_handler.enabled = true - elsif pkg = package_handlers[m] - pkg.enabled = true - else - Autoproj.warn "osdep handler #{m.inspect} has no handler, available handlers are #{package_handlers.keys.map(&:inspect).sort.join(", ")}" - end - end - os_package_handler.silent = self.silent? - package_handlers.each_value do |v| - v.silent = self.silent? - end - - enabled_handlers = [] - if os_package_handler.enabled? - enabled_handlers << os_package_handler - end - package_handlers.each_value do |v| - if v.enabled? - enabled_handlers << v - end - end - enabled_handlers end - - # Requests that packages that are handled within the autoproj project - # (i.e. gems) are restored to pristine condition - # - # This is usually called as a rebuild step to make sure that all these - # packages are updated to whatever required the rebuild - def pristine(packages, options = Hash.new) - install(packages, options.merge(install_only: true)) - packages = resolve_os_dependencies(packages) - - _, other_packages = - packages.partition { |handler, list| handler == os_package_handler } - other_packages.each do |handler, list| - if handler.respond_to?(:pristine) - handler.pristine(list) - end - end - end - - # Requests the installation of the given set of packages - def install(packages, options = Hash.new) - # Remove the set of packages that have already been installed - packages = packages.to_set - installed_packages - return false if packages.empty? - - filter_options, options = - filter_options options, install_only: !Autobuild.do_update - setup_package_handlers(options) - - packages = resolve_os_dependencies(packages) - - needs_filter = (filter_uptodate_packages? || filter_options[:install_only]) - packages = packages.map do |handler, list| - if needs_filter && handler.respond_to?(:filter_uptodate_packages) - list = handler.filter_uptodate_packages(list, filter_options) - end - - if !list.empty? - [handler, list] - end - end.compact - return false if packages.empty? - - # Install OS packages first, as the other package handlers might - # depend on OS packages - os_packages, other_packages = packages.partition { |handler, list| handler == os_package_handler } - [os_packages, other_packages].each do |packages| - packages.each do |handler, list| - handler.install(list) - @installed_packages |= list.to_set - end - end - true - end - - def reinstall(options = Hash.new) - # We also reinstall the osdeps that provide the - # functionality - managers = setup_package_handlers(options) - managers.each do |mng| - if mng.enabled? && mng.respond_to?(:reinstall) - mng.reinstall - end - end - end end end -module Autobuild - class << self - # Configure the programs used by different packages - attr_reader :programs - # A cache of entries in programs to their resolved full path - # - # @return [{String=>[String,String,String]}] the triplet (full path, - # tool name, value of ENV['PATH']). The last two values are used to - # invalidate the cache when needed - # - # @see tool_in_path - attr_reader :programs_in_path - # Get a given program, using its name as default value. For - # instance - # tool('automake') - # will return 'automake' unless the autobuild script defined - # another automake program in Autobuild.programs by doing - # Autobuild.programs['automake'] = 'automake1.9' - def tool(name) - programs[name.to_sym] || programs[name.to_s] || name.to_s - end - - def find_in_path(file) - path = ENV['PATH'].split(File::PATH_SEPARATOR). - find { |dir| File.exist?(File.join(dir, file)) } - if path - return File.join(path, file) - end - end - - # Resolves the absolute path to a given tool - def tool_in_path(name) - path, path_name, path_env = programs_in_path[name] - current = tool(name) - if path_env != ENV['PATH'] || path_name != current - # Delete the current entry given that it is invalid - programs_in_path.delete(name) - if current[0, 1] == "/" - # This is already a full path - path = current - else - path = find_in_path(current) - end - - if !path - raise ArgumentError, "tool #{name}, set to #{current}, can not be found in PATH=#{path_env}" - end - - # Verify that the new value is a file and is executable - if !File.file?(path) - raise ArgumentError, "tool #{name} is set to #{current}, but this resolves to #{path} which is not a file" - elsif !File.executable?(path) - raise ArgumentError, "tool #{name} is set to #{current}, but this resolves to #{path} which is not executable" - end - programs_in_path[name] = [path, current, ENV['PATH']] - end - - return path - end - end - - @programs = Hash.new - @programs_in_path = Hash.new -end - - -module Autoproj - # OS-independent creation of symbolic links. Note that on windows, it only - # works for directories - def self.create_symlink(from, to) - if Autobuild.windows? - Dir.create_junction(to, from) - else - FileUtils.ln_sf from, to - end - end - - # Returns true if +path+ is part of an autoproj installation - def self.in_autoproj_installation?(path) - root_dir(File.expand_path(path)) - true - rescue UserError - false - end - - # Forcefully sets the root directory - # - # This is mostly useful during bootstrapping (i.e. when the search would - # fail) - def self.root_dir=(dir) - if @workspace && dir != @workspace.root_dir - raise WorkspaceAlreadyCreated, "cannot switch global root directory after a workspace object got created" - end - @root_dir = dir - end - - # Returns the root directory of the current autoproj installation. - # - # If the current directory is not in an autoproj installation, - # raises UserError. - def self.root_dir(dir = Dir.pwd) - if @root_dir - return @root_dir - end - path = Workspace.find_root_dir(dir) - if !path - raise UserError, "not in a Autoproj installation" - end - path - end - - # @deprecated use workspace.config_dir instead - def self.config_dir - Autoproj.warn "#{__method__} is deprecated, use workspace.config_dir instead" - caller.each { |l| Autoproj.warn " #{l}" } - workspace.config_dir - end - # @deprecated use workspace.overrides_dir instead - def self.overrides_dir - Autoproj.warn "#{__method__} is deprecated, use workspace.overrides_dir instead" - caller.each { |l| Autoproj.warn " #{l}" } - workspace.overrides_dir - end - # @deprecated use Autobuild.find_in_path instead - # - # Warning: the autobuild method returns nil (instead of raising) if the - # argument cannot be found - def self.find_in_path(name) - Autoproj.warn "#{__method__} is deprecated, use Autobuild.find_in_path instead" - caller.each { |l| Autoproj.warn " #{l}" } - if path = Autobuild.find_in_path(name) - return path - else raise ArgumentError, "cannot find #{name} in PATH (#{ENV['PATH']})" - end - end - # @deprecated use workspace.prefix_dir instead - def self.prefix - Autoproj.warn_deprecated(__method__, 'workspace.prefix_dir') - workspace.prefix_dir - end - # @deprecated use workspace.prefix_dir= instead - def self.prefix=(path) - Autoproj.warn_deprecated(__method__, 'workspace.prefix_dir=') - workspace.prefix_dir = path - end - # @deprecated use workspace.prefix_dir instead - def self.build_dir - Autoproj.warn_deprecated(__method__, 'workspace.prefix_dir') - workspace.prefix_dir - end - # @deprecated compute the full path with File.join(config_dir, file) - # directly instead - def self.config_file(file) - Autoproj.warn_deprecated(__method__, 'compute the full path with File.join(config_dir, ...) instead') - File.join(config_dir, file) - end - # @deprecated use workspace.remotes_dir instead - def self.remotes_dir - Autoproj.warn_deprecated(__method__, 'use workspace.remotes_dir instead') - workspace.remotes_dir - end - # @deprecated use workspace.load or add a separate Loader object to your class - def self.load(package_set, *path) - Autoproj.warn_deprecated( - __method__, - 'use workspace.load or add a separate Loader object to your class') - workspace.load(package_set, *path) - end - # @deprecated use workspace.load_if_present or add a separate Loader object to your class - def self.load_if_present(package_set, *path) - Autoproj.warn_deprecated( - __method__, - 'use workspace.load_if_present or add a separate Loader object to your class') - workspace.load_if_present(package_set, *path) - end - - # Run the provided command as user - def self.run_as_user(*args) - if !system(*args) - raise "failed to run #{args.join(" ")}" - end - end - # Run the provided command as root, using sudo to gain root access - def self.run_as_root(*args) - if !system(Autobuild.tool_in_path('sudo'), *args) - raise "failed to run #{args.join(" ")} as root" - end - end - - # Look into +dir+, searching for shared libraries. For each library, display - # a warning message if this library has undefined symbols. - def self.validate_solib_dependencies(dir, exclude_paths = []) - Find.find(File.expand_path(dir)) do |name| - next unless name =~ /\.so$/ - next if exclude_paths.find { |p| name =~ p } - - output = `ldd -r #{name} 2>&1` - if output =~ /undefined symbol/ - Autoproj.message(" WARN: #{name} has undefined symbols", :magenta) - end - end - end -end - - - -DEFS = <<EODEFS ---- -none: ignore -ruby19: - debian: - - ruby1.9.1 - - ruby1.9.1-dev - - rubygems1.9.1 - - rake - - rubygems-integration - ubuntu: - '12.04': - - ruby1.9.1 - - ruby1.9.1-dev - - rubygems1.9.1 - - ri1.9.1 - - libopenssl-ruby1.9.1 - - rake - default: - - ruby1.9.1 - - ruby1.9.1-dev - - rubygems1.9.1 - - ri1.9.1 - - libopenssl-ruby1.9.1 - - rake - - rubygems-integration - gentoo: - - dev-lang/ruby:1.9 - - rake - fedora: - '17': - - ruby - - rubygems - macos-port: - - ruby19 - - rake - macos-brew: - - gem: rake - opensuse: ruby19-devel - default: ignore -ruby20: - debian: - - ruby2.0 - - ruby2.0-dev - - rake - - rubygems-integration - ubuntu: - 13.10,14.04: - - ruby2.0 - - ruby2.0-dev - - rake - - rubygems-integration - fedora: - '20': - - ruby - - ruby-devel - - rubygem-rake - opensuse: ruby20-devel - macos-brew: - - gem: rake - default: ignore -ruby21: - debian: - - ruby2.1 - - ruby2.1-dev - - rake - - rubygems-integration - ubuntu: - '14.10': - - ruby2.1 - - ruby2.1-dev - - rake - - rubygems-integration - default: ignore - fedora: ruby-devel - macos-brew: - - gem: rake - default: ignore -build-essential: - debian,ubuntu: build-essential - gentoo: ignore - arch: base-devel - fedora: - - gcc-c++ - - make - - glibc-devel - darwin: ignore - opensuse: - - "@devel_C_C++" - - gcc-c++ - default: clang -autobuild: -- gem: autobuild -- osdep: readline -autoproj: -- gem: autoproj -- osdep: readline -readline: - debian,ubuntu: libreadline-dev - fedora: readline-devel - opensuse: readline-devel - arch: core/readline - macos-brew: readline - default: ignore -git: - debian: - lenny: git - default: git-core - ubuntu: git-core - gentoo: dev-vcs/git - arch: git - fedora: git - macos-port: git - macos-brew: git - opensuse: git - freebsd: git -hg: - debian,ubuntu: mercurial - gentoo: dev-vcs/mercurial - arch: mercurial - fedora: mercurial - darwin: mercurial - opensuse: mercurial - freebsd: mercurial -svn: - debian,ubuntu: subversion - gentoo: dev-util/subversion - arch: subversion - fedora: subversion - darwin: subversion - opensuse: subversion - freebsd: subversion -cmake: - debian,ubuntu: cmake - gentoo: dev-util/cmake - arch: cmake - fedora: cmake - darwin: cmake - opensuse: cmake - freebsd: cmake -autotools: - debian,ubuntu: - - automake - - autoconf - gentoo: - - sys-devel/automake - - sys-devel/autoconf - arch: - - automake - - autoconf - fedora: - - automake - - autoconf - darwin: - - automake - - autoconf - opensuse: - - automake - - autoconf - freebsd: - - automake - - autoconf -archive: - debian,ubuntu: - - tar - - unzip - gentoo: - - app-arch/tar - - app-arch/unzip - arch: - - tar - - unzip - fedora: - - tar - - unzip - macos-port: - - gnutar - - unzip - macos-brew: - - gnu-tar - opensuse: - - tar - - unzip - default: ignore -cvs: - debian,ubuntu: cvs - fedora: cvs - darwin: cvs - arch: cvs - opensuse: cvs - freebsd: cvs -pip: - debian,ubuntu: python-pip - arch: python2-pip - opensuse: python-pip - fedora: python-pip - freebsd: pip -sudo: - default: sudo - -EODEFS - -# Override Autoproj.root_dir -module Autoproj - def self.root_dir - @root_dir - end - @root_dir = Dir.pwd -end - -if File.directory?(File.join(Autoproj.root_dir, 'autoproj')) - STDERR.puts "there is already an autoproj/ directory here, cannot bootstrap" - STDERR.puts "Either delete it and attempt bootstrapping again, or source env.sh" - STDERR.puts "and use the usual autoproj workflow" - exit 1 -end - -if defined? Encoding # This is a 1.9-only thing - Encoding.default_internal = Encoding::UTF_8 - Encoding.default_external = Encoding::UTF_8 -end - -if ENV['AUTOPROJ_CURRENT_ROOT'] && ENV['AUTOPROJ_CURRENT_ROOT'] != Dir.pwd - STDERR.puts "the env.sh from #{ENV['AUTOPROJ_CURRENT_ROOT']} seem to already be sourced" - STDERR.puts "start a new shell and try to bootstrap again" - exit 1 -end - -require 'set' -curdir_entries = Dir.entries('.').to_set - [".", "..", "autoproj_bootstrap", ".gems", 'env.sh'].to_set -if !curdir_entries.empty? && ENV['AUTOPROJ_BOOTSTRAP_IGNORE_NONEMPTY_DIR'] != '1' - while true - print "The current directory is not empty, continue bootstrapping anyway ? [yes] " - STDOUT.flush - answer = STDIN.readline.chomp - if answer == "no" - exit - elsif answer == "" || answer == "yes" - # Set the environment variable since we might restart the - # autoproj_bootstrap script and -- anyway -- will run "autoproj - # bootstrap" later on - break - else - STDOUT.puts "invalid answer. Please answer 'yes' or 'no'" - STDOUT.flush - end - end -end - -# While in here, we don't have utilrb -module Kernel - def filter_options(options, defaults) - options = options.dup - filtered = Hash.new - defaults.each do |k, v| - filtered[k] = - if options.has_key?(k) then options.delete(k) - else v - end - end - return filtered, options - end - def validate_options(options, defaults) - defaults.merge(options) - end -end - -# Environment is clean, so just mark it as so unconditionally -ENV['AUTOPROJ_BOOTSTRAP_IGNORE_NONEMPTY_DIR'] = '1' - -gem_home = ENV['AUTOPROJ_GEM_HOME'] || File.join(Dir.pwd, '.gems') -gem_path = ([gem_home] + Gem.default_path).join(":") -Gem.paths = Hash['GEM_HOME' => gem_home, 'GEM_PATH' => gem_path] - -ENV['GEM_HOME'] = gem_home -ENV['GEM_PATH'] = gem_path -ENV['PATH'] = "#{ENV['GEM_HOME']}/bin:#{ENV['PATH']}" - -Autoproj::OSDependencies.define_osdeps_mode_option -osdeps_mode = Autoproj::OSDependencies.osdeps_mode.join(",") -ENV['AUTOPROJ_OSDEPS_MODE'] = osdeps_mode - -# First thing we do is install a proper ruby environment. We make sure that we -# aren't installing any gems for now (as we need to choose the right gem -# binary) by setting Autobuild.programs['gem'] to nil -Autobuild.programs['gem'] = nil -Autoproj::OSDependencies.autodetect_ruby -Autoproj::OSDependencies.autodetect_ruby_program - -osdeps_management = - if ENV['AUTOPROJ_DEFAULT_OSDEPS'] - Autoproj::OSDependencies.load(ENV['AUTOPROJ_DEFAULT_OSDEPS']) - else - Autoproj::OSDependencies.new(YAML.load(DEFS)) - end -osdeps_management.silent = false -Autoproj::PackageManagers::GemManager.gem_home = gem_home -Autoproj::PackageManagers::GemManager.use_cache_dir - -begin - STDERR.puts "autoproj: installing a proper Ruby environment (this can take a long time)" - osdeps_management.install(['ruby']) -rescue Autoproj::ConfigError => e - STDERR.puts "failed: #{e.message}" - exit(1) -end - -# Now try to find out the name of the gem binary -PACKAGES = ['build-essential', 'sudo'] - -STDERR.puts "autoproj: installing autoproj and its dependencies (this can take a long time)" -# First install the dependencies of autoproj, as we don't want them to be -# affected by the prerelease flag -begin - if !PACKAGES.empty? - osdeps_management.install(PACKAGES) - end -rescue Autoproj::ConfigError => e - STDERR.puts "failed: #{e.message}" - exit(1) -end - -File.open('env.sh', 'w') do |io| - io.write <<-EOSHELL -export RUBYOPT=-rubygems -export GEM_PATH=#{gem_path} -export GEM_HOME=#{gem_home} -export PATH=$GEM_HOME/bin:$PATH - EOSHELL -end - -# If the user specifies "dev" on the command line, install the prerelease -# version of autoproj. If it is "localdev", expect him to install autoproj and -# run autoproj bootstrap manually. -if ARGV.first != "localdev" - if ARGV.first == "dev" - ENV['AUTOPROJ_USE_PRERELEASE'] = '1' - ARGV.shift - end - - Autoproj::PackageManagers::GemManager.with_prerelease = - (ENV['AUTOPROJ_USE_PRERELEASE'] == '1') - begin - osdeps_management.install(['autobuild']) - osdeps_management.install(['autoproj']) - rescue Autoproj::ConfigError => e - STDERR.puts "failed: #{e.message}" - exit(1) - end - Autoproj::PackageManagers::GemManager.with_prerelease = false - - if !system('autoproj', 'bootstrap', *ARGV) - STDERR.puts "ERROR: failed to run autoproj bootstrap #{ARGV.join(", ")}" - exit 1 - end -end - +ops = Autoproj::Ops::Install.new(Dir.pwd) +ops.parse_options(ARGV) +ops.run