#! /usr/bin/ruby if RUBY_VERSION < "2.3.0" STDERR.puts "autoproj requires Ruby >= 2.3.0" exit 1 elsif ENV['AUTOPROJ_CURRENT_ROOT'] && (ENV['AUTOPROJ_CURRENT_ROOT'] != Dir.pwd) STDERR.puts "it seems that you've already loaded an env.sh script in this console, open a new console and try again" exit 1 end require 'pathname' require 'optparse' require 'fileutils' require 'yaml' module Autoproj module Ops # This class contains the functionality necessary to install autoproj in a # clean root # # It can be required standalone (i.e. does not depend on anything else than # ruby and the ruby standard library) class Install class UnexpectedBinstub < RuntimeError; end # The created workspace's root directory attr_reader :root_dir # Content of the Gemfile generated to install autoproj itself attr_accessor :gemfile # The environment that is passed to the bundler installs attr_reader :env # The configuration hash attr_reader :config # A set of options that should be passed to autoproj when calling it # in a subprocess attr_reader :autoproj_options # The Ruby interpreter we use for this install attr_reader :ruby_executable # The URL of the source to be used to get gems attr_accessor :gem_source def initialize(root_dir) @root_dir = root_dir @gem_source = "https://rubygems.org" @gemfile = nil @skip_stage2 = false @autoproj_options = Array.new @env = Hash.new env['RUBYOPT'] = [] env['RUBYLIB'] = [] env['PATH'] = self.class.sanitize_env(ENV['PATH'] || "") env['BUNDLE_GEMFILE'] = [] load_config if config['ruby_executable'] != Gem.ruby raise "this autoproj installation was already bootstrapped using "\ "#{config['ruby_executable']}, but you are currently running "\ "under #{Gem.ruby}. Changing the ruby interpreter in a given "\ "workspace is not supported, you need to do a clean bootstrap" end @ruby_executable = config['ruby_executable'] @local = false unless @gems_install_path install_gems_in_gem_user_dir end env['GEM_HOME'] = [gems_gem_home] env['GEM_PATH'] = [gems_gem_home] end def env_for_child(env = self.env) env.inject(Hash.new) do |h, (k, v)| h[k] = if v && !v.empty? then v.join(File::PATH_SEPARATOR) end h end end def apply_env(env) env.each do |k, v| if v ENV[k] = v else ENV.delete(k) end end end def self.sanitize_env(value) value.split(File::PATH_SEPARATOR). find_all { |p| !in_workspace?(p) } end def self.in_workspace?(base_dir) path = Pathname.new(base_dir) while !path.root? if (path + ".autoproj").exist? || (path + "autoproj").exist? return true end path = path.parent end return false end # The path to the .autoproj configuration directory # # @return [String] def dot_autoproj; File.join(root_dir, '.autoproj') end # The path to the gemfile used to install autoproj # # @return [String] def autoproj_gemfile_path; File.join(dot_autoproj, 'Gemfile') end # The path to the autoproj configuration file # # @return [String] def autoproj_config_path; File.join(dot_autoproj, 'config.yml') end # Whether the stage2 install should be called or not def skip_stage2?; !!@skip_stage2 end # (see #skip_stage2?) def skip_stage2=(flag); @skip_stage2 = flag end # Whether we can access the network while installing def local?; !!@local end # (see #local?) def local=(flag); @local = flag end # The user-wide place where RubyGems installs gems def dot_gem_dir File.join(Gem.user_home, ".gem") end # The version and platform-specific suffix under {#dot_gem_dir} # # This is also the suffix used by bundler to install gems def gem_path_suffix @gem_path_suffix ||= Pathname.new(Gem.user_dir). relative_path_from(Pathname.new(dot_gem_dir)).to_s end # The path into which the workspace's gems should be installed # # They are installed in a versioned subdirectory of this path, e.g. # {#gem_path_suffix}. # # @return [String] attr_reader :gems_install_path # The GEM_HOME under which the workspace's gems should be installed # # @return [String] def gems_gem_home File.join(gems_install_path, gem_path_suffix) end # Sets where the workspace's gems should be installed # # @param [String] path the absolute path that should be given to # bundler. The gems themselves will be installed in the # {#gem_path_suffix} subdirectory under this def gems_install_path=(path) @gems_install_path = path end private def xdg_var(varname, default) if (env = ENV[varname]) && !env.empty? env else default end end # Install autoproj in Gem's default user dir def install_gems_in_gem_user_dir xdg_default_gem_path = xdg_var('XDG_DATA_HOME', File.join(Dir.home, '.local', 'share', 'autoproj', 'gems')) default_gem_path = File.join( Dir.home, '.autoproj', 'gems') @gems_install_path = if File.directory?(xdg_default_gem_path) xdg_default_gem_path elsif File.directory?(default_gem_path) default_gem_path else xdg_default_gem_path end end # Whether autoproj should prefer OS-independent packages over their # OS-packaged equivalents (e.g. the thor gem vs. the ruby-thor # Debian package) def prefer_indep_over_os_packages? @prefer_indep_over_os_packages end # (see #prefer_index_over_os_packages?) def prefer_indep_over_os_packages=(flag) @prefer_indep_over_os_packages = !!flag end def self.guess_gem_program ruby_bin = RbConfig::CONFIG['RUBY_INSTALL_NAME'] ruby_bindir = RbConfig::CONFIG['bindir'] candidates = ['gem'] if ruby_bin =~ /^ruby(.+)$/ candidates.unshift "gem#{$1}" end candidates.each do |gem_name| if File.file?(gem_full_path = File.join(ruby_bindir, gem_name)) return gem_full_path end end raise ArgumentError, "cannot find a gem program "\ "(tried #{candidates.sort.join(", ")} in #{ruby_bindir})" end # The content of the default {#gemfile} # # @param [String] autoproj_version a constraint on the autoproj version # that should be used # @return [String] def default_gemfile_contents(autoproj_version = ">= 2.0.0") ["source \"#{gem_source}\"", "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)", "gem \"autoproj\", \"#{autoproj_version}\"", "gem \"utilrb\", \">= 3.0.1\""].join("\n") end # Parse the provided command line options and returns the non-options def parse_options(args = ARGV) options = OptionParser.new do |opt| opt.on '--local', 'do not access the network (may fail)' do @local = true end opt.on '--skip-stage2', 'do not run the stage2 install' do @skip_stage2 = true end opt.on '--debug', 'Run in debug mode' do @autoproj_options << '--debug' end opt.on '--gem-source=URL', String, "use this source for RubyGems "\ "instead of rubygems.org" do |url| @gem_source = url end opt.on '--gems-path=PATH', "install gems under this path instead "\ "of ~/.autoproj/gems" do |path| self.gems_install_path = path end opt.on '--public-gems', "install gems in the default gem location" do self.install_gems_in_gem_user_dir end opt.on '--version=VERSION_CONSTRAINT', String, 'use the provided "\ "string as a version constraint for autoproj' do |version| if @gemfile raise "cannot give both --version and --gemfile" end @gemfile = default_gemfile_contents(version) end opt.on '--gemfile=PATH', String, 'use the given Gemfile to install "\ "autoproj instead of the default' do |path| if @gemfile raise "cannot give both --version and --gemfile" end @gemfile = File.read(path) end opt.on '--seed-config=PATH', String, 'path to a seed file that "\ "should be used to initialize the configuration' do |path| @config.merge!(YAML.load(File.read(path))) end opt.on '--prefer-os-independent-packages', 'prefer OS-independent "\ "packages (such as a RubyGem) over their OS-packaged equivalent "\ "(e.g. the thor gem vs. the ruby-thor debian package)' do @prefer_indep_over_os_packages = true end opt.on '--[no-]color', 'do not use colored output (enabled by "\ "default if the terminal supports it)' do |color| if color then @autoproj_options << "--color" else @autoproj_options << '--no-color' end end opt.on '--[no-]progress', 'do not use progress output (enabled by "\ "default if the terminal supports it)' do |progress| if progress then @autoproj_options << "--progress" else @autoproj_options << '--no-progress' end end opt.on '--[no-]interactive', 'if non-interactive, use default "\ "answer for questions' do |flag| if flag then @autoproj_options << "--interactive" else @autoproj_options << "--no-interactive" end end end args = options.parse(ARGV) @autoproj_options + args end def find_bundler(gem_program) setup_paths = IO.popen([env_for_child, Gem.ruby, gem_program, 'which','-a', 'bundler/setup']) do |io| io.read end return unless $?.success? bundler_path = File.join(gems_gem_home, 'bin', 'bundle') setup_paths.each_line do |setup_path| if File.exist?(bundler_path) && setup_path.start_with?(gems_gem_home) return bundler_path end end return end def install_bundler(gem_program, silent: false) local = ['--local'] if local? redirection = Hash.new if silent redirection = Hash[out: :close] end # Shut up the bundler warning about 'bin' not being in PATH env = self.env env['PATH'] += [File.join(gems_gem_home, 'bin')] result = system( env_for_child(env), Gem.ruby, gem_program, 'install', '--env-shebang', '--no-document', '--no-format-executable', '--clear-sources', '--source', gem_source, '--no-user-install', '--install-dir', gems_gem_home, *local, "--bindir=#{File.join(gems_gem_home, 'bin')}", 'bundler', **redirection) if !result STDERR.puts "FATAL: failed to install bundler in #{gems_gem_home}" nil end if (bundler_path = find_bundler(gem_program)) bundler_path else STDERR.puts "gem install bundler returned successfully, but still "\ "cannot find bundler in #{gems_gem_home}" nil end end def install_autoproj(bundler) # Force bundler to update. If the user does not want this, let # him specify a Gemfile with tighter version constraints lockfile = File.join(dot_autoproj, 'Gemfile.lock') if File.exist?(lockfile) FileUtils.rm lockfile end clean_env = env_for_child.dup opts = Array.new opts << '--local' if local? opts << "--path=#{gems_install_path}" shims_path = File.join(dot_autoproj, 'bin') result = system(clean_env, Gem.ruby, bundler, 'install', "--gemfile=#{autoproj_gemfile_path}", "--shebang=#{Gem.ruby}", "--binstubs=#{shims_path}", *opts, chdir: dot_autoproj) if !result STDERR.puts "FATAL: failed to install autoproj in #{dot_autoproj}" exit 1 end ensure self.class.rewrite_shims(shims_path, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home) end EXCLUDED_FROM_SHIMS = %w{rake thor} def self.rewrite_shims(shim_path, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home) FileUtils.mkdir_p shim_path File.open(File.join(shim_path, 'ruby'), 'w') do |io| io.puts "#! /bin/sh" io.puts "exec #{ruby_executable} \"$@\"" end FileUtils.chmod 0755, File.join(shim_path, 'ruby') Dir.glob(File.join(shim_path, '*')) do |bin_script| next unless File.file?(bin_script) bin_name = File.basename(bin_script) if EXCLUDED_FROM_SHIMS.include?(bin_name) FileUtils.rm_f bin_script next end next if bin_name == 'ruby' bin_shim = File.join(shim_path, bin_name) bin_script_lines = File.readlines(bin_script) next if has_autoproj_preamble?(bin_script_lines) File.open(bin_shim, 'w') do |io| if bin_name == 'bundler' || bin_name == 'bundle' io.puts shim_bundler(bin_script_lines, ruby_executable, autoproj_gemfile_path, gems_gem_home) else io.puts shim_script(bin_script_lines, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home) end end FileUtils.chmod 0755, bin_shim end end def self.new_style_bundler_binstub?(script_lines) script_lines.any? { |l| l =~ /This file was generated by Bundler/ } end def self.has_autoproj_preamble?(script_lines) script_lines.any? { |l| l =~ /Autoproj generated preamble/ } end def self.shim_bundler(script_lines, ruby_executable, autoproj_gemfile_path, gems_gem_home) return shim_bundler_old(ruby_executable, autoproj_gemfile_path, gems_gem_home) \ unless new_style_bundler_binstub?(script_lines) script_lines.insert(1, <<-RESTART_BUNDLER) # # This file was generated by Bundler. # # Autoproj generated preamble if defined?(Bundler) Bundler.with_clean_env do exec($0, *ARGV) end end ENV['BUNDLE_GEMFILE'] ||= '#{autoproj_gemfile_path}' ENV['GEM_HOME'] = '#{gems_gem_home}' ENV.delete('GEM_PATH') Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => ''] RESTART_BUNDLER script_lines.join end def self.shim_bundler_old(ruby_executable, autoproj_gemfile_path, gems_gem_home) "#! #{ruby_executable} if defined?(Bundler) Bundler.with_clean_env do exec($0, *ARGV) end end ENV['BUNDLE_GEMFILE'] ||= '#{autoproj_gemfile_path}' ENV['GEM_HOME'] = '#{gems_gem_home}' ENV.delete('GEM_PATH') Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => ''] load Gem.bin_path('bundler', 'bundler')" end def self.shim_script(script_lines, ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home) new_style = !script_lines.empty? && script_lines.any? do |l| l =~ /This file was generated by Bundler/ end load_line = script_lines.grep(/load Gem.bin_path/).first return shim_script_old(ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home, load_line) \ unless new_style script_lines.insert(1, <<-AUTOPROJ_PREAMBLE) # # This file was generated by Bundler. # # Autoproj generated preamble, v1 if defined?(Bundler) Bundler.with_clean_env do exec(Hash['RUBYLIB' => nil], $0, *ARGV) end elsif ENV['RUBYLIB'] exec(Hash['RUBYLIB' => nil], $0, *ARGV) end ENV['BUNDLE_GEMFILE'] = '#{autoproj_gemfile_path}' ENV['AUTOPROJ_CURRENT_ROOT'] = '#{root_dir}' Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => ''] AUTOPROJ_PREAMBLE return script_lines.join end def self.shim_script_old(ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home, load_line) "#! #{ruby_executable} if defined?(Bundler) Bundler.with_clean_env do exec(Hash['RUBYLIB' => nil], $0, *ARGV) end elsif ENV['RUBYLIB'] exec(Hash['RUBYLIB' => nil], $0, *ARGV) end ENV['BUNDLE_GEMFILE'] = '#{autoproj_gemfile_path}' ENV['AUTOPROJ_CURRENT_ROOT'] = '#{root_dir}' require 'rubygems' Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => ''] require 'bundler/setup' #{load_line}" end def save_env_sh(*vars) env = Autobuild::Environment.new env.prepare vars.each do |kv| k, *v = kv.split("=") v = v.join("=") if v.empty? env.unset k else env.set k, *v.split(File::PATH_SEPARATOR) end end # Generate environment files right now, we can at least use bundler File.open(File.join(dot_autoproj, 'env.sh'), 'w') do |io| env.export_env_sh(io) end # And now the root envsh env = Autobuild::Environment.new env.source_before File.join(dot_autoproj, 'env.sh') env.set('AUTOPROJ_CURRENT_ROOT', root_dir) File.open(File.join(root_dir, 'env.sh'), 'w') do |io| env.export_env_sh(io) end end def save_gemfile gemfile = if @gemfile @gemfile elsif File.file?(autoproj_gemfile_path) File.read(autoproj_gemfile_path) else default_gemfile_contents end gemfile += [ "", "config_path = File.join(__dir__, 'config.yml')", "if File.file?(config_path)", " require 'yaml'", " config = YAML.load(File.read(config_path)) || Hash.new", " (config['plugins'] || Hash.new).", " each do |plugin_name, (version, options)|", " gem plugin_name, version, **options", " end", "end" ].join("\n") FileUtils.mkdir_p File.dirname(autoproj_gemfile_path) File.open(autoproj_gemfile_path, 'w') do |io| io.write gemfile end end ENV_BUNDLE_GEMFILE_RX = /^(\s*ENV\[['"]BUNDLE_GEMFILE['"]\]\s*)(?:\|\|)?=/ def find_in_clean_path(command, *additional_paths) clean_path = env_for_child['PATH'].split(File::PATH_SEPARATOR) + additional_paths clean_path.each do |p| full_path = File.join(p, command) if File.file?(full_path) return full_path end end nil end # The path of the bin/ folder for installed gems def gem_bindir return @gem_bindir if @gem_bindir # Here, we're getting into the esotheric # # The problem is that e.g. Ubuntu and Debian install an # operating_system.rb file that sets proper OS defaults. Some # autoproj installs have it in their RUBYLIB but should not # because of limitations of autoproj 1.x. This leads to # Gem.bindir being *not* valid for subprocesses # # So, we're calling 'gem' as a subcommand to discovery the # actual bindir bindir = IO.popen(env_for_child, [Gem.ruby, '-e', 'puts "#{Gem.user_dir}/bin"']).read if bindir @gem_bindir = bindir.chomp else raise "FATAL: cannot run #{Gem.ruby} -e 'puts Gem.bindir'" end end def install if ENV['BUNDLER_GEMFILE'] raise "cannot run autoproj_install or autoproj_bootstrap while "\ "under a 'bundler exec' subcommand or having loaded an env.sh. "\ "Open a new console and try again" end gem_program = self.class.guess_gem_program puts "Detected 'gem' to be #{gem_program}" env['GEM_HOME'] = [gems_gem_home] env['GEM_PATH'] = [gems_gem_home] if bundler = find_bundler(gem_program) puts "Detected bundler at #{bundler}" else puts "Installing bundler in #{gems_gem_home}" if !(bundler = install_bundler(gem_program)) exit 1 end end self.class.rewrite_shims( File.join(dot_autoproj, 'bin'), ruby_executable, root_dir, autoproj_gemfile_path, gems_gem_home) env['PATH'].unshift File.join(dot_autoproj, 'bin') save_gemfile puts "Installing autoproj in #{gems_gem_home}" install_autoproj(bundler) end def load_config v1_config_path = File.join(root_dir, 'autoproj', 'config.yml') config = Hash.new if File.file?(v1_config_path) config.merge!(YAML.load(File.read(v1_config_path)) || Hash.new) end if File.file?(autoproj_config_path) config.merge!(YAML.load(File.read(autoproj_config_path)) || Hash.new) end ruby = RbConfig::CONFIG['RUBY_INSTALL_NAME'] ruby_bindir = RbConfig::CONFIG['bindir'] ruby_executable = File.join(ruby_bindir, ruby) if current = config['ruby_executable'] # When upgrading or reinstalling if current != ruby_executable raise "this workspace has already been initialized using "\ "#{current}, you cannot run autoproj install with "\ "#{ruby_executable}. If you know what you're doing, "\ "delete the ruby_executable line in config.yml and try again" end else config['ruby_executable'] = ruby_executable end @config = config %w{gems_install_path prefer_indep_over_os_packages}.each do |flag| instance_variable_set "@#{flag}", config.fetch(flag, false) end end def save_config config['gems_install_path'] = gems_install_path config['prefer_indep_over_os_packages'] = prefer_indep_over_os_packages? File.open(autoproj_config_path, 'w') { |io| YAML.dump(config, io) } end def autoproj_path File.join(dot_autoproj, 'bin', 'autoproj') end def run_autoproj(*args) system env_for_child.merge('BUNDLE_GEMFILE' => autoproj_gemfile_path), Gem.ruby, autoproj_path, *args, *@autoproj_options end def v1_workspace? File.file?(File.join(root_dir, 'autoproj', 'config.yml')) && !File.directory?(File.join(root_dir, '.autoproj')) end def stage1 if v1_workspace? && File.file?(v1_envsh = File.join(root_dir, 'env.sh')) FileUtils.cp v1_envsh, 'env.sh-autoproj-v1' end FileUtils.mkdir_p dot_autoproj save_config install rescue Exception FileUtils.rm_rf dot_autoproj raise end def call_stage2 clean_env = env_for_child stage2_vars = clean_env.map { |k, v| "#{k}=#{v}" } puts "starting the newly installed autoproj for stage2 install" if !run_autoproj('install-stage2', root_dir, *stage2_vars, *@autoproj_options) raise "failed to execute autoproj install-stage2" end end def stage2(*vars) require 'autobuild' puts "saving temporary env.sh and .autoproj/env.sh" save_env_sh(*vars) puts "running 'autoproj envsh' to generate a proper env.sh" if !system(Gem.ruby, autoproj_path, 'envsh', *@autoproj_options) STDERR.puts "failed to run autoproj envsh on the newly installed "\ "autoproj (#{autoproj_path})" exit 1 end # This is really needed on an existing install to install the # gems that were present in the v1 layout puts "running 'autoproj osdeps' to re-install missing gems" if !system(Gem.ruby, autoproj_path, 'osdeps', *@autoproj_options) STDERR.puts "failed to run autoproj osdeps on the newly installed "\ "autoproj (#{autoproj_path})" exit 1 end end end end end ENV.delete('BUNDLE_GEMFILE') ENV.delete('RUBYLIB') ops = Autoproj::Ops::Install.new(Dir.pwd) ops.parse_options(ARGV) ops.stage1 if !ops.skip_stage2? ops.call_stage2 end