#! /usr/bin/env ruby require 'autoproj' require 'autoproj/autobuild' require 'open-uri' require 'highline' include Autoproj InputError = Autoproj::InputError module Autoproj @verbose = false @console = HighLine.new class << self attr_accessor :verbose attr_reader :console end end # Some configuration variables mail_config = Hash.new only_update_sources = false only_do_status = false no_os_deps = false debug = false # Parse the configuration options parser = OptionParser.new do |opts| opts.banner = <<-EOBANNER autoproj mode [options] where 'mode' is one of: build: import, build and install all selected packages doc: generate and install documentation for packages that have some update: only import/update packages, do not build them status: displays the state of the packages w.r.t. their source VCS bootstrap: starts a new autoproj installation. Usage: autoproj bootstrap [manifest_url|source_vcs source_url opt1=value1 opt2=value2 ...] list-sets: list all available sources update-sets: update all remote sources, but do not build Additional options: EOBANNER opts.on("--reconfigure", "re-ask all configuration options (build mode only)") do Autoproj.reconfigure = true end opts.on("--no-update", "do not update already checked-out packages (build mode only)") do Autobuild.do_update = false end opts.on("--no-osdeps", "do not install prepackaged dependencies") do no_os_deps = true end opts.on("--verbose", "verbose output") do Autoproj.verbose = true Autobuild.verbose = true Rake.application.options.trace = false end opts.on("--debug", "debugging output") do Autoproj.verbose = true Autobuild.verbose = true Rake.application.options.trace = true debug = true end opts.on('--nice NICE', Integer, 'nice the subprocesses to the given value') do |value| Autobuild.nice = value end opts.on("-h", "--help", "Show this message") do puts opts exit end opts.on("--mail-from EMAIL", String, "From: field of the sent mails") do |from_email| mail[:from] = from_email end opts.on("--mail-to EMAILS", String, "comma-separated list of emails to which the reports should be sent") do |emails| mail[:to] ||= [] mail[:to] += emails.split(',') end opts.on("--mail-subject SUBJECT", String, "Subject: field of the sent mails") do |subject_email| mail[:subject] = subject_email end opts.on("--mail-smtp HOSTNAME", String, " address of the mail server written as hostname[:port]") do |smtp| raise "invalid SMTP specification #{smtp}" unless smtp =~ /^([^:]+)(?::(\d+))?$/ mail[:smtp] = $1 mail[:port] = Integer($2) if $2 && !$2.empty? end opts.on("--mail-only-errors", "send mail only on errors") do mail[:only_errors] = true end end args = ARGV.dup parser.parse!(args) mode = args.shift selected_packages = args.dup all_env_sh = Array.new def color(*args) Autoproj.console.color(*args) end def do_status(packages) console = Autoproj.console Autobuild::Reporting.report do last_was_in_sync = false packages.sort.each do |pkg| pkg = Autobuild::Package[pkg] if !pkg.importer.respond_to?(:status) # This importer does not support status display STDERR.puts color(" this package's importer does not support status display", :bold, :red) next end lines = [] status = pkg.importer.status(pkg) if status.uncommitted_code lines << color(" contains uncommitted modifications", :red) end case status.status when Autobuild::Importer::Status::UP_TO_DATE if !status.uncommitted_code if last_was_in_sync STDERR.print ", #{pkg.name}" else STDERR.print pkg.name end last_was_in_sync = true next else lines << color(" local and remote are in sync", :green) end when Autobuild::Importer::Status::ADVANCED lines << color(" local contains #{status.local_commits.size} commit that remote does not have:", :magenta) status.local_commits.each do |line| lines << color(" #{line}", :magenta) end when Autobuild::Importer::Status::SIMPLE_UPDATE lines << color(" remote contains #{status.remote_commits.size} commit that local does not have:", :magenta) status.remote_commits.each do |line| lines << color(" #{line}", :magenta) end when Autobuild::Importer::Status::NEEDS_MERGE lines << color(" local and remote have diverged with respectively #{status.local_commits.size} and #{status.remote_commits.size} commits each", :magenta) lines << " -- local commits --" status.local_commits.each do |line| lines << color(" #{line}", :magenta) end lines << " -- remote commits --" status.remote_commits.each do |line| lines << color(" #{line}", :magenta) end end if last_was_in_sync STDERR.puts color(": local and remote are in sync", :green) end last_was_in_sync = false STDERR.print "#{pkg.name}:" if lines.size == 1 STDERR.puts lines.first else STDERR.puts STDERR.puts lines.join("\n") end end if last_was_in_sync STDERR.puts color(": local and remote are in sync", :green) end end end def do_bootstrap(*args) if File.exists?(File.join("autoproj", "manifest")) raise ConfigError, "this installation is already bootstrapped. Remove the autoproj directory if it is not the case" end if args.empty? # no argument, simply add a manifest template sample_dir = File.expand_path(File.join("..", "samples"), File.dirname(__FILE__)) manifest_data = File.read(File.join(sample_dir, "manifest")) FileUtils.mkdir_p "autoproj" File.open(File.join(Autoproj.config_dir, "manifest"), "w") do |io| io.write(manifest_data) end elsif args.size == 1 # must be a manifest file manifest_url = args.first STDERR.puts color("autoproj: downloading manifest file #{manifest_url}", :bold) manifest_data = begin open(manifest_url) { |file| file.read } rescue raise ConfigError, "cannot read #{manifest_url}, did you mean 'autoproj bootstrap VCSTYPE #{manifest_url}' ?" end FileUtils.mkdir_p "autoproj" File.open(File.join(Autoproj.config_dir, "manifest"), "w") do |io| io.write(manifest_data) end else # is a VCS definition for the manifest itself ... vcs_def = Hash.new vcs_def[:type] = args.shift vcs_def[:url] = args.shift while !args.empty? name, value = args.shift.split("=") vcs_def[name] = value end vcs = Autoproj.normalize_vcs_definition(vcs_def) Autoproj::Manifest.import_whole_installation(vcs, File.join(Dir.pwd, "autoproj")) # Now write it in the config file File.open(File.join(Autoproj.config_dir, "config.yml"), "a") do |io| io.puts <<-EOTEXT manifest_source: type: #{vcs_def.delete(:type)} url: #{vcs_def.delete(:url)} #{vcs_def.map { |k, v| "#{k}: #{v}" }.join("\n ")} EOTEXT end end end # Find the autoproj root dir begin case mode when "bootstrap" # If there is no argument, We need one more argument. It is either a VCS type or a path to a # manifest. # # In the first case, we create a new manifest and add the source given on # the command line to it. In the second case, we simply use it as a # manifest. do_bootstrap(*args) if args.empty? # no package sets defined, we're done exit(0) end only_update_sources = true when "build" when "update" Autobuild.do_build = false when "status" only_do_status = true Autobuild.do_update = false no_os_deps = true when "update-sets" only_update_sources = true when "list-sets" only_update_sources = true Autobuild.do_update = false when "doc" Autobuild.do_update = false Autobuild.do_doc = true Autobuild.only_doc = true else puts parser exit(1) end root_dir = Autoproj.root_dir Dir.chdir(root_dir) # Load user configuration Autoproj.load_config # If we are under rubygems, check that the GEM_HOME is right ... if $LOADED_FEATURES.any? { |l| l =~ /rubygems/ } if ENV['GEM_HOME'] != Autoproj.gem_home raise ConfigError, "RubyGems is already loaded with a different GEM_HOME" end end # Set the initial environment Autoproj.set_initial_env # Set up some important autobuild parameters Autobuild.prefix = Autoproj.build_dir Autobuild.srcdir = root_dir Autobuild.doc_errors = false Autobuild.do_doc = false Autobuild::Reporting << Autoproj::Reporter.new if mail_config[:to] Autobuild::Reporting << MailReporter.new(mail_config) end manifest_path = File.join(Autoproj.config_dir, 'manifest') # Load the installation's manifest a first time, to check if we should # update it ... We assume that the OS dependencies for this VCS is already # installed (i.e. that the user did not remove it) manifest = Manifest.load(manifest_path) if manifest.vcs && mode != "bootstrap" Autobuild::Reporting.report do manifest.update_yourself end manifest = Manifest.load(manifest_path) end Autoproj.manifest = manifest source_os_dependencies = manifest.each_remote_source(false). inject(Set.new) do |set, source| set << source.vcs.type if !source.local? end # If we need to install some packages to import our remote sources, do it if !no_os_deps && !source_os_dependencies.empty? STDERR.puts color("autoproj: installing prepackaged dependencies to access the source definitions", :bold) osdeps = manifest.known_os_packages osdeps.install(source_os_dependencies) end # Update the remote sources if there are any if manifest.has_remote_sources? STDERR.puts color("autoproj: updating remote definitions of package sets", :bold) Autobuild::Reporting.report do manifest.update_remote_sources end STDERR.puts end # Load init.rb files. each_source must not load the source.yml file, as # init.rb may define configuration options that are used there manifest.each_source(false) do |source| init_rb = File.join(source.local_dir, "init.rb") if File.exists?(init_rb) begin load init_rb rescue Exception => e Autoproj.filter_load_exception(e, source, init_rb) end end end # Load the required autobuild definitions STDERR.puts color("autoproj: loading build files and configuring build", :bold) if !Autoproj.reconfigure? STDERR.puts color("run 'autoproj --reconfigure' to change configuration values", :bold) end manifest.each_autobuild_file do |source, name| Autoproj.import_autobuild_file source, name end # The user is asked for configuration values both during the manifest # loading and the loading of autobuild files. Save it now. Autoproj.save_config # Now, load the package's importer configurations (from the various # source.yml files) manifest.load_importers if Autoproj.verbose # List defined packages, and in which autobuild files they are defined STDERR.puts "Available packages:" Autoproj.manifest.packages.each_value do |package, source, file| STDERR.puts " #{package.name}: #{file} from #{source.name}" end end # If in verbose mode, or if we only update sources, list the sources # # Note that we can't have the Manifest class load the source.yml file, as it # cannot resolve all constants. So we need to do it ourselves to get the # name ... if Autoproj.verbose || only_update_sources sources = manifest.each_source(false).to_a if sources.empty? STDERR.puts color("autoproj: no package sets defined in autoproj/manifest", :bold, :red) else STDERR.puts color("autoproj: available package sets", :bold) manifest.each_source(false) do |source| source_yml = source.raw_description_file STDERR.puts " #{source_yml['name']}" if source.local? STDERR.puts " local source in #{source.local_dir}" else STDERR.puts " vcs: #{source.vcs}, #{source.vcs.options.inspect}" STDERR.puts " local: #{source.local_dir}" end source.each_package. map { |pkg| pkg.name }. sort. each do |name| STDERR.puts " #{name}" end end end end if only_update_sources exit(0) end # Create the build target from the manifest if the user did not provide an # explicit one if selected_packages.empty? selected_packages = Autoproj.manifest.default_packages end if Autoproj.verbose STDERR.puts "will install #{selected_packages.to_a.join(", ")}" end if selected_packages.empty? # no packages, terminate STDERR.puts STDERR.puts color("autoproj: no packages defined", :red) exit 0 end selected_packages = selected_packages.to_set # Install prepackaged dependencies needed to import and build the packages if !no_os_deps STDERR.puts STDERR.puts color("autoproj: installing prepackaged dependencies for build system & version control", :bold) osdeps = manifest.known_os_packages osdeps.install(Autoproj.build_system_dependencies - source_os_dependencies) end # Now starts a different stage of the whole build. Until now, we were # working on the whole package set. Starting from now, we need to build the # package sets based on the layout file Autoproj.manifest.each_package_set(selected_packages) do |name, packages, enabled_packages| srcdir = File.join(Autoproj.root_dir, name) prefix = File.join(Autoproj.build_dir, name) logdir = File.join(prefix, "log") Autobuild.logdir = logdir packages.each do |pkg_name| pkg = Autobuild::Package[pkg_name] pkg.srcdir = File.join(srcdir, pkg_name) pkg.prefix = prefix pkg.logdir = logdir end # We are doing a status, now is the right time STDERR.puts if only_do_status do_status(packages) next end # ... but build only the selected packages (and avoid display of # progress messages if there is nothing to build) if !enabled_packages.empty? STDERR.puts STDERR.puts color("autoproj: now building #{name}", :bold) STDERR.puts color(" updating packages", :bold) Autobuild::Reporting.report do import_targets = enabled_packages.map { |pkg| "#{pkg}-import" } task "autoproj-#{name}-import" => import_targets Rake::Task["autoproj-#{name}-import"].invoke end # Load package manifests, apply dependencies to the autobuild definitions # and install OS packages manifest.load_package_manifests(packages) if !no_os_deps STDERR.puts color(" installing prepackaged dependencies", :bold) manifest.install_os_dependencies end # Call the prepare target now, after we did the import *and* loaded # the manifests Autobuild::Reporting.report do prepare_targets = enabled_packages.map { |pkg| "#{pkg}-prepare" } task "autoproj-#{name}-prepare" => prepare_targets Rake::Task["autoproj-#{name}-prepare"].invoke end # And now build if Autobuild.only_doc STDERR.puts color(" building and installing documentation", :bold) else STDERR.puts color(" building and installing packages", :bold) end Autobuild::Reporting.report do Autobuild.apply(enabled_packages, "autoproj-#{name}") Autobuild::Reporting.success end end # Now call the prepare target for all packages as it may be useful for # the rest of the builds and for the generation of the env.sh file # # Note that we don't really have to care about dependencies, but we do # it anyway old_update_flag = Autobuild.do_update begin Autobuild.do_update = false Autobuild::Reporting.report do prepare_targets = (packages - enabled_packages).map { |pkg_name| "#{pkg_name}-prepare" } task "autoproj-#{name}-prepare" => prepare_targets Rake::Task["autoproj-#{name}-prepare"].invoke end ensure Autobuild.do_update = old_update_flag end all_env_sh << name Autoproj.export_env_sh(name) end STDERR.puts < e STDERR.puts STDERR.puts color(e.message, :red, :bold) if debug then raise else exit 1 end rescue Interrupt STDERR.puts STDERR.puts color("Interrupted by user", :red, :bold) if debug then raise else exit 1 end end