require 'yaml'
require 'utilrb/kernel/options'
require 'set'
require 'rexml/document'

module Autoproj
    @build_system_dependencies = Set.new

    # Declare OS packages that are required to import and build the packages
    #
    # It is used by autoproj itself to install the importers and/or the build
    # systems for the packages.
    def self.add_build_system_dependency(*names)
        @build_system_dependencies |= names.to_set
    end
    class << self
        # Returns the set of OS packages that are needed to build and/or import
        # the packages
        #
        # See Autoproj.add_build_system_dependency
        attr_reader :build_system_dependencies
    end

    # Expand build options in +value+.
    #
    # The method will expand in +value+ patterns of the form $NAME, replacing
    # them with the corresponding build option.
    def self.expand_environment(value)
        # Perform constant expansion on the defined environment variables,
        # including the option set
        options = Autoproj.option_set
        options.each_key do |k|
            options[k] = options[k].to_s
        end

        loop do
            new_value = Autoproj.single_expansion(value, options)
            if new_value == value
                break
            else
                value = new_value
            end
        end
        value
    end

    @env_inherit = Set.new
    # Returns true if the given environment variable must not be reset by the
    # env.sh script, but that new values should simply be prepended to it.
    #
    # See Autoproj.env_inherit
    def self.env_inherit?(name)
        @env_inherit.include?(name)
    end
    # Declare that the given environment variable must not be reset by the
    # env.sh script, but that new values should simply be prepended to it.
    #
    # See Autoproj.env_inherit?
    def self.env_inherit(*names)
        @env_inherit |= names
    end

    # Sets an environment variable
    #
    # This sets (or resets) the environment variable +name+ to the given value.
    # If multiple values are given, they are joined with ':'
    #
    # The values can contain configuration parameters using the
    # $CONF_VARIABLE_NAME syntax.
    def self.env_set(name, *value)
        Autobuild.env_clear(name)
        env_add(name, *value)
    end

    # Adds new values to a given environment variable
    #
    # Adds the given value(s) to the environment variable named +name+. The
    # values are added using the ':' marker.
    #
    # The values can contain configuration parameters using the
    # $CONF_VARIABLE_NAME syntax.
    def self.env_add(name, *value)
        value = value.map { |v| expand_environment(v) }
        Autobuild.env_add(name, *value)
    end

    # Sets an environment variable which is a path search variable (such as
    # PATH, RUBYLIB, PYTHONPATH)
    #
    # This sets (or resets) the environment variable +name+ to the given value.
    # If multiple values are given, they are joined with ':'. Unlike env_set,
    # duplicate values will be removed.
    #
    # The values can contain configuration parameters using the
    # $CONF_VARIABLE_NAME syntax.
    def self.env_set_path(name, *value)
        Autobuild.env_clear(name)
        env_add_path(name, *value)
    end

    # Adds new values to a given environment variable, which is a path search
    # variable (such as PATH, RUBYLIB, PYTHONPATH)
    #
    # Adds the given value(s) to the environment variable named +name+. The
    # values are added using the ':' marker. Unlike env_set, duplicate values
    # will be removed.
    #
    # The values can contain configuration parameters using the
    # $CONF_VARIABLE_NAME syntax.
    #
    # This is usually used in package configuration blocks to add paths
    # dependent on the place of install, such as
    #
    #   cmake_package 'test' do |pkg|
    #     Autoproj.env_add_path 'RUBYLIB', File.join(pkg.srcdir, 'bindings', 'ruby')
    #   end
    def self.env_add_path(name, *value)
        value = value.map { |v| expand_environment(v) }
        Autobuild.env_add_path(name, *value)
    end

    # Requests that autoproj source the given shell script in its own env.sh
    # script
    def self.env_source_file(file)
        Autobuild.env_source_file(file)
    end

    # Representation of a VCS definition contained in a source.yml file or in
    # autoproj/manifest
    class VCSDefinition
        attr_reader :type
        attr_reader :url
        attr_reader :options

        # The original spec in hash form. Set if this VCSDefinition object has
        # been created using VCSDefinition.from_raw
        attr_reader :raw

        def initialize(type, url, options, raw = nil)
            if raw && !raw.respond_to?(:to_ary)
                raise ArgumentError, "wrong format for the raw field (#{raw.inspect})"
            end

            @type, @url, @options = type, url, options
            if type != "none" && type != "local" && !Autobuild.respond_to?(type)
                raise ConfigError.new, "version control #{type} is unknown to autoproj"
            end
            @raw = raw
        end

        def local?
            @type == 'local'
        end

        # Updates the VCS specification +old+ by the information contained in
        # +new+
        #
        # Both +old+ and +new+ are supposed to be in hash form. It is assumed
        # that +old+ has already been normalized by a call to
        # Autoproj.vcs_definition_to_hash. +new+ can be in "raw" form.
        def self.update_raw_vcs_spec(old, new)
            new = vcs_definition_to_hash(new)
            if new.has_key?(:type) && (old[:type] != new[:type])
                # The type changed. We replace the old definition by the new one
                # completely, and we make sure that the new definition is valid
                from_raw(new)
                new
            else
                old.merge(new)
            end
        end

        # Normalizes a VCS definition contained in a YAML file into a hash
        #
        # It handles custom source handler expansion, as well as the bad habit
        # of forgetting a ':' at the end of a line:
        #
        #   - package_name
        #     branch: value
        def self.vcs_definition_to_hash(spec)
            options = Hash.new

            plain = Array.new
            filtered_spec = Hash.new
            spec.each do |key, value|
                keys = key.to_s.split(/\s+/)
                plain.concat(keys[0..-2])
                filtered_spec[keys[-1].to_sym] = value
            end
            spec = filtered_spec

            if plain.size > 1
                raise ConfigError.new, "invalid syntax"
            elsif plain.size == 1
                short_url = plain.first
                vcs, *url = short_url.split(':')

                # Check if VCS is a known version control system or source handler
                # shortcut. If it is not, look for a local directory called
                # short_url
                if Autobuild.respond_to?(vcs)
                    spec.merge!(:type => vcs, :url => url.join(':'))
                elsif Autoproj.has_source_handler?(vcs)
                    spec = Autoproj.call_source_handler(vcs, url.join(':'), spec)
                else
                    source_dir = File.expand_path(File.join(Autoproj.config_dir, short_url))
                    if !File.directory?(source_dir)
                        raise ConfigError.new, "'#{spec.inspect}' is neither a remote source specification, nor a local source definition"
                    end
                    spec.merge!(:type => 'local', :url => source_dir)
                end
            end

            spec, vcs_options = Kernel.filter_options spec, :type => nil, :url => nil
            spec.merge!(vcs_options)
            if !spec[:url]
                # Verify that none of the keys are source handlers. If it is the
                # case, convert
                filtered_spec = Hash.new
                spec.dup.each do |key, value|
                    if Autoproj.has_source_handler?(key)
                        spec.delete(key)
                        spec = Autoproj.call_source_handler(key, value, spec)
                        break
                    end
                end
            end

            spec
        end

        # Autoproj configuration files accept VCS definitions in three forms:
        #  * as a plain string, which is a relative/absolute path
        #  * as a plain string, which is a vcs_type:url string
        #  * as a hash
        #
        # This method returns the VCSDefinition object matching one of these
        # specs. It raises ConfigError if there is no type and/or url
        def self.from_raw(spec, raw_spec = [[nil, spec]])
            spec = vcs_definition_to_hash(spec)
            if !(spec[:type] && (spec[:type] == 'none' || spec[:url]))
                raise ConfigError.new, "the source specification #{spec.inspect} misses either the VCS type or an URL"
            end

            spec, vcs_options = Kernel.filter_options spec, :type => nil, :url => nil
            return VCSDefinition.new(spec[:type], spec[:url], vcs_options, raw_spec)
        end

        def ==(other_vcs)
            return false if !other_vcs.kind_of?(VCSDefinition)
            if local?
                other_vcs.local? && url == other.url
            elsif !other_vcs.local?
                this_importer = create_autobuild_importer
                other_importer = other_vcs.create_autobuild_importer
                this_importer.repository_id == other_importer.repository_id
            end
        end

        def self.to_absolute_url(url, root_dir = nil)
            # NOTE: we MUST use nil as default argument of root_dir as we don't
            # want to call Autoproj.root_dir unless completely necessary
            # (to_absolute_url might be called on installations that are being
            # bootstrapped, and as such don't have a root dir yet).
            url = Autoproj.single_expansion(url, 'HOME' => ENV['HOME'])
            if url && url !~ /^(\w+:\/)?\/|^[:\w]+\@|^(\w+\@)?[\w\.-]+:/
                url = File.expand_path(url, root_dir || Autoproj.root_dir)
            end
            url
        end

        # Returns a properly configured instance of a subclass of
        # Autobuild::Importer that match this VCS definition
        #
        # Returns nil if the VCS type is 'none'
        def create_autobuild_importer
            return if type == "none"

            url = VCSDefinition.to_absolute_url(self.url)
            Autobuild.send(type, url, options)
        end

        # Returns a pretty representation of this VCS definition
        def to_s 
            if type == "none"
                "none"
            else
                desc = "#{type}:#{url}"
                if !options.empty?
                    desc = "#{desc} #{options.to_a.sort_by { |key, _| key.to_s }.map { |key, value| "#{key}=#{value}" }.join(" ")}"
                end
                desc
            end
        end
    end

    @custom_source_handlers = Hash.new

    # Returns true if +vcs+ refers to a source handler name added by
    # #add_source_handler
    def self.has_source_handler?(vcs)
        @custom_source_handlers.has_key?(vcs.to_s)
    end

    # Returns the source handlers associated with +vcs+
    #
    # Source handlers are added by Autoproj.add_source_handler. The returned
    # value is an object that responds to #call(url, options) and return a VCS
    # definition as a hash
    def self.call_source_handler(vcs, url, options)
        handler = @custom_source_handlers[vcs.to_s]
        if !handler
            raise ArgumentError, "there is no source handler for #{vcs}"
        else
            return handler.call(url, options)
        end
    end

    # call-seq:
    #   Autoproj.add_source_handler name do |url, options|
    #     # build a hash that represent source configuration
    #     # and return it
    #   end
    #
    # Add a custom source handler named +name+
    #
    # Custom source handlers are shortcuts that can be used to represent VCS
    # information. For instance, the gitorious_server_configuration method
    # defines a source handler that allows to easily add new gitorious packages:
    #
    #   gitorious_server_configuration 'GITORIOUS', 'gitorious.org'
    #
    # defines the "gitorious" source handler, which allows people to write
    #
    #
    #   version_control:
    #       - tools/orocos.rb
    #         gitorious: rock-toolchain/orocos-rb
    #         branch: test
    #
    # 
    def self.add_source_handler(name, &handler)
        @custom_source_handlers[name.to_s] = lambda(&handler)
    end

    # Does a non-recursive expansion in +data+ of configuration variables
    # ($VAR_NAME) listed in +definitions+
    #
    # If the values listed in +definitions+ also contain configuration
    # variables, they do not get expanded
    def self.single_expansion(data, definitions)
        if !data.respond_to?(:to_str)
            return data
        end
        definitions = { 'HOME' => ENV['HOME'] }.merge(definitions)

        data = data.gsub /\$(\w+)/ do |constant_name|
            constant_name = constant_name[1..-1]
            if !(value = definitions[constant_name])
                if !(value = Autoproj.user_config(constant_name))
                    if !block_given? || !(value = yield(constant_name))
                        raise ArgumentError, "cannot find a definition for $#{constant_name}"
                    end
                end
            end
            value
        end
        data
    end

    def self.expand(value, definitions = Hash.new)
        if value.respond_to?(:to_hash)
            value.dup.each do |name, definition|
                value[name] = expand(definition, definitions)
            end
            value
        elsif value.respond_to?(:to_ary)
            value.map { |val| expand(val, definitions) }
        else
            value = single_expansion(value, definitions)
            if contains_expansion?(value)
                raise ConfigError.new, "some expansions are not defined in #{value.inspect}"
            end
            value
        end
    end

    # True if the given string contains expansions
    def self.contains_expansion?(string); string =~ /\$/ end

    def self.resolve_one_constant(name, value, result, definitions)
        result[name] = single_expansion(value, result) do |missing_name|
            result[missing_name] = resolve_one_constant(missing_name, definitions.delete(missing_name), result, definitions)
        end
    end

    # Resolves all possible variable references from +constants+
    #
    # I.e. replaces variables by their values, so that no value in +constants+
    # refers to variables defined in +constants+
    def self.resolve_constant_definitions(constants)
        constants = constants.dup
        constants['HOME'] = ENV['HOME']
        
        result = Hash.new
        while !constants.empty?
            name  = constants.keys.first
            value = constants.delete(name)
            resolve_one_constant(name, value, result, constants)
        end
        result
    end

    # A package set is a version control repository which contains general
    # information with package version control information (source.yml file),
    # package definitions (.autobuild files), and finally definition of
    # dependencies that are provided by the operating system (.osdeps file).
    class PackageSet
        attr_reader :manifest
        # The VCSDefinition object that defines the version control holding
        # information for this source. Local package sets (i.e. the ones that are not
        # under version control) use the 'local' version control name. For them,
        # local? returns true.
        attr_accessor :vcs

        # The set of OSDependencies object that represent the osdeps files
        # available in this package set
        attr_reader :all_osdeps

        # The OSDependencies which is a merged version of all OSdeps in
        # #all_osdeps
        attr_reader :osdeps

        # If this package set has been imported from another package set, this
        # is the other package set object
        attr_accessor :imported_from

        # If true, this package set has been loaded because another set imports
        # it. If false, it is loaded explicitely by the user
        def explicit?; !@imported_from end

        attr_reader :source_definition
        attr_reader :constants_definitions

        # Sets the auto_imports flag. See #auto_imports?
        attr_writer :auto_imports
        # If true (the default), imports listed in this package set will be
        # automatically loaded by autoproj
        def auto_imports?; !!@auto_imports end

        # Returns the Metapackage object that has the same name than this
        # package set
        def metapackage
            manifest.metapackage(name)
        end

        # List of the packages that are built if the package set is selected in
        # the layout
        def default_packages
            metapackage.packages
        end

        # Create this source from a VCSDefinition object
        def initialize(manifest, vcs)
            @manifest = manifest
            @vcs = vcs
            @osdeps = OSDependencies.new
            @all_osdeps = []

            @provides = Set.new
            @imports  = Array.new
            @auto_imports = true
        end

        # Load a new osdeps file for this package set
        def load_osdeps(file)
            new_osdeps = OSDependencies.load(file)
            @all_osdeps << new_osdeps
            @osdeps.merge(@all_osdeps.last)
            new_osdeps
        end

        # Enumerate all osdeps package names from this package set
        def each_osdep(&block)
            @osdeps.definitions.each_key(&block)
        end

        # True if this source has already been checked out on the local autoproj
        # installation
        def present?; File.directory?(raw_local_dir) end
        # True if this source is local, i.e. is not under a version control
        def local?; vcs.local? end
        # True if this source defines nothing
        def empty?
            !source_definition['version_control'] && !source_definition['overrides']
                !each_package.find { true } &&
                !File.exists?(File.join(raw_local_dir, "overrides.rb")) &&
                !File.exists?(File.join(raw_local_dir, "init.rb"))
        end

        # Create a PackageSet instance from its description as found in YAML
        # configuration files
        def self.from_spec(manifest, raw_spec, load_description)
            if raw_spec.respond_to?(:to_str)
                local_path = File.join(Autoproj.config_dir, raw_spec)
                if File.directory?(local_path)
                    raw_spec = { :type => 'local', :url => local_path }
                end
            end
            spec = VCSDefinition.vcs_definition_to_hash(raw_spec)
            options, vcs_spec = Kernel.filter_options spec, :auto_imports => true

            # Look up for short notation (i.e. not an explicit hash). It is
            # either vcs_type:url or just url. In the latter case, we expect
            # 'url' to be a path to a local directory
            vcs_spec = Autoproj.expand(vcs_spec, manifest.constant_definitions)
            vcs_def  = VCSDefinition.from_raw(vcs_spec, [[nil, raw_spec]])

            source = PackageSet.new(manifest, vcs_def)
            source.auto_imports = options[:auto_imports]
            if load_description
                if source.present?
                    source.load_description_file
                else
                    raise InternalError, "cannot load description file as it has not been checked out yet"
                end
            else
                # Try to load just the name from the source.yml file
                source.load_minimal
            end

            source
        end

        # Returns a string that uniquely represents the version control
        # information for this package set.
        #
        # I.e. for two package sets set1 and set2, if set1.repository_id ==
        # set2.repository_id, it means that both package sets are checked out
        # from exactly the same source.
        def repository_id
            if local?
                local_dir
            else
                importer = vcs.create_autobuild_importer
                if importer.respond_to?(:repository_id)
                    importer.repository_id
                else
                    vcs.to_s
                end
            end
        end

        # Remote sources can be accessed through a hidden directory in
        # $AUTOPROJ_ROOT/.remotes, or through a symbolic link in
        # autoproj/remotes/
        #
        # This returns the former. See #user_local_dir for the latter.
        #
        # For local sources, is simply returns the path to the source directory.
        def raw_local_dir
            if local?
                File.expand_path(vcs.url)
            else
                File.expand_path(File.join(Autoproj.remotes_dir, vcs.to_s.gsub(/[^\w]/, '_')))
            end
        end

        # Remote sources can be accessed through a hidden directory in
        # $AUTOPROJ_ROOT/.remotes, or through a symbolic link in
        # autoproj/remotes/
        #
        # This returns the latter. See #raw_local_dir for the former.
        #
        # For local sources, is simply returns the path to the source directory.
        def user_local_dir
            if local?
                return vcs.url 
            else
                File.join(Autoproj.config_dir, 'remotes', name)
            end
        end

        # The directory in which data for this source will be checked out
        def local_dir
            ugly_dir   = raw_local_dir
            pretty_dir = user_local_dir
            if File.symlink?(pretty_dir) && File.readlink(pretty_dir) == ugly_dir
                pretty_dir
            else
                ugly_dir
            end
        end

        def required_autoproj_version
            definition = @source_definition || raw_description_file
            definition['required_autoproj_version'] || '0'
        end

        # Returns the source name
        def name
            if @name
                @name
            else
                vcs.to_s
            end
        end

        # Loads the source.yml file, validates it and returns it as a hash
        #
        # Raises InternalError if the source has not been checked out yet (it
        # should have), and ConfigError if the source.yml file is not valid.
        def raw_description_file
            if !present?
                raise InternalError, "source #{vcs} has not been fetched yet, cannot load description for it"
            end

            source_file = File.join(raw_local_dir, "source.yml")
            if !File.exists?(source_file)
                raise ConfigError.new, "source #{vcs.type}:#{vcs.url} should have a source.yml file, but does not"
            end

            source_definition = Autoproj.in_file(source_file, Autoproj::YAML_LOAD_ERROR) do
                YAML.load(File.read(source_file))
            end

            if !source_definition || !source_definition['name']
                raise ConfigError.new(source_file), "in #{source_file}: missing a 'name' field"
            end

            source_definition
        end

        # Load and validate the self-contained information from the YAML hash
        def load_minimal
            # If @source_definition is set, it means that load_description_file
            # has been called and that therefore all information has already
            # been parsed
            definition = @source_definition || raw_description_file
            @name = definition['name']

            if @name !~ /^[\w_\.-]+$/
                raise ConfigError.new(source_file),
                    "in #{source_file}: invalid source name '#{@name}': source names can only contain alphanumeric characters, and .-_"
            elsif @name == "local"
                raise ConfigError.new(source_file),
                    "in #{source_file}: the name 'local' is a reserved name"
            end

            @provides = (definition['provides'] || Set.new).to_set
            @imports  = (definition['imports'] || Array.new).map do |set_def|
                pkg_set = Autoproj.in_file(source_file) do
                    PackageSet.from_spec(manifest, set_def, false)
                end

                pkg_set.imported_from = self
                pkg_set
            end

        rescue InternalError
            # This ignores raw_description_file error if the package set is not
            # checked out yet
        end

        # Yields the imports this package set declares, as PackageSet instances
        def each_imported_set(&block)
            @imports.each(&block)
        end

        # Path to the source.yml file
        def source_file
            File.join(local_dir, 'source.yml')
        end

        # Load the source.yml file and resolves all information it contains.
        #
        # This for instance requires configuration options to be defined. Use
        # PackageSet#load_minimal to load only self-contained information
        def load_description_file
            if @source_definition
                return
            end

            @source_definition = raw_description_file
            load_minimal

            # Compute the definition of constants
            Autoproj.in_file(source_file) do
                constants = source_definition['constants'] || Hash.new
                @constants_definitions = Autoproj.resolve_constant_definitions(constants)
            end
        end

        def single_expansion(data, additional_expansions = Hash.new)
            if !source_definition
                load_description_file
            end
            Autoproj.single_expansion(data, additional_expansions.merge(constants_definitions))
        end

        # Expands the given string as much as possible using the expansions
        # listed in the source.yml file, and returns it. Raises if not all
        # variables can be expanded.
        def expand(data, additional_expansions = Hash.new)
            if !source_definition
                load_description_file
            end
            Autoproj.expand(data, additional_expansions.merge(constants_definitions))
        end

        # Returns the default importer definition for this package set, as a
        # VCSDefinition instance
        def default_importer
            importer_definition_for('default')
        end

        # Returns an importer definition for the given package, if one is
        # available. Otherwise returns nil.
        #
        # The returned value is a VCSDefinition object.
        def version_control_field(package_name, section_name, validate = true)
            urls = source_definition['urls'] || Hash.new
            urls['HOME'] = ENV['HOME']

            all_vcs     = source_definition[section_name]
            if all_vcs
                if all_vcs.kind_of?(Hash)
                    raise ConfigError.new, "wrong format for the #{section_name} section, you forgot the '-' in front of the package names"
                elsif !all_vcs.kind_of?(Array)
                    raise ConfigError.new, "wrong format for the #{section_name} section"
                end
            end

            raw = []
            vcs_spec = Hash.new

            if all_vcs
                all_vcs.each do |spec|
                    spec = spec.dup
                    if spec.values.size != 1
                        # Maybe the user wrote the spec like
                        #   - package_name:
                        #     type: git
                        #     url: blah
                        #
                        # or as
                        #   - package_name
                        #     type: git
                        #     url: blah
                        #
                        # In that case, we should have the package name as
                        # "name => nil". Check that.
                        name, _ = spec.find { |n, v| v.nil? }
                        if name
                            spec.delete(name)
                        else
                            name, _ = spec.find { |n, v| n =~ / \w+$/ }
                            name =~ / (\w+)$/
                            spec[$1] = spec.delete(name)
                            name = name.gsub(/ \w+$/, '')
                        end
                    else
                        name, spec = spec.to_a.first
                        if name =~ / (\w+)/
                            spec = { $1 => spec }
                            name = name.gsub(/ \w+$/, '')
                        end

                        if spec.respond_to?(:to_str)
                            if spec == "none"
                                spec = { :type => "none" }
                            else
                                raise ConfigError.new, "invalid VCS specification in the #{section_name} section '#{name}: #{spec}'"
                            end
                        end
                    end

                    name_match = name
                    if name_match =~ /[^\w\/_-]/
                        name_match = Regexp.new("^" + name_match)
                    end
                    if name_match === package_name
                        raw << [self.name, spec]
                        vcs_spec =
                            begin
                                VCSDefinition.update_raw_vcs_spec(vcs_spec, spec)
                            rescue ConfigError => e
                                raise ConfigError.new, "invalid VCS definition in the #{section_name} section for '#{name}': #{e.message}", e.backtrace
                            end
                    end
                end
            end

            if !vcs_spec.empty?
                expansions = Hash["PACKAGE" => package_name,
                    "PACKAGE_BASENAME" => File.basename(package_name),
                    "AUTOPROJ_ROOT" => Autoproj.root_dir,
                    "AUTOPROJ_CONFIG" => Autoproj.config_dir,
                    "AUTOPROJ_SOURCE_DIR" => local_dir]

                vcs_spec = expand(vcs_spec, expansions)
                vcs_spec.dup.each do |name, value|
                    vcs_spec[name] = expand(value, expansions)
                end

                # If required, verify that the configuration is a valid VCS
                # configuration
                if validate
                    begin
                        VCSDefinition.from_raw(vcs_spec)
                    rescue ConfigError => e
                        raise ConfigError.new, "invalid resulting VCS definition for package #{package_name}: #{e.message}", e.backtrace
                    end
                end
                return vcs_spec, raw
            else
                return nil, []
            end
        end

        # Returns the VCS definition for +package_name+ as defined in this
        # source, or nil if the source does not have any.
        #
        # The definition is an instance of VCSDefinition
        def importer_definition_for(package_name)
            Autoproj.in_file source_file do
                vcs_spec, raw = version_control_field(package_name, 'version_control')
                if vcs_spec
                    VCSDefinition.from_raw(vcs_spec, raw)
                end
            end
        end

        # Enumerates the Autobuild::Package instances that are defined in this
        # source
        def each_package
            if !block_given?
                return enum_for(:each_package)
            end

            Autoproj.manifest.packages.each_value do |pkg|
                if pkg.package_set.name == name
                    yield(pkg.autobuild)
                end
            end
        end

        # True if this package set provides the given package set name. I.e. if
        # it has this name or the name is listed in the "replaces" field of
        # source.yml
        def provides?(name)
            name == self.name ||
                provides.include?(name)
        end
    end

    # Specialization of the PackageSet class for the overrides listed in autoproj/
    class LocalPackageSet < PackageSet
        def initialize(manifest)
            super(manifest, VCSDefinition.from_raw(:type => 'local', :url => Autoproj.config_dir))
        end

        def name
            'local'
        end
        def load_minimal
        end
        def repository_id
            'local'
        end

        def source_file
            File.join(Autoproj.config_dir, "overrides.yml")
        end

        # Returns the default importer for this package set
        def default_importer
            importer_definition_for('default') ||
                VCSDefinition.from_raw(:type => 'none')
        end

        def raw_description_file
            path = source_file
            if File.file?(path)
                data = Autoproj.in_file(path, Autoproj::YAML_LOAD_ERROR) do
                    YAML.load(File.read(path)) || Hash.new
                end
                data['name'] = 'local'
                data
            else
                { 'name' => 'local' }
            end
        end
    end

    # DEPRECATED. For backward-compatibility only.
    Source = PackageSet
    # DEPRECATED. For backward-compatibility only.
    LocalSource = LocalPackageSet

    # Class used to store information about a package definition
    class PackageDefinition
        attr_reader :autobuild
        attr_reader :user_blocks
        attr_reader :package_set
        attr_reader :file
        def setup?; !!@setup end
        attr_writer :setup
        attr_accessor :vcs

        def initialize(autobuild, package_set, file)
            @autobuild, @package_set, @file =
                autobuild, package_set, file
            @user_blocks = []
        end

        def name
            autobuild.name
        end

        def add_setup_block(block)
            user_blocks << block
            if setup?
                block.call(autobuild)
            end
        end
    end

    # A set of packages that can be referred to by name
    class Metapackage
        # The metapackage name
        attr_reader :name
        # The packages listed in this metapackage
        attr_reader :packages

        def initialize(name)
            @name = name
            @packages = []
        end
        # Adds a package to this metapackage
        def add(pkg)
            @packages << pkg
        end
        def each_package(&block)
            @packages.each(&block)
        end
        def include?(pkg)
            if !pkg.respond_to?(:to_str)
                pkg = pkg.name
            end
            @packages.any? { |p| p.name == pkg }
        end
    end

    # The Manifest class represents the information included in the main
    # manifest file, and allows to manipulate it
    class Manifest

        # Data structure used to use autobuild importers without a package, to
        # import configuration data.
        #
        # It has to match the interface of Autobuild::Package that is relevant
        # for importers
        class FakePackage < Autobuild::Package
            attr_reader :srcdir
            attr_reader :importer

            # Used by the autobuild importers
            attr_accessor :updated

            def initialize(text_name, srcdir, importer = nil)
                super(text_name)
                @srcdir = srcdir
                @importer = importer
                @@packages.delete(text_name)
            end

            def import
                importer.import(self)
            end

            def add_stat(*args)
            end
        end

        # The set of packages that are selected by the user, either through the
        # manifest file or through the command line, as a set of package names
        attr_accessor :explicit_selection

        # Returns true if +pkg_name+ has been explicitely selected
        def explicitly_selected_package?(pkg_name)
            explicit_selection && explicit_selection.include?(pkg_name)
        end

        # Loads the manifest file located at +file+ and returns the Manifest
        # instance that represents it
	def self.load(file)
	    manifest = Manifest.new
            manifest.load(file)
            manifest
	end

        # Load the manifest data contained in +file+
        def load(file)
            if !File.exists?(file)
                raise ConfigError.new(File.dirname(file)), "expected an autoproj configuration in #{File.dirname(file)}, but #{file} does not exist"
            end

            data = Autoproj.in_file(file, Autoproj::YAML_LOAD_ERROR) do
                YAML.load(File.read(file))
            end

            @file = file
            @data = data

            if data['constants']
                @constant_definitions = Autoproj.resolve_constant_definitions(data['constants'])
            end
        end

        # The manifest data as a Hash
        attr_reader :data

        # The set of packages defined so far as a mapping from package name to 
        # [Autobuild::Package, package_set, file] tuple
        attr_reader :packages

        # A mapping from package names into PackageManifest objects
        attr_reader :package_manifests

        # The path to the manifest file that has been loaded
        attr_reader :file

        # True if osdeps should be handled in update and build, or left to the
        # osdeps command
        def auto_osdeps?
            if data.has_key?('auto_osdeps')
                !!data['auto_osdeps']
            else true
            end
        end

        # True if autoproj should run an update automatically when the user
        # uses" build"
        def auto_update?
            !!data['auto_update']
        end

        attr_reader :constant_definitions

        attr_reader :metapackages

	def initialize
            @file = nil
	    @data = nil
            @packages = Hash.new
            @package_manifests = Hash.new
            @automatic_exclusions = Hash.new
            @constants_definitions = Hash.new
            @disabled_imports = Set.new
            @moved_packages = Hash.new
            @osdeps_overrides = Hash.new
            @metapackages = Hash.new
            @ignored_os_dependencies = Set.new

            @constant_definitions = Hash.new
            if Autoproj.has_config_key?('manifest_source')
                @vcs = VCSDefinition.from_raw(Autoproj.user_config('manifest_source'))
            end
	end


        # Call this method to ignore a specific package. It must not be used in
        # init.rb, as the manifest is not yet loaded then
        def ignore_package(package_name)
            list = (data['ignore_packages'] ||= Array.new)
            list << package_name
        end

        # True if the given package should not be built, with the packages that
        # depend on him have this dependency met.
        #
        # This is useful if the packages are already installed on this system.
        def ignored?(package_name)
            if data['ignore_packages']
                data['ignore_packages'].any? do |l|
                    if package_name == l
                        true
                    elsif (pkg_set = metapackages[l]) && pkg_set.include?(package_name)
                        true
                    else
                        false
                    end
                end
            else
                false
            end
        end

        # Removes all registered exclusions
        def clear_exclusions
            automatic_exclusions.clear
            if excl = data['exclude_packages']
                excl.clear
            end
        end

        # Removes all registered ignored packages
        def clear_ignored
            if ignored = data['ignore_packages']
                ignored.clear
            end
        end

        # The set of package names that are listed in the excluded_packages
        # section of the manifest
        def manifest_exclusions
            data['exclude_packages'] || Set.new
        end

        # A package_name => reason map of the exclusions added with #add_exclusion.
        # Exclusions listed in the manifest file are returned by #manifest_exclusions
        attr_reader :automatic_exclusions

        # Exclude +package_name+ from the build. +reason+ is a string describing
        # why the package is to be excluded.
        def add_exclusion(package_name, reason)
            automatic_exclusions[package_name] = reason
        end

        # If +package_name+ is excluded from the build, returns a string that
        # tells why. Otherwise, returns nil
        #
        # Packages can either be excluded because their name is listed in the
        # excluded_packages section of the manifest, or because they are
        # disabled on this particular operating system.
        def exclusion_reason(package_name)
            if manifest_exclusions.any? { |l| Regexp.new(l) =~ package_name }
                "#{package_name} is listed in the excluded_packages section of the manifest"
            else
                automatic_exclusions[package_name]
            end
        end

        # True if the given package should not be built and its dependencies
        # should be considered as met.
        #
        # This is useful to avoid building packages that are of no use for the
        # user.
        def excluded?(package_name)
            if manifest_exclusions.any? { |l| Regexp.new(l) =~ package_name }
                true
            elsif automatic_exclusions.any? { |pkg_name, | pkg_name == package_name }
                true
            else
                false
            end
        end

        # Lists the autobuild files that are in the package sets we know of
	def each_autobuild_file(source_name = nil, &block)
            if !block_given?
                return enum_for(:each_source_file, source_name)
            end

            # This looks very inefficient, but it is because source names are
            # contained in source.yml and we must therefore load that file to
            # check the package set name ...
            #
            # And honestly I don't think someone will have 20 000 package sets
            done_something = false
            each_package_set(false) do |source| 
                next if source_name && source.name != source_name
                done_something = true

                Dir.glob(File.join(source.local_dir, "*.autobuild")).each do |file|
                    yield(source, file)
                end
            end

            if source_name && !done_something
                raise ConfigError.new(file), "in #{file}: package set '#{source_name}' does not exist"
            end
	end

        # Yields each osdeps definition files that are present in our sources
        def each_osdeps_file
            if !block_given?
                return enum_for(:each_source_file)
            end

            each_package_set(false) do |source|
		Dir.glob(File.join(source.local_dir, "*.osdeps")).each do |file|
		    yield(source, file)
		end
            end
        end

        # True if some of the sources are remote sources
        def has_remote_sources?
            each_remote_source(false).any? { true }
        end

        # True if calling update_remote_sources will actually do anything
        def should_update_remote_sources
            if Autobuild.do_update
                return true
            end

            each_remote_source(false) do |source|
                if !File.directory?(source.local_dir)
                    return true
                end
            end
            false
        end

        # Like #each_package_set, but filters out local package sets
        def each_remote_package_set(load_description = true)
            if !block_given?
                enum_for(:each_remote_package_set, load_description)
            else
                each_package_set(load_description) do |source|
                    if !source.local?
                        yield(source)
                    end
                end
            end
        end

        def each_remote_source(*args, &block)
            each_remote_package_set(*args, &block)
        end

        # Helper method for #each_package_set
        def enumerate_package_set(pkg_set, explicit_sets, all_sets) # :nodoc:
            if @disabled_imports.include?(pkg_set.name)
                pkg_set.auto_imports = false
            end

            result = []
            if pkg_set.auto_imports?
                pkg_set.each_imported_set do |imported_set|
                    if explicit_sets.any? { |src| src.vcs == imported_set.vcs } ||
                        all_sets.any? { |src| src.vcs == imported_set.vcs }
                        next
                    end

                    all_sets << imported_set
                    result.concat(enumerate_package_set(imported_set, explicit_sets, all_sets))
                end
            end
            result << pkg_set
            result
        end

        # call-seq:
        #   each_package_set { |pkg_set| ... }
        #
        # Lists all package sets defined in this manifest, by yielding a
        # PackageSet object that describes it.
        def each_package_set(load_description = true, &block)
            if !block_given?
                return enum_for(:each_package_set, load_description)
            end

            if @package_sets
                if load_description
                    @package_sets.each do |src|
                        if !src.source_definition
                            src.load_description_file
                        end
                    end
                end
                return @package_sets.each(&block)
            end

	    explicit_sets = (data['package_sets'] || []).map do |spec|
                Autoproj.in_file(self.file) do
                    PackageSet.from_spec(self, spec, load_description)
                end
            end

            all_sets = Array.new
            explicit_sets.each do |pkg_set|
                all_sets.concat(enumerate_package_set(pkg_set, explicit_sets, all_sets + [pkg_set]))
            end

            # Now load the local source 
            local = LocalPackageSet.new(self)
            if load_description
                local.load_description_file
            else
                local.load_minimal
            end
            if !load_description || !local.empty?
                all_sets << local
            end
            
            if load_description
                all_sets.each(&:load_description_file)
            else
                all_sets.each(&:load_minimal)
            end
            all_sets.each(&block)
        end

        # DEPRECATED. For backward-compatibility only
        #
        # use #each_package_set instead
        def each_source(*args, &block)
            each_package_set(*args, &block)
        end

        def local_package_set
            each_package_set.find { |s| s.kind_of?(LocalPackageSet) }
        end

        # Save the currently known package sets. After this call,
        # #each_package_set will always return the same set regardless of
        # changes on the manifest's data structures
        def cache_package_sets
            @package_sets = each_package_set(false).to_a
            @package_sets.each do |pkg_set|
                @metapackages[pkg_set.name] ||= Metapackage.new(pkg_set.name)
                @metapackages["#{pkg_set.name}.all"] ||= Metapackage.new("#{pkg_set.name}.all")
            end
        end

        # Register a new package
        def register_package(package, block, source, file)
            pkg = PackageDefinition.new(package, source, file)
            if block
                pkg.add_setup_block(block)
            end
            @packages[package.name] = pkg
            @metapackages[pkg.package_set.name].add(pkg.autobuild)
            @metapackages["#{pkg.package_set.name}.all"].add(pkg.autobuild)
        end

        def definition_source(package_name)
            if pkg_def = @packages[package_name]
                pkg_def.package_set
            end
        end
        def definition_file(package_name)
            if pkg_def = @packages[package_name]
                pkg_def.file
            end
        end

        def package(name)
            packages[name]
        end

        # Lists all defined packages as PackageDefinition objects
        def each_package_definition(&block)
            if !block_given?
                return enum_for(:each_package_definition)
            end
            packages.each_value(&block)
        end

        # Lists all defined autobuild packages as instances of
        # Autobuild::Package and its subclasses
        def each_autobuild_package
            if !block_given?
                return enum_for(:each_package)
            end
            packages.each_value { |pkg| yield(pkg.autobuild) }
        end

        # DEPRECATED: use either #each_autobuild_package and
        # #each_package_definition
        def each_package(&block)
            each_autobuild_package(&block)
        end

        # The VCS object for the main configuration itself
        attr_reader :vcs

        def each_configuration_source
            if !block_given?
                return enum_for(:each_configuration_source)
            end

            if vcs
                yield(vcs, "autoproj main configuration", Autoproj.config_dir)
            end

            each_remote_source(false) do |source|
                yield(source.vcs, source.name || source.vcs.url, source.local_dir)
            end
            self
        end

        # Creates an autobuild package whose job is to allow the import of a
        # specific repository into a given directory.
        #
        # +vcs+ is the VCSDefinition file describing the repository, +text_name+
        # the name used when displaying the import progress, +pkg_name+ the
        # internal name used to represent the package and +into+ the directory
        # in which the package should be checked out.
        def self.create_autobuild_package(vcs, text_name, into)
            importer     = vcs.create_autobuild_importer
            return if !importer # updates have been disabled by using the 'none' type

            FakePackage.new(text_name, into, importer)

        rescue Autobuild::ConfigException => e
            raise ConfigError.new, "cannot import #{name}: #{e.message}", e.backtrace
        end

        # Imports or updates a source (remote or otherwise).
        #
        # See create_autobuild_package for informations about the arguments.
        def self.update_package_set(vcs, text_name, into)
            fake_package = create_autobuild_package(vcs, text_name, into)
            fake_package.import

        rescue Autobuild::ConfigException => e
            raise ConfigError.new, "cannot import #{name}: #{e.message}", e.backtrace
        end

        # Updates the main autoproj configuration
        def update_yourself
            Manifest.update_package_set(vcs, "autoproj main configuration", Autoproj.config_dir)
        end

        def update_remote_set(pkg_set, remotes_symlinks_dir = nil)
            Manifest.update_package_set(
                pkg_set.vcs,
                pkg_set.name,
                pkg_set.raw_local_dir)

            if remotes_symlinks_dir
                pkg_set.load_minimal
                symlink_dest = File.join(remotes_symlinks_dir, pkg_set.name)

                # Check if the current symlink is valid, and recreate it if it
                # is not
                if File.symlink?(symlink_dest)
                    dest = File.readlink(symlink_dest)
                    if dest != pkg_set.raw_local_dir
                        FileUtils.rm_f symlink_dest
                        FileUtils.ln_sf pkg_set.raw_local_dir, symlink_dest
                    end
                else
                    FileUtils.rm_f symlink_dest
                    FileUtils.ln_sf pkg_set.raw_local_dir, symlink_dest
                end

                symlink_dest
            end
        end

        # Updates all the remote sources in ROOT_DIR/.remotes, as well as the
        # symbolic links in ROOT_DIR/autoproj/remotes
        def update_remote_package_sets
            remotes_symlinks_dir = File.join(Autoproj.config_dir, 'remotes')
            FileUtils.mkdir_p remotes_symlinks_dir

            # Iterate on the remote sources, without loading the source.yml
            # file (we're not ready for that yet)
            #
            # Do it iteratively to properly take imports into account, but we
            # first unconditionally update all the existing sets to properly
            # handle imports that have been removed
            updated_sets     = Hash.new
            known_remotes = []
            each_remote_package_set(false) do |pkg_set|
                next if !pkg_set.explicit?
                if pkg_set.present?
                    known_remotes << update_remote_set(pkg_set, remotes_symlinks_dir)
                    updated_sets[pkg_set.repository_id] = pkg_set
                end
            end

            old_updated_sets = nil
            while old_updated_sets != updated_sets
                old_updated_sets = updated_sets.dup
                each_remote_package_set(false) do |pkg_set|
                    next if updated_sets.has_key?(pkg_set.repository_id)

                    if !pkg_set.explicit?
                        Autoproj.message "  #{pkg_set.imported_from.name}: auto-importing #{pkg_set.name}"
                    end
                    known_remotes << update_remote_set(pkg_set, remotes_symlinks_dir)
                    updated_sets[pkg_set.repository_id] = pkg_set
                end
            end

            # Check for directories in ROOT_DIR/.remotes that do not map to a
            # source repository, and remove them
            Dir.glob(File.join(Autoproj.remotes_dir, '*')).each do |dir|
                dir = File.expand_path(dir)
                if File.directory?(dir) && !updated_sets.values.find { |pkg| pkg.raw_local_dir == dir }
                    FileUtils.rm_rf dir
                end
            end

            # Now remove obsolete symlinks
            Dir.glob(File.join(remotes_symlinks_dir, '*')).each do |file|
                if File.symlink?(file) && !known_remotes.include?(file)
                    FileUtils.rm_f file
                end
            end
        end

        # DEPRECATED. For backward-compatibility only
        def update_remote_sources(*args, &block)
            update_remote_package_sets(*args, &block)
        end

        def importer_definition_for(package_name, package_source = nil)
            if !package_source
                package_source = packages.values.
                    find { |pkg| pkg.autobuild.name == package_name }.
                    package_set
            end

            sources = each_package_set.to_a.dup

            # Remove sources listed before the package source
            while !sources.empty? && sources[0].name != package_source.name
                sources.shift
            end
            package_source = sources.shift
            if !package_source
                raise InternalError, "cannot find the package set that defines #{package_name}"
            end

            # Get the version control information from the package source. There
            # must be one
            vcs_spec, raw = Autoproj.in_file package_source.source_file do
                package_source.version_control_field(package_name, 'version_control')
            end
            return if !vcs_spec

            sources.each do |src|
                overrides_spec, raw_additional = src.version_control_field(package_name, 'overrides', false)
                raw = raw.concat(raw_additional)
                if overrides_spec
                    vcs_spec = Autoproj.in_file src.source_file do
                        begin
                            VCSDefinition.update_raw_vcs_spec(vcs_spec, overrides_spec)
                        rescue ConfigError => e
                            raise ConfigError.new, "invalid resulting VCS specification in the overrides section for package #{package_name}: #{e.message}"
                        end
                    end
                end
            end
            VCSDefinition.from_raw(vcs_spec, raw)
        end

        # Sets up the package importers based on the information listed in
        # the source's source.yml 
        #
        # The priority logic is that we take the package sets one by one in the
        # order listed in the autoproj main manifest, and first come first used.
        #
        # A set that defines a particular package in its autobuild file
        # *must* provide the corresponding VCS line in its source.yml file.
        # However, it is possible for a source that does *not* define a package
        # to override the VCS
        #
        # In other words: if package P is defined by source S1, and source S0
        # imports S1, then
        #  * S1 must have a VCS line for P
        #  * S0 can have a VCS line for P, which would override the one defined
        #    by S1
        def load_importers
            packages.each_value do |pkg|
                vcs = importer_definition_for(pkg.autobuild.name, pkg.package_set) ||
                    pkg.package_set.default_importer

                if vcs
                    Autoproj.add_build_system_dependency vcs.type
                    pkg.vcs = vcs
                    pkg.autobuild.importer = vcs.create_autobuild_importer
                else
                    raise ConfigError.new, "source #{pkg.package_set.name} defines #{pkg.autobuild.name}, but does not provide a version control definition for it"
                end
            end
        end

        # Returns true if +name+ is the name of a package set known to this
        # autoproj installation
        def has_package_set?(name)
            each_package_set(false).find { |set| set.name == name }
        end

        # Returns the PackageSet object for the given package set, or raises
        # ArgumentError if none exists with that name
        def package_set(name)
            set = each_package_set(false).find { |set| set.name == name }
            if !set
                raise ArgumentError, "no package set called #{name} exists"
            end
            set
        end

        # Exception raised when a caller requires to use an excluded package
        class ExcludedPackage < ConfigError
            attr_reader :name
            def initialize(name)
                @name = name
            end
        end

        # Resolves the given +name+, where +name+ can either be the name of a
        # source or the name of a package.
        #
        # The returned value is a list of pairs:
        #
        #   [type, package_name]
        #
        # where +type+ can either be :package or :osdeps (as symbols)
        #
        # The returned array can be empty if +name+ is an ignored package
        def resolve_package_name(name)
            if pkg_set = find_metapackage(name)
                if !pkg_set
                    raise ConfigError.new, "#{name} is neither a package nor a package set name. Packages in autoproj must be declared in an autobuild file."
                end
                pkg_names = pkg_set.each_package.map(&:name)
            else
                pkg_names = [name]
            end

            result = []
            pkg_names.each do |pkg|
                result.concat(resolve_single_package_name(pkg))
            end
            result
        end

        # Resolves all the source-based dependencies of this package (excluding
        # the OS dependencies). The result is returned as a set of package
        # names.
        def resolve_packages_dependencies(*root_names)
            result = Set.new
            queue = root_names.dup
            while pkg_name = queue.shift
                next if result.include?(pkg_name)
                result << pkg_name

                pkg = Autobuild::Package[pkg_name]
                pkg.dependencies.each do |dep_name|
                    queue << dep_name
                end
            end
            result
        end

        # Resolves a package name, where +name+ cannot be resolved as a
        # metapackage
        #
        # This is a helper method for #resolve_package_name. Do not use
        # directly
        def resolve_single_package_name(name) # :nodoc:
            if ignored?(name)
                return []
            end

            explicit_selection  = explicitly_selected_package?(name)
	    osdeps_availability = Autoproj.osdeps.availability_of(name)
            available_as_source = Autobuild::Package[name]

            osdeps_overrides = Autoproj.manifest.osdeps_overrides[name]
            if osdeps_overrides
                source_packages    = osdeps_overrides[:packages].dup
                force_source_usage = osdeps_overrides[:force]
                begin
                    source_packages = source_packages.inject([]) do |result, src_pkg_name|
                        result.concat(resolve_package_name(src_pkg_name))
                    end.uniq
                    available_as_source = true
                rescue ExcludedPackage
                    force_source_usage = false
                    available_as_source = false
                end

                if source_packages.empty?
                    source_packages << [:package, name]
                end
            end

            if force_source_usage
                return source_packages
            elsif !explicit_selection 
                if osdeps_availability == Autoproj::OSDependencies::AVAILABLE
                    return [[:osdeps, name]]
                elsif osdeps_availability == Autoproj::OSDependencies::IGNORE
                    return []
                end

                if osdeps_availability == Autoproj::OSDependencies::UNKNOWN_OS
                    # If we can't handle that OS, but other OSes have a
                    # definition for it, we assume that it can be installed as
                    # an external package. However, if it is also available as a
                    # source package, prompt the user
                    if !available_as_source || explicit_osdeps_selection(name)
                        return [[:osdeps, name]]
                    end
                end

                # No source, no osdeps. Call osdeps again, but this time to get
                # a proper error message.
                if !available_as_source
                    begin
                        Autoproj.osdeps.resolve_os_dependencies([name].to_set)
                    rescue Autoproj::ConfigError => e
                        if osdeps_availability != Autoproj::OSDependencies::NO_PACKAGE && !Autoproj.osdeps.installs_os_packages?
                            Autoproj.warn "in #{File.join(srcdir, 'manifest.xml')}: #{e.message}"
                            Autoproj.warn "this osdeps dependency is simply ignored as you asked autoproj to not install osdeps packages"
                            @ignored_os_dependencies << name
                            # We are not asked to install OS packages, just ignore
                            return []
                        end
                        raise
                    end
                    # Should never reach further than that
                end
            elsif !available_as_source
                raise ConfigError, "cannot resolve #{name}: it is not a package, not a metapackage and not an osdeps"
            end
            return [[:package, name]]
        end

        # +name+ can either be the name of a source or the name of a package. In
        # the first case, we return all packages defined by that source. In the
        # latter case, we return the singleton array [name]
        def resolve_package_set(name)
            if Autobuild::Package[name]
                [name]
            else
                pkg_set = find_metapackage(name)
                if !pkg_set
                    raise ConfigError.new, "#{name} is neither a package nor a package set name. Packages in autoproj must be declared in an autobuild file."
                end
                pkg_set.each_package.
                    map(&:name).
                    find_all { |pkg_name| !Autoproj.osdeps || !Autoproj.osdeps.has?(pkg_name) }
            end
        end

        def find_metapackage(name)
            @metapackages[name.to_s]
        end

        # call-seq:
        #   metapackage 'meta_name' => Metapackage
        #   metapackage 'meta_name', 'pkg1', 'pkg2' => Metapackage
        #
        # Metapackage definition
        #
        # In the first form, returns a Metapackage instance for the metapackage
        # named 'meta_name'.
        #
        # In the second form, adds the listed packages to the metapackage and
        # returns the Metapackage instance
        def metapackage(name, *packages, &block)
            meta = (@metapackages[name.to_s] ||= Metapackage.new(name))
            packages.each do |pkg_name|
                package_names = resolve_package_set(pkg_name)
                package_names.each do |pkg_name|
                    meta.add(Autobuild::Package[pkg_name])
                end
            end

            if block
                meta.instance_eval(&block)
            end
            meta
        end

        # Lists all defined metapackages
        #
        # Autoproj defines one metapackage per package set, which by default
        # includes all the packages that the package set defines.
        def each_metapackage(&block)
            metapackages.each_value(&block)
        end

        # Returns the packages contained in the provided layout definition
        #
        # If recursive is false, yields only the packages at this level.
        # Otherwise, return all packages.
        def layout_packages(result, validate)
            Autoproj.in_file(self.file) do
                normalized_layout.each_key do |pkg_or_set|
                    begin
                        resolve_package_set(pkg_or_set).each do |pkg_name|
                            if !excluded?(pkg_name) && !ignored?(pkg_name)
                                result << pkg_name
                            end
                            Autobuild::Package[pkg_name].all_dependencies(result)
                        end
                    rescue ConfigError
                        raise if validate
                    end
                end
            end
            result
        end

        # Enumerates the sublayouts defined in +layout_def+.
        def each_sublayout(layout_def)
            layout_def.each do |value|
                if value.kind_of?(Hash)
                    name, layout = value.find { true }
                    yield(name, layout)
                end
            end
        end

        # Returns the set of package names that are explicitely listed in the
        # layout, minus the excluded and ignored ones
        def all_layout_packages(validate = true)
            default_packages(validate)
        end

        # Returns all defined package names, minus the excluded and ignored ones
        def all_package_names
            Autobuild::Package.each.map { |name, _| name }.to_set
        end

        # Returns all the packages that can be built in this installation
        def all_packages
            result = Set.new
            each_package_set do |pkg_set|
                result |= metapackage(pkg_set.name).packages.map(&:name).to_set
            end
            result.to_a.
                find_all { |pkg_name| !Autoproj.osdeps || !Autoproj.osdeps.has?(pkg_name) }
        end

        # Returns true if +name+ is a valid package and is included in the build
        #
        # If +validate+ is true, the method will raise ArgumentError if the
        # package does not exists. 
        #
        # If it is false, the method will simply return false on non-defined
        # packages 
        def package_enabled?(name, validate = true)
            if !Autobuild::Package[name] && !Autoproj.osdeps.has?(name)
                if validate
                    raise ArgumentError, "package #{name} does not exist"
                end
                return false
            end

            !excluded?(name)
        end

        # Returns true if +name+ is a valid package and is neither excluded from
        # the build, nor ignored from the build
        #
        # If +validate+ is true, the method will raise ArgumentError if the
        # package does not exists. 
        #
        # If it is false, the method will simply return false on non-defined
        # packages 
        def package_selected?(name, validate = true)
            if package_enabled?(name)
                !ignored?(name)
            end
        end

        # Returns the set of packages that are selected by the layout
        def all_selected_packages
            result = default_packages.to_set
            result.each do |pkg_name|
                Autobuild::Package[pkg_name].all_dependencies(result)
            end
            result
        end

        # Returns the set of packages that should be built if the user does not
        # specify any on the command line
        def default_packages(validate = true)
            names = if layout = data['layout']
                        layout_packages(Set.new, validate)
                    else
                        # No layout, all packages are selected
                        all_packages
                    end

            names.delete_if { |pkg_name| excluded?(pkg_name) || ignored?(pkg_name) }
            names.to_set
        end

        def normalized_layout(result = Hash.new, layout_level = '/', layout_data = (data['layout'] || Hash.new))
            layout_data.each do |value|
                if value.kind_of?(Hash)
                    subname, subdef = value.find { true }
                    if subdef
                        normalized_layout(result, "#{layout_level}#{subname}/", subdef)
                    end
                else
                    result[value] = layout_level
                end
            end
            result
        end

        # Returns the package directory for the given package name
        def whereis(package_name)
            Autoproj.in_file(self.file) do
                set_name = definition_source(package_name).name
                actual_layout = normalized_layout
                return actual_layout[package_name] || actual_layout[set_name] || '/'
            end
        end

        def resolve_optional_dependencies
            packages.each_value do |pkg|
                pkg.autobuild.resolve_optional_dependencies
            end
        end

        # Loads the package's manifest.xml file for the current package
        #
        # Right now, the absence of a manifest makes autoproj only issue a
        # warning. This will later be changed into an error.
        def load_package_manifest(pkg_name)
            pkg = packages.values.
                find { |pkg| pkg.autobuild.name == pkg_name }
            package, package_set, file = pkg.autobuild, pkg.package_set, pkg.file

            if !pkg_name
                raise ArgumentError, "package #{pkg_name} is not defined"
            end

            manifest_paths =
                [File.join(package_set.local_dir, "manifests", package.name + ".xml"), File.join(package.srcdir, "manifest.xml")]
            manifest_path = manifest_paths.find do |path|
                File.directory?(File.dirname(path)) &&
                    File.file?(path)
            end

            if !manifest_path
                Autoproj.warn "#{package.name} from #{package_set.name} does not have a manifest"
                return
            end

            manifest = PackageManifest.load(package, manifest_path)
            pkg.autobuild.description = manifest
            package_manifests[package.name] = manifest

            manifest.each_dependency do |name, is_optional|
                begin
                    if is_optional
                        package.optional_dependency name
                    else
                        package.depends_on name
                    end
                rescue Autobuild::ConfigException => e
                    raise ConfigError.new(manifest_path),
                        "manifest #{manifest_path} of #{package.name} from #{package_set.name} lists '#{name}' as dependency, which is listed in the layout of #{file} but has no autobuild definition", e.backtrace
                rescue ConfigError => e
                    raise ConfigError.new(manifest_path),
                        "manifest #{manifest_path} of #{package.name} from #{package_set.name} lists '#{name}' as dependency, but it is neither a normal package nor an osdeps package. osdeps reports: #{e.message}", e.backtrace
                end
            end
        end

        # Loads the manifests for all packages known to this project.
        #
        # See #load_package_manifest
        def load_package_manifests(selected_packages)
            selected_packages.each(&:load_package_manifest)
        end

        # Disable all automatic imports from the given package set name
        def disable_imports_from(pkg_set_name)
            @disabled_imports << pkg_set_name
        end

        # call-seq:
        #   list_os_dependencies(packages) => required_packages, ospkg_to_pkg
        #
        # Returns the set of dependencies required by the listed packages.
        #
        # +required_packages+ is the set of osdeps names that are required for
        # +packages+ and +ospkg_to_pkg+ a mapping from the osdeps name to the
        # set of packages that require this OS package.
        def list_os_dependencies(packages)
            required_os_packages = Set.new
            package_os_deps = Hash.new { |h, k| h[k] = Array.new }
            packages.each do |pkg_name|
                pkg = Autobuild::Package[pkg_name]
                if !pkg
                    raise InternalError, "internal error: #{pkg_name} is not a package"
                end

                pkg.os_packages.each do |osdep_name|
                    package_os_deps[osdep_name] << pkg_name
                    required_os_packages << osdep_name
                end
            end

            return required_os_packages, package_os_deps
        end

        # Installs the OS dependencies that are required by the given packages
        def install_os_dependencies(packages)
            required_os_packages, package_os_deps = list_os_dependencies(packages)
            required_os_packages.delete_if do |pkg|
                if excluded?(pkg)
                    raise ConfigError.new, "the osdeps package #{pkg} is excluded from the build in #{file}. It is required by #{package_os_deps[pkg].join(", ")}"
                end
                if ignored?(pkg)
                    if Autoproj.verbose
                        Autoproj.message "ignoring osdeps package #{pkg}"
                    end
                    true
                end
            end
            Autoproj.osdeps.install(required_os_packages, package_os_deps)
        end

        # The set of overrides added with #add_osdeps_overrides
        attr_reader :osdeps_overrides

        # Declares that autoproj should use normal package(s) to provide the
        # +osdeps_name+ OS package in cases +osdeps_name+ does not exist.
        #
        # The full syntax is
        #
        #   Autoproj.add_osdeps_overrides 'opencv', :package => 'external/opencv'
        #
        # If more than one packages should be built, use the :packages option
        # with an array:
        #
        #   Autoproj.add_osdeps_overrides 'opencv', :packages => ['external/opencv', 'external/test']
        #
        # The :force option allows to force the usage of the source package(s),
        # regardless of the availability of the osdeps package.
        def add_osdeps_overrides(osdeps_name, options)
            options = Kernel.validate_options options, :package => nil, :packages => [], :force => false
            if pkg = options.delete(:package)
                options[:packages] << pkg
            end
            @osdeps_overrides[osdeps_name.to_s] = options
        end

        # Remove any OSDeps override that has previously been added with
        # #add_osdeps_overrides
        def remove_osdeps_overrides(osdep_name)
            @osdeps_overrides.delete(osdeps_name.to_s)
        end

        # Package selection can be done in three ways:
        #  * as a subdirectory in the layout
        #  * as a on-disk directory
        #  * as a package name
        #
        # This method converts the first two directories into the third one
        def expand_package_selection(selection)
            base_dir = Autoproj.root_dir

            # The expanded selection
            expanded_packages = Set.new
            # All the packages that are available on this installation
            all_layout_packages = self.all_selected_packages

            # A selection to packages map that represents the matches found
            matches = Hash.new { |h, k| h[k] = Set.new }

            # First, remove packages that are directly referenced by name or by
            # package set names
            selection.each do |sel|
                match_pkg_name = Regexp.new(Regexp.quote(sel))

                packages = all_layout_packages.
                    find_all { |pkg_name| pkg_name =~ match_pkg_name }.
                    to_set
                if !packages.empty?
                    matches[sel] = packages
                    expanded_packages |= packages
                end

                each_metapackage do |pkg|
                    if pkg.name =~ match_pkg_name
                        packages = resolve_package_set(pkg.name).to_set
                        packages = (packages & all_layout_packages)
                        matches[sel] |= packages
                        expanded_packages |= packages
                    end
                end
            end

            # Finally, check for package source directories
            all_packages = self.all_package_names
            selection.each do |sel|
                match_pkg_name = Regexp.new(Regexp.quote(sel))
                all_packages.each do |pkg_name|
                    pkg = Autobuild::Package[pkg_name]
                    if pkg_name =~ match_pkg_name || "#{sel}/" =~ Regexp.new("^#{Regexp.quote(pkg.srcdir)}/") || pkg.srcdir =~ Regexp.new("^#{Regexp.quote(sel)}")
                        # Check-out packages that are not in the manifest only
                        # if they are explicitely selected
                        if !all_layout_packages.include?(pkg.name)
                            if !File.directory?(pkg.srcdir) && pkg_name != sel && pkg.srcdir != sel
                                next
                            end
                        end

                        matches[sel] << pkg_name
                        expanded_packages << pkg_name
                    end
                end
            end

            # Remove packages that are explicitely excluded and/or ignored
            #
            # Raise an error if an explicit selection expands only to an
            # excluded package, and display a warning for ignored packages
            matches.each do |sel, expansion|
                next if expansion.empty?
                excluded, other = expansion.partition { |pkg_name| excluded?(pkg_name) }
                ignored,  ok    = other.partition { |pkg_name| ignored?(pkg_name) }

                if ok.empty? && ignored.empty?
                    packages = excluded.map do |pkg_name|
                        [pkg_name, Autoproj.manifest.exclusion_reason(pkg_name)]
                    end
                    raise ConfigError.new, "selection #{sel} expands to #{packages.map(&:first).join(", ")} which are excluded from the build:\n  #{packages.map { |name, reason| "#{name}: #{reason}" }.join("\n  ")}"
                elsif !ignored.empty?
                    ignored.each do |pkg_name|
                        Autoproj.warn "#{pkg_name} was selected for #{sel}, but is explicitely ignored in the manifest"
                    end
                end
            end

            expanded_packages.delete_if do |pkg_name|
                excluded?(pkg_name) || ignored?(pkg_name)
            end
            return expanded_packages.to_set, (selection - matches.keys)
        end

        attr_reader :moved_packages

        # Moves the given package name from its current subdirectory to the
        # provided one.
        #
        # For instance, for a package called drivers/xsens_imu
        #
        #   move("drivers/xsens_imu", "data_acquisition")
        #
        # will move the package into data_acquisition/xsens_imu
        def move_package(package_name, new_dir)
            moved_packages[package_name] = File.join(new_dir, File.basename(package_name))
        end
    end

    class << self
        # The singleton manifest object on which the current run works
        attr_accessor :manifest

        # The operating system package definitions
        attr_accessor :osdeps
    end

    def self.load_osdeps_from_package_sets
        manifest.each_osdeps_file do |source, file|
            osdeps.merge(source.load_osdeps(file))
        end
        osdeps
    end

    # Access to the information contained in a package's manifest.xml file
    #
    # Use PackageManifest.load to create
    class PackageManifest
        # Load a manifest.xml file and returns the corresponding
        # PackageManifest object
        def self.load(package, file)
            doc =
                begin REXML::Document.new(File.read(file))
                rescue REXML::ParseException => e
                    raise Autobuild::PackageException.new(package.name, 'prepare'), "invalid #{file}: #{e.message}"
                end

            PackageManifest.new(package, doc)
        end

        # The Autobuild::Package instance this manifest applies on
        attr_reader :package
        # The raw XML data as a Nokogiri document
        attr_reader :xml

        # The list of tags defined for this package
        #
        # Tags are defined as multiple <tags></tags> blocks, each of which can
        # contain multiple comma-separated tags 
        def tags
            result = []
            xml.elements.each('package/tags') do |node|
                result.concat((node.text || "").strip.split(','))
            end
            result
        end

        def documentation
            xml.elements.each('package/description') do |node|
                doc = (node.text || "").strip
                if !doc.empty?
                    return doc
                end
            end
            return short_documentation
        end

        def short_documentation
            xml.elements.each('package/description') do |node|
                doc = node.attributes['brief']
                if doc
                    doc = doc.to_s.strip
                end
                if doc && !doc.empty?
                    return doc.to_s
                end
            end
            "no documentation available for #{package.name} in its manifest.xml file"
        end

        def initialize(package, doc)
            @package = package
            @xml = doc
        end

        def each_dependency(&block)
            if block_given?
                each_os_dependency(&block)
                each_package_dependency(&block)
            else
                enum_for(:each_dependency, &block)
            end
        end

        def each_os_dependency
            if block_given?
                xml.elements.each('package/rosdep') do |node|
                    yield(node.attributes['name'], false)
                end
                package.os_packages.each do |name|
                    yield(name, false)
                end
            else
                enum_for :each_os_dependency
            end
        end

        def each_package_dependency
            if block_given?
                depend_nodes = xml.elements.to_a('package/depend') +
                    xml.elements.to_a('package/depend_optional')

                depend_nodes.each do |node|
                    dependency = node.attributes['package']
                    optional = (node.attributes['optional'].to_s == '1' || node.name == "depend_optional")

                    if dependency
                        yield(dependency, optional)
                    else
                        raise ConfigError.new, "manifest of #{package.name} has a <depend> tag without a 'package' attribute"
                    end
                end
            else
                enum_for :each_package_dependency
            end
        end

        # Enumerates the name and email of each author. If no email is present,
        # yields (name, nil)
        def each_author
            if !block_given?
                return enum_for(:each_author)
            end

            xml.elements.each('package/author') do |author|
                (author.text || "").strip.split(',').each do |str|
                    name, email = str.split('/').map(&:strip)
                    email = nil if email && email.empty?
                    yield(name, email)
                end
            end
        end

        # If +name+ points to a text element in the XML document, returns the
        # content of that element. If no element matches +name+, or if the
        # content is empty, returns nil
        def text_node(name)
            xml.elements.each(name) do |str|
                str = (str.text || "").strip
                if !str.empty?
                    return str
                end
            end
            nil
        end

        # The package associated URL, usually meant to direct to a website
        #
        # Returns nil if there is none
        def url
            return text_node('package/url')
        end

        # The package license name
        #
        # Returns nil if there is none
        def license
            return text_node('package/license')
        end

        # The package version number
        #
        # Returns 0 if none is declared
        def version
            return text_node("version")
        end
    end
    def self.add_osdeps_overrides(*args, &block)
        manifest.add_osdeps_overrides(*args, &block)
    end
end