require_relative "packager" require "chef/cookbook/chefignore" require "chef/util/path_helper" module Berkshelf class Berksfile class << self # Instantiate a Berksfile from the given options. This method is used # heavily by the CLI to reduce duplication. # # @param (see Berksfile#initialize) def from_options(options = {}) options[:berksfile] ||= File.join(Dir.pwd, Berkshelf::DEFAULT_FILENAME) symbolized = Hash[options.map { |k, v| [k.to_sym, v] }] from_file(options[:berksfile], symbolized.select { |k,| %i{except only delete}.include? k }) end # @param [#to_s] file # a path on disk to a Berksfile to instantiate from # # @return [Berksfile] def from_file(file, options = {}) raise BerksfileNotFound.new(file) unless File.exist?(file) begin new(file, options).evaluate_file(file) rescue => ex raise BerksfileReadError.new(ex) end end end DEFAULT_API_URL = "https://supermarket.chef.io".freeze # Don't vendor VCS files. # Reference GNU tar --exclude-vcs: https://www.gnu.org/software/tar/manual/html_section/tar_49.html EXCLUDED_VCS_FILES_WHEN_VENDORING = [".arch-ids", "{arch}", ".bzr", ".bzrignore", ".bzrtags", "CVS", ".cvsignore", "_darcs", ".git", ".hg", ".hgignore", ".hgrags", "RCS", "SCCS", ".svn", "**/.git", "**/.svn"].freeze include Mixin::Logging include Cleanroom extend Forwardable # @return [String] # The path on disk to the file representing this instance of Berksfile attr_reader :filepath # @return [Symbol] # The solver engine required by this instance of Berksfile attr_reader :required_solver # @return [Symbol] # The solver engine preferred by this instance of Berksfile attr_reader :preferred_solver # Create a new Berksfile object. # # @param [String] path # path on disk to the file containing the contents of this Berksfile # # @option options [Symbol, Array] :except # Group(s) to exclude which will cause any dependencies marked as a member of the # group to not be installed # @option options [Symbol, Array] :only # Group(s) to include which will cause any dependencies marked as a member of the # group to be installed and all others to be ignored def initialize(path, options = {}) @filepath = File.expand_path(path) @dependencies = {} @sources = {} @delete = options[:delete] # defaults for what solvers to use @required_solver = nil @preferred_solver = :gecode if options[:except] && options[:only] raise ArgumentError, "Cannot specify both :except and :only!" elsif options[:except] except = Array(options[:except]).collect(&:to_sym) @filter = ->(dependency) { (except & dependency.groups).empty? } elsif options[:only] only = Array(options[:only]).collect(&:to_sym) @filter = ->(dependency) { !(only & dependency.groups).empty? } else @filter = ->(dependency) { true } end end # Activate a Berkshelf extension at runtime. # # @example Activate the Mercurial extension # extension 'hg' # # @raise [LoadError] # if the extension cannot be loaded # # @param [String] name # the name of the extension to activate # # @return [true] def extension(name) require "berkshelf/#{name}" true rescue LoadError raise LoadError, "Could not load an extension by the name `#{name}'. " \ "Please make sure it is installed." end expose :extension # Add a cookbook dependency to the Berksfile to be retrieved and have its dependencies recursively retrieved # and resolved. # # @example a cookbook dependency that will be retrieved from one of the default locations # cookbook 'artifact' # # @example a cookbook dependency that will be retrieved from a path on disk # cookbook 'artifact', path: '/Users/reset/code/artifact' # # @example a cookbook dependency that will be retrieved from a Git server # cookbook 'artifact', git: 'https://github.com/chef/artifact-cookbook.git' # # @overload cookbook(name, version_constraint, options = {}) # @param [#to_s] name # @param [#to_s] version_constraint # # @option options [Symbol, Array] :group # the group or groups that the cookbook belongs to # @option options [String] :path # a filepath to the cookbook on your local disk # @option options [String] :git # the Git URL to clone # # @see PathLocation # @see GitLocation # @overload cookbook(name, options = {}) # @param [#to_s] name # # @option options [Symbol, Array] :group # the group or groups that the cookbook belongs to # @option options [String] :path # a filepath to the cookbook on your local disk # @option options [String] :git # the Git URL to clone # # @see PathLocation # @see GitLocation def cookbook(*args) options = args.last.is_a?(Hash) ? args.pop : {} name, constraint = args options[:path] &&= File.expand_path(options[:path], File.dirname(filepath)) options[:group] = Array(options[:group]) if @active_group options[:group] += @active_group end add_dependency(name, constraint, **options) end expose :cookbook def group(*args) @active_group = args yield @active_group = nil end expose :group # Use a Cookbook metadata file to determine additional cookbook dependencies to retrieve. All # dependencies found in the metadata will use the default locations set in the Berksfile (if any are set) # or the default locations defined by Berkshelf. # # @param [Hash] options # # @option options [String] :path # path to the metadata file def metadata(options = {}) path = options[:path] || File.dirname(filepath) loader = Chef::Cookbook::CookbookVersionLoader.new(path) loader.load! cookbook_version = loader.cookbook_version metadata = cookbook_version.metadata add_dependency(metadata.name, nil, path: path, metadata: true) end expose :metadata # Add a Berkshelf API source to use when building the index of known cookbooks. The indexes will be # searched in the order they are added. If a cookbook is found in the first source then a cookbook # in a second source would not be used. # # @example # source "https://supermarket.chef.io" # source "https://berks-api.riotgames.com" # # @param [String] api_url # url for the api to add # # @param [Hash] options # extra source options # # @raise [InvalidSourceURI] # # @return [Array] def source(api_url, **options) source = Source.new(self, api_url, **options) @sources[source.uri.to_s] = source end expose :source # Configure a specific engine for the 'solve' gem to use when computing dependencies. You may # optionally specify how strong a requirement this is. If omitted, the default precedence is # :preferred. # # If :required is specified and cannot be loaded, Resolver#resolve will raise an ArgumentError. # If :preferred is specified and cannot be loaded, Resolver#resolve silently catch any errors and # use whatever default method the 'solve' gem provides (as of 2.0.1, solve defaults to :ruby). # # @example # solver :gecode # solver :gecode, :preferred # solver :gecode, :required # solver :ruby # solver :ruby, :preferred # solver :ruby, :required # # @param [Symbol] name # name of engine for solver gem to use for depsolving # # @param [Symbol] precedence # how strong a requirement using this solver is # valid values are :required, :preferred # # @raise [ArgumentError] def solver(name, precedence = :preferred) if name && precedence == :required @required_solver = name elsif name && precedence == :preferred @preferred_solver = name else raise ArgumentError, "Invalid solver precedence ':#{precedence}'" end end expose :solver # @return [Array] def sources if @sources.empty? raise NoAPISourcesDefined else @sources.values end end # @param [Dependency] dependency # the dependency to find the source for def source_for(name, version) sources.find { |source| source.cookbook(name, version) } end # Add a dependency of the given name and constraint to the array of dependencies. # # @param [String] name # the name of the dependency to add # @param [String, Semverse::Constraint] constraint # the constraint to lock the dependency to # # @option options [Symbol, Array] :group # the group or groups that the cookbook belongs to # @option options [String] :path # a filepath to the cookbook on your local disk # @option options [String] :git # the Git URL to clone # # @raise [DuplicateDependencyDefined] if a dependency is added whose name conflicts # with a dependency who has already been added. # # @return [Array] def dependencies @dependencies.values.sort.select(&@filter) end # # Behaves the same as {Berksfile#dependencies}, but this method returns an # array of CachedCookbook objects instead of dependency objects. This method # relies on the {Berksfile#retrieve_locked} method to load the proper # cached cookbook from the Berksfile + lockfile combination. # # @see [Berksfile#dependencies] # for a description of the +options+ hash # @see [Berksfile#retrieve_locked] # for a list of possible exceptions that might be raised and why # # @return [Array] # def cookbooks dependencies.map { |dependency| retrieve_locked(dependency) } end # Find a dependency defined in this berksfile by name. # # @param [String] name # the name of the cookbook dependency to search for # @return [Dependency, nil] # the cookbook dependency, or nil if one does not exist def find(name) @dependencies[name] end # @return [Hash] # a hash containing group names as keys and an array of Dependencies # that are a member of that group as values # # Example: # { # nautilus: [ # # 1.0.0)>, # # 1.2.4)> # ], # skarner: [ # # 1.0.0)> # ] # } def groups {}.tap do |groups| dependencies.each do |dependency| dependency.groups.each do |group| groups[group] ||= [] groups[group] << dependency end end end end # @param [String] name # name of the dependency to return # # @return [Dependency] def [](name) @dependencies[name] end alias_method :get_dependency, :[] # Install the dependencies listed in the Berksfile, respecting the locked # versions in the Berksfile.lock. # # 1. Check that a lockfile exists. If a lockfile does not exist, all # dependencies are considered to be "unlocked". If a lockfile is specified, a # definition is created via the following algorithm: # # - For each source, see if there exists a locked version that still # satisfies the version constraint in the Berksfile. If # there exists such a source, remove it from the list of unlocked # sources. If not, then either a version constraint has changed, # or a new source has been added to the Berksfile. In the event that # a locked_source exists, but it no longer satisfies the constraint, # this method will raise a {OutdatedCookbookSource}, and # inform the user to run berks update COOKBOOK to remedy the issue. # - Remove any locked sources that no longer exist in the Berksfile # (i.e. a cookbook source was removed from the Berksfile). # # 2. Resolve the collection of locked and unlocked dependencies. # # 3. Write out a new lockfile. # # @raise [OutdatedDependency] # if the lockfile constraints do not satisfy the Berksfile constraints # # @return [Array] def install Installer.new(self).run end # Update the given set of dependencies (or all if no names are given). # # @option options [String, Array] :cookbooks # Names of the cookbooks to retrieve dependencies for def update(*names) validate_lockfile_present! validate_cookbook_names!(names) Berkshelf.log.info "Updating cookbooks" # Calculate the list of cookbooks to unlock if names.empty? Berkshelf.log.debug " Unlocking all the things!" lockfile.unlock_all else names.each do |name| Berkshelf.log.debug " Unlocking #{name}" lockfile.unlock(name, true) end end # NOTE: We intentionally do NOT pass options to the installer install end # Retrieve information about a given cookbook that is installed by this Berksfile. # Unlike {#find}, which returns a dependency, this method returns the corresponding # CachedCookbook for the given name. # # @raise [LockfileNotFound] # if there is no lockfile containing that cookbook # @raise [CookbookNotFound] # if there is a lockfile with a cookbook, but the cookbook is not downloaded # # @param [Dependency] name # the name of the cookbook to find # # @return [CachedCookbook] # the CachedCookbook that corresponds to the given name parameter def retrieve_locked(dependency) lockfile.retrieve(dependency) end # The cached cookbooks installed by this Berksfile. # # @raise [LockfileNotFound] # if there is no lockfile # @raise [CookbookNotFound] # if a listed source could not be found # # @return [Hash] # the list of dependencies as keys and the cached cookbook as the value def list validate_lockfile_present! validate_lockfile_trusted! validate_dependencies_installed! lockfile.graph.locks.values end # List of all the cookbooks which have a newer version found at a source # that satisfies the constraints of your dependencies. # # @param [Boolean] include_non_satisfying # include cookbooks that would not satisfy the given constraints in the # +Berksfile+. Defaults to false. # # @return [Hash] # a hash of cached cookbooks and their latest version grouped by their # remote API source. The hash will be empty if there are no newer # cookbooks for any of your dependencies (that still satisfy the given) # constraints in the +Berksfile+. # # @example # berksfile.outdated #=> { # "nginx" => { # "local" => #, # "remote" => { # # #=> # # } # } # } def outdated(*names, include_non_satisfying: false) validate_lockfile_present! validate_lockfile_trusted! validate_dependencies_installed! validate_cookbook_names!(names) lockfile.graph.locks.inject({}) do |hash, (name, dependency)| sources.each do |source| cookbooks = source.versions(name) latest = cookbooks.select do |cookbook| (include_non_satisfying || dependency.version_constraint.satisfies?(cookbook.version)) && Semverse::Version.coerce(cookbook.version) > dependency.locked_version end.max_by(&:version) unless latest.nil? hash[name] ||= { "local" => dependency.locked_version, "remote" => { source => Semverse::Version.coerce(latest.version), }, } end end hash end end # Upload the cookbooks installed by this Berksfile # # @overload upload(names = []) # @param [Array] names # the list of cookbooks (by name) to upload to the remote Chef Server # # # @overload upload(names = [], options = {}) # @param [Array] names # the list of cookbooks (by name) to upload to the remote Chef Server # @param [Hash] options # the list of options to pass to the uploader # # @option options [Boolean] :force (false) # upload the cookbooks even if the version already exists and is frozen # on the remote Chef Server # @option options [Boolean] :freeze (true) # freeze the uploaded cookbooks on the remote Chef Server so that it # cannot be overwritten on future uploads # @option options [Hash] :ssl_verify (true) # use SSL verification while connecting to the remote Chef Server # @option options [Boolean] :halt_on_frozen (false) # raise an exception ({FrozenCookbook}) if one of the cookbooks already # exists on the remote Chef Server and is frozen # @option options [String] :server_url # the URL (endpoint) to the remote Chef Server # @option options [String] :client_name # the client name for the remote Chef Server # @option options [String] :client_key # the client key (pem) for the remote Chef Server # # # @example Upload all cookbooks # berksfile.upload # # @example Upload the 'apache2' and 'mysql' cookbooks # berksfile.upload('apache2', 'mysql') # # @example Upload and freeze all cookbooks # berksfile.upload(freeze: true) # # @example Upload and freeze the `chef-sugar` cookbook # berksfile.upload('chef-sugar', freeze: true) # # # @raise [UploadFailure] # if you are uploading cookbooks with an invalid or not-specified client key # @raise [DependencyNotFound] # if one of the given cookbooks is not a dependency defined in the Berksfile # @raise [FrozenCookbook] # if the cookbook being uploaded is a {metadata} cookbook and is already # frozen on the remote Chef Server; indirect dependencies or non-metadata # dependencies are just skipped # # @return [Array] # the list of cookbooks that were uploaded to the Chef Server def upload(*args) validate_lockfile_present! validate_lockfile_trusted! validate_dependencies_installed! Uploader.new(self, *args).run end # Package the given cookbook for distribution outside of berkshelf. If the # name attribute is not given, all cookbooks in the Berksfile will be # packaged. # # @param [String] path # the path where the tarball will be created # # @raise [PackageError] # # @return [String] # the path to the package def package(path) packager = Packager.new(path) packager.validate! outdir = Dir.mktmpdir do |temp_dir| Berkshelf.ui.mute { vendor(File.join(temp_dir, "cookbooks")) } packager.run(temp_dir) end Berkshelf.formatter.package(outdir) outdir end # backcompat with ridley lookup of chefignore def find_chefignore(path) filename = "chefignore" Pathname.new(path).ascend do |dir| next unless dir.directory? [ dir.join(filename), dir.join("cookbooks", filename), dir.join(".chef", filename), ].each do |possible| return possible.expand_path.to_s if possible.exist? end end nil end # Install the Berksfile or Berksfile.lock and then sync the cached cookbooks # into directories within the given destination matching their name. # # @param [String] destination # filepath to vendor cookbooks to # # @return [String, nil] # the expanded path cookbooks were vendored to or nil if nothing was vendored def vendor(destination) Dir.mktmpdir("vendor") do |scratch| cached_cookbooks = install return nil if cached_cookbooks.empty? cached_cookbooks.each do |cookbook| Berkshelf.formatter.vendor(cookbook, destination) cookbook_destination = File.join(scratch, cookbook.cookbook_name) FileUtils.mkdir_p(cookbook_destination) # Dir.glob does not support backslash as a File separator src = cookbook.path.to_s.tr("\\", "/") files = FileSyncer.glob(File.join(src, "**/*")) # strip directories files.reject! { |file_path| File.directory?(file_path) } # convert to relative Pathname objects for chefignore files.map! { |file_path| Chef::Util::PathHelper.relative_path_from(cookbook.path.to_s, file_path) } chefignore = Chef::Cookbook::Chefignore.new(find_chefignore(cookbook.path.to_s) || cookbook.path.to_s) # apply chefignore files.reject! { |file_path| chefignore.ignored?(file_path) } # convert Pathname objects back to strings files.map!(&:to_s) # copy each file to destination files.each do |rpath| FileUtils.mkdir_p( File.join(cookbook_destination, File.dirname(rpath)) ) FileUtils.cp( File.join(cookbook.path.to_s, rpath), File.join(cookbook_destination, rpath) ) end cookbook.compile_metadata(cookbook_destination) end # Don't vendor the raw metadata (metadata.rb). The raw metadata is # unecessary for the client, and this is required until compiled metadata # (metadata.json) takes precedence over raw metadata in the Chef-Client. # # We can change back to including the raw metadata in the future after # this has been fixed or just remove these comments. There is no # circumstance that I can currently think of where raw metadata should # ever be read by the client. # # - Jamie # # See the following tickets for more information: # # * https://tickets.opscode.com/browse/CHEF-4811 # * https://tickets.opscode.com/browse/CHEF-4810 FileSyncer.sync(scratch, destination, exclude: EXCLUDED_VCS_FILES_WHEN_VENDORING, delete: @delete) end destination end # Perform a validation with `Validator#validate` on each cached cookbook associated # with the Lockfile of this Berksfile. # # This function will return true or raise the first errors encountered. def verify validate_lockfile_present! validate_lockfile_trusted! Berkshelf.formatter.msg "Verifying (#{lockfile.cached.length}) cookbook(s)..." Validator.validate(lockfile.cached) true end # Visualize the current Berksfile as a "graph" using DOT. # # @param [String] outfile # the name/path to outfile the file # # @return [String] path # the path where the image was written def viz(outfile = nil, format = "png") outfile = File.join(Dir.pwd, outfile || "graph.png") validate_lockfile_present! validate_lockfile_trusted! vizualiser = Visualizer.from_lockfile(lockfile) case format when "dot" vizualiser.to_dot_file(outfile) when "png" vizualiser.to_png(outfile) else raise ConfigurationError, "Vizualiser format #{format} not recognised." end end # Get the lockfile corresponding to this Berksfile. This is necessary because # the user can specify a different path to the Berksfile. So assuming the lockfile # is named "Berksfile.lock" is a poor assumption. # # @return [Lockfile] # the lockfile corresponding to this berksfile, or a new Lockfile if one does # not exist def lockfile @lockfile ||= Lockfile.from_berksfile(self) end private # Ensure the lockfile is present on disk. # # @raise [LockfileNotFound] # if the lockfile does not exist on disk # # @return [true] def validate_lockfile_present! raise LockfileNotFound unless lockfile.present? true end # Ensure that all dependencies defined in the Berksfile exist in this # lockfile. # # @raise [LockfileOutOfSync] # if there are dependencies specified in the Berksfile which do not # exist (or are not satisifed by) the lockfile # # @return [true] def validate_lockfile_trusted! raise LockfileOutOfSync unless lockfile.trusted? true end # Ensure that all dependencies in the lockfile are installed on this # system. You should validate that the lockfile can be trusted before # using this method. # # @raise [DependencyNotInstalled] # if the dependency in the lockfile is not in the Berkshelf shelf on # this system # # @return [true] def validate_dependencies_installed! lockfile.graph.locks.each do |_, dependency| unless dependency.installed? raise DependencyNotInstalled.new(dependency) end end true end # Determine if any cookbooks were specified that aren't in our shelf. # # @param [Array] names # a list of cookbook names # # @raise [DependencyNotFound] # if a cookbook name is given that does not exist def validate_cookbook_names!(names) missing = names - lockfile.graph.locks.keys unless missing.empty? raise DependencyNotFound.new(missing) end end end end