lib/berkshelf/lockfile.rb in berkshelf-3.0.0.beta7 vs lib/berkshelf/lockfile.rb in berkshelf-3.0.0.beta8

- old
+ new

@@ -19,14 +19,14 @@ filepath = File.join(File.dirname(File.expand_path(berksfile.filepath)), Lockfile::DEFAULT_FILENAME) new(berksfile: berksfile, filepath: filepath) end end - DEFAULT_FILENAME = 'Berksfile.lock' + DEFAULT_FILENAME = 'Berksfile.lock'.freeze - DEPENDENCIES = 'DEPENDENCIES' - GRAPH = 'GRAPH' + DEPENDENCIES = 'DEPENDENCIES'.freeze + GRAPH = 'GRAPH'.freeze include Berkshelf::Mixin::Logging # @return [Pathname] # the path to this Lockfile @@ -77,31 +77,97 @@ # Determine if we can "trust" this lockfile. A lockfile is trustworthy if: # # 1. All dependencies defined in the Berksfile are present in this # lockfile - # 2. Each dependency's constraint in the Berksfile is still satisifed by + # 2. Each dependency's transitive dependencies are contained and locked + # in the lockfile + # 3. Each dependency's constraint in the Berksfile is still satisifed by # the currently locked version # # This method does _not_ account for leaky dependencies (i.e. dependencies # defined in the lockfile that are no longer present in the Berksfile); this # edge case is handed by the installer. # # @return [Boolean] # true if this lockfile is trusted, false otherwise def trusted? - berksfile.dependencies.all? do |dependency| - locked = find(dependency) - graphed = graph.find(dependency) - constraint = dependency.version_constraint + Berkshelf.log.info 'Checking if lockfile is trusted' - locked && graphed && - dependency.location == locked.location && - constraint.satisfies?(graphed.version) + checked = {} + + berksfile.dependencies.each do |dependency| + Berkshelf.log.debug "Checking #{dependency}" + + checked[dependency.name] = true + + locked = find(dependency) + if locked.nil? + Berkshelf.log.debug " Not in lockfile - cannot be trusted!" + return false + end + + graphed = graph.find(dependency) + if graphed.nil? + Berkshelf.log.debug " Not in graph - cannot be trusted!" + return false + end + + if cookbook = dependency.cached_cookbook + Berkshelf.log.debug " Detected there is a cached cookbook" + + unless (cookbook.dependencies.keys - graphed.dependencies.keys).empty? + Berkshelf.log.debug " Cached cookbook has different dependencies - cannot be trusted!" + return false + end + end + + unless dependency.location == locked.location + Berkshelf.log.debug " Different location - cannot be trusted!" + Berkshelf.log.debug " Dependency location: #{dependency.location.inspect}" + Berkshelf.log.debug " Locked location: #{locked.location.inspect}" + return false + end + + unless dependency.version_constraint.satisfies?(graphed.version) + Berkshelf.log.debug " Version constraint is not satisified - cannot be trusted!" + return false + end + + unless satisfies_transitive?(graphed, checked) + Berkshelf.log.debug " Transitive dependencies not satisfies - cannot be trusted!" + return false + end end + + true end + # Recursive helper method for checking if transitive dependencies (i.e. + # those dependencies defined in the metadata) are satisfied. This method is + # used in calculating the trustworthiness of a lockfile. + # + # @param [GraphItem] graph_item + # the graph item to check transitive dependencies for + # @param [Hash] checked + # the list of already checked dependencies + # + # @return [Boolean] + def satisfies_transitive?(graph_item, checked) + graph_item.dependencies.all? do |name, constraint| + return true if checked[name] + + checked[name] = true + + graphed = graph.find(name) + return false if graphed.nil? + + Solve::Constraint.new(constraint).satisfies?(graphed.version) && + satisfies_transitive?(graphed, checked) + end + end + # Resolve this Berksfile and apply the locks found in the generated # +Berksfile.lock+ to the target Chef environment # # @param [String] name # the name of the environment to apply the locks to @@ -177,11 +243,11 @@ # Retrieve information about a given cookbook that is in this lockfile. # # @raise [DependencyNotFound] # if this lockfile does not have the given dependency # @raise [CookbookNotFound] - # if this lockfile has the dependency, but the cookbook is not downloaded + # if this lockfile has the dependency, but the cookbook is not installed # # @param [String, Dependency] dependency # the dependency or name of the dependency to find # # @return [CachedCookbook] @@ -191,13 +257,14 @@ if locked.nil? raise DependencyNotFound.new(Dependency.name(dependency)) end - unless locked.downloaded? - raise CookbookNotFound, "Could not find cookbook '#{locked.to_s}'. " \ - "Run `berks install` to download and install the missing cookbook." + unless locked.installed? + name = locked.name + version = locked.locked_version || locked.version_constraint + raise CookbookNotFound.new(name, version, 'in the cookbook store') end locked.cached_cookbook end @@ -219,24 +286,94 @@ # # This method first removes the dependency from the list of top-level # dependencies. Then it uses a recursive algorithm to safely remove any # other dependencies from the graph that are no longer needed. # - # @raise [Berkshelf::CookbookNotFound] + # @raise [CookbookNotFound] # if the provided dependency does not exist # # @param [String] dependency # the name of the cookbook to remove def unlock(dependency) - unless dependency?(dependency) - raise Berkshelf::CookbookNotFound, "'#{dependency}' does not exist in this lockfile!" - end - @dependencies.delete(Dependency.name(dependency)) graph.remove(dependency) end + # Iterate over each top-level dependency defined in the lockfile and + # check if that dependency is still defined in the Berksfile. + # + # If the dependency is no longer present in the Berksfile, it is "safely" + # removed using {Lockfile#unlock} and {Lockfile#remove}. This prevents + # the lockfile from "leaking" dependencies when they have been removed + # from the Berksfile, but still remained locked in the lockfile. + # + # If the dependency exists, a constraint comparison is conducted to verify + # that the locked dependency still satisifes the original constraint. This + # handles the edge case where a user has updated or removed a constraint + # on a dependency that already existed in the lockfile. + # + # @raise [OutdatedDependency] + # if the constraint exists, but is no longer satisifed by the existing + # locked version + # + # @return [Array<Dependency>] + def reduce! + # Store a list of cookbooks to ungraph + to_ungraph = {} + to_ignore = {} + + # Unlock any locked dependencies that are no longer in the Berksfile + dependencies.each do |dependency| + unless berksfile.has_dependency?(dependency.name) + unlock(dependency) + + # Keep a record. We know longer trust these dependencies, but simply + # unlocking them does not guarantee their removal from the graph. + # Instead, we keep a record of the dependency to unlock it later (in + # case it is actually removable because it's parent requirer is also + # being removed in this reduction). It's a form of science. Don't + # question it too much. + to_ungraph[dependency.name] = true + to_ignore[dependency.name] = true + end + end + + # Remove any transitive dependencies + berksfile.dependencies.each do |dependency| + graphed = graph.find(dependency) + next if graphed.nil? + + unless dependency.version_constraint.satisfies?(graphed.version) + raise OutdatedDependency.new(graphed, dependency) + end + + if cookbook = dependency.cached_cookbook + graphed.dependencies.each do |name, constraint| + # Unless the cookbook still depends on this key, we want to queue it + # for unlocking. This is the magic that prevents transitive + # dependency leaking. + unless cookbook.dependencies.has_key?(name) + to_ungraph[name] = true + + # We also want to ignore the top-level dependency. We can no + # longer trust the graph that we have been given for that + # dependency and therefore need to reduce it. + to_ignore[dependency.name] = true + end + end + end + end + + # Now remove all the unlockable items + ignore = to_ungraph.merge(to_ignore).keys + + to_ungraph.each do |name, _| + graph.remove(name, ignore: ignore) + end + end + + # Write the contents of the current statue of the lockfile to disk. This # method uses an atomic file write. A temporary file is created, written, # and then copied over the existing one. This ensures any partial updates # or failures do no affect the lockfile. The temporary file is ensured # deletion. @@ -246,19 +383,12 @@ def save return false if dependencies.empty? tempfile = Tempfile.new(['Berksfile', '.lock']) - tempfile.write(DEPENDENCIES) - tempfile.write("\n") - dependencies.sort.each do |dependency| - tempfile.write(dependency.to_lock) - end + tempfile.write(to_lock) - tempfile.write("\n") - tempfile.write(graph.to_lock) - tempfile.rewind tempfile.close # Move the lockfile into place FileUtils.cp(tempfile.path, filepath) @@ -267,10 +397,21 @@ ensure tempfile.unlink if tempfile end # @private + def to_lock + out = "#{DEPENDENCIES}\n" + dependencies.sort.each do |dependency| + out << dependency.to_lock + end + out << "\n" + out << graph.to_lock + out + end + + # @private def to_s "#<Berkshelf::Lockfile #{Pathname.new(filepath).basename}>" end # @private @@ -278,315 +419,336 @@ "#<Berkshelf::Lockfile #{Pathname.new(filepath).basename}, dependencies: #{dependencies.inspect}>" end private - # The class responsible for parsing the lockfile and turning it into a - # useful data structure. - class LockfileParser - NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?' - DEPENDENCY_PATTERN = /^ {2}#{NAME_VERSION}$/ - DEPENDENCIES_PATTERN = /^ {4}#{NAME_VERSION}$/ - OPTION_PATTERN = /^ {4}(.+)\: (.+)/ + # The class responsible for parsing the lockfile and turning it into a + # useful data structure. + class LockfileParser + NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'.freeze + DEPENDENCY_PATTERN = /^ {2}#{NAME_VERSION}$/.freeze + DEPENDENCIES_PATTERN = /^ {4}#{NAME_VERSION}$/.freeze + OPTION_PATTERN = /^ {4}(.+)\: (.+)/.freeze - # Create a new lockfile parser. - # - # @param [Lockfile] - def initialize(lockfile) - @lockfile = lockfile - @berksfile = lockfile.berksfile - end + # Create a new lockfile parser. + # + # @param [Lockfile] + def initialize(lockfile) + @lockfile = lockfile + @berksfile = lockfile.berksfile + end - # Parse the lockfile contents, adding the correct things to the lockfile. - # - # @return [true] - def run - @parsed_dependencies = {} + # Parse the lockfile contents, adding the correct things to the lockfile. + # + # @return [true] + def run + @parsed_dependencies = {} - contents = File.read(@lockfile.filepath) + contents = File.read(@lockfile.filepath) - if contents.strip.empty? - Berkshelf.formatter.warn "Your lockfile at '#{@lockfile.filepath}' " \ - "is empty. I am going to parse it anyway, but there is a chance " \ - "that a larger problem is at play. If you manually edited your " \ - "lockfile, you may have corrupted it." - end + if contents.strip.empty? + Berkshelf.formatter.warn "Your lockfile at '#{@lockfile.filepath}' " \ + "is empty. I am going to parse it anyway, but there is a chance " \ + "that a larger problem is at play. If you manually edited your " \ + "lockfile, you may have corrupted it." + end - if contents.strip[0] == '{' - Berkshelf.formatter.warn "It looks like you are using an older " \ - "version of the lockfile. Attempting to convert..." + if contents.strip[0] == '{' + Berkshelf.formatter.warn "It looks like you are using an older " \ + "version of the lockfile. Attempting to convert..." - dependencies = "#{Lockfile::DEPENDENCIES}\n" - graph = "#{Lockfile::GRAPH}\n" + dependencies = "#{Lockfile::DEPENDENCIES}\n" + graph = "#{Lockfile::GRAPH}\n" - begin - hash = JSON.parse(contents) - rescue JSON::ParserError - Berkshelf.formatter.warn "Could not convert lockfile! This is a " \ - "problem. You see, previous versions of the lockfile were " \ - "actually a lie. It lied to you about your version locks, and we " \ - "are really sorry about that.\n\n" \ - "Here's the good news - we fixed it!\n\n" \ - "Here's the bad news - you probably should not trust your old " \ - "lockfile. You should manually delete your old lockfile and " \ - "re-run the installer." - end + begin + hash = JSON.parse(contents) + rescue JSON::ParserError + Berkshelf.formatter.warn "Could not convert lockfile! This is a " \ + "problem. You see, previous versions of the lockfile were " \ + "actually a lie. It lied to you about your version locks, and we " \ + "are really sorry about that.\n\n" \ + "Here's the good news - we fixed it!\n\n" \ + "Here's the bad news - you probably should not trust your old " \ + "lockfile. You should manually delete your old lockfile and " \ + "re-run the installer." + end - hash['dependencies'] && hash['dependencies'].sort .each do |name, info| - dependencies << " #{name} (>= 0.0.0)\n" - info.each do |key, value| - unless key == 'locked_version' - dependencies << " #{key}: #{value}\n" + hash['dependencies'] && hash['dependencies'].sort .each do |name, info| + dependencies << " #{name} (>= 0.0.0)\n" + info.each do |key, value| + unless key == 'locked_version' + dependencies << " #{key}: #{value}\n" + end end + + graph << " #{name} (#{info['locked_version']})\n" end - graph << " #{name} (#{info['locked_version']})\n" + contents = "#{dependencies}\n#{graph}" end - contents = "#{dependencies}\n#{graph}" - end + contents.split(/(?:\r?\n)+/).each do |line| + if line == Lockfile::DEPENDENCIES + @state = :dependency + elsif line == Lockfile::GRAPH + @state = :graph + else + send("parse_#{@state}", line) + end + end - contents.split(/(?:\r?\n)+/).each do |line| - if line == Lockfile::DEPENDENCIES - @state = :dependency - elsif line == Lockfile::GRAPH - @state = :graph - else - send("parse_#{@state}", line) + @parsed_dependencies.each do |name, options| + dependency = Dependency.new(@berksfile, name, options) + @lockfile.add(dependency) end - end - @parsed_dependencies.each do |name, options| - dependency = Dependency.new(@berksfile, name, options) - @lockfile.add(dependency) + true end - true - end + private - private + # Parse a dependency line. + # + # @param [String] line + def parse_dependency(line) + if line =~ DEPENDENCY_PATTERN + name, version = $1, $2 - # Parse a dependency line. - # - # @param [String] line - def parse_dependency(line) - if line =~ DEPENDENCY_PATTERN - name, version = $1, $2 + @parsed_dependencies[name] ||= {} + @parsed_dependencies[name][:constraint] = version if version + @current_dependency = @parsed_dependencies[name] + elsif line =~ OPTION_PATTERN + key, value = $1, $2 + @current_dependency[key.to_sym] = value + end + end - @parsed_dependencies[name] ||= {} - @parsed_dependencies[name][:constraint] = version if version - @current_dependency = @parsed_dependencies[name] - elsif line =~ OPTION_PATTERN - key, value = $1, $2 - @current_dependency[key.to_sym] = value - end + # Parse a graph line. + # + # @param [String] line + def parse_graph(line) + if line =~ DEPENDENCY_PATTERN + name, version = $1, $2 + + @lockfile.graph.find(name) || @lockfile.graph.add(name, version) + @current_lock = name + elsif line =~ DEPENDENCIES_PATTERN + name, constraint = $1, $2 + @lockfile.graph.find(@current_lock).add_dependency(name, constraint) + end + end end - # Parse a graph line. - # - # @param [String] line - def parse_graph(line) - if line =~ DEPENDENCY_PATTERN - name, version = $1, $2 - - @lockfile.graph.find(name) || @lockfile.graph.add(name, version) - @current_lock = name - elsif line =~ DEPENDENCIES_PATTERN - name, constraint = $1, $2 - @lockfile.graph.find(@current_lock).add_dependency(name, constraint) + # The class representing an internal graph. + class Graph + # Create a new Lockfile graph. + # + # Some clarifying terminology: + # + # yum-epel (0.2.0) <- lock + # yum (~> 3.0) <- dependency + # + # @return [Graph] + def initialize(lockfile) + @lockfile = lockfile + @berksfile = lockfile.berksfile + @graph = {} end - end - end - # The class representing an internal graph. - class Graph - # Create a new Lockfile graph. - # - # Some clarifying terminology: - # - # yum-epel (0.2.0) <- lock - # yum (~> 3.0) <- dependency - # - # @return [Graph] - def initialize(lockfile) - @lockfile = lockfile - @berksfile = lockfile.berksfile - @graph = {} - end + # The list of locks for this graph. Dependencies are retrieved from the + # lockfile, then the Berksfile, and finally a new dependency object is + # created if none of those exist. + # + # @return [Hash<String, Dependency>] + # a key-value hash where the key is the name of the cookbook and the + # value is the locked dependency + def locks + @graph.sort.inject({}) do |hash, (name, item)| + dependency = @lockfile.find(name) || + @berksfile && @berksfile.find(name) || + Dependency.new(@berksfile, name) + dependency.locked_version = item.version - # The list of locks for this graph. Dependencies are retrieved from the - # lockfile, then the Berksfile, and finally a new dependency object is - # created if none of those exist. - # - # @return [Hash<String, Dependency>] - # a key-value hash where the key is the name of the cookbook and the - # value is the locked dependency - def locks - @graph.sort.inject({}) do |hash, (name, item)| - dependency = @lockfile.find(name) || - @berksfile && @berksfile.find(name) || - Dependency.new(@berksfile, name) - dependency.locked_version = item.version + hash[item.name] = dependency + hash + end + end - hash[item.name] = dependency - hash + # Find a given dependency in the graph. + # + # @param [Dependency, String] + # the name/dependency to find + # + # @return [GraphItem, nil] + # the item for the name + def find(dependency) + @graph[Dependency.name(dependency)] end - end - # Find a given dependency in the graph. - # - # @param [Dependency, String] - # the name/dependency to find - # - # @return [GraphItem, nil] - # the item for the name - def find(dependency) - @graph[Dependency.name(dependency)] - end + # Find if the given lock exists? + # + # @param [Dependency, String] + # the name/dependency to find + # + # @return [true, false] + def lock?(dependency) + !find(dependency).nil? + end + alias_method :has_lock?, :lock? - # Find if the given lock exists? - # - # @param [Dependency, String] - # the name/dependency to find - # - # @return [true, false] - def lock?(dependency) - !find(dependency).nil? - end - alias_method :has_lock?, :lock? + # Determine if this graph contains the given dependency. This method is + # used by the lockfile when adding or removing dependencies to see if a + # dependency can be safely removed. + # + # @param [Dependency, String] dependency + # the name/dependency to find + # + # @option options [String, Array<String>] :ignore + # the list of dependencies to ignore + def dependency?(dependency, options = {}) + name = Dependency.name(dependency) + ignore = Hash[*Array(options[:ignore]).map { |i| [i, true] }.flatten] - # Determine if this graph contains the given dependency. This method is - # used by the lockfile when adding or removing dependencies to see if a - # dependency can be safely removed. - # - # @param [Dependency, String] dependency - # the name/dependency to find - def dependency?(dependency) - @graph.values.any? do |item| - item.dependencies.key?(Dependency.name(dependency)) - end - end - alias_method :has_dependency?, :dependency? + @graph.values.each do |item| + next if ignore[item.name] - # Add each a new {GraphItem} to the graph. - # - # @param [#to_s] name - # the name of the cookbook - # @param [#to_s] version - # the version of the lock - # - # @return [GraphItem] - def add(name, version) - @graph[name.to_s] = GraphItem.new(name, version) - end + if item.dependencies.key?(name) + return true + end + end - # Recursively remove any dependencies from the graph unless they exist as - # top-level dependencies or nested dependencies. - # - # @param [Dependency, String] dependency - # the name/dependency to remove - def remove(dependency) - name = Dependency.name(dependency) + false + end + alias_method :has_dependency?, :dependency? - return if @lockfile.dependency?(name) || dependency?(name) + # Add each a new {GraphItem} to the graph. + # + # @param [#to_s] name + # the name of the cookbook + # @param [#to_s] version + # the version of the lock + # + # @return [GraphItem] + def add(name, version) + @graph[name.to_s] = GraphItem.new(name, version) + end - # Grab the nested dependencies for this particular entry so we can - # recurse and try to remove them from the graph. - locked = @graph[name] - nested_dependencies = locked && locked.dependencies.keys || [] + # Recursively remove any dependencies from the graph unless they exist as + # top-level dependencies or nested dependencies. + # + # @param [Dependency, String] dependency + # the name/dependency to remove + # + # @option options [String, Array<String>] :ignore + # the list of dependencies to ignore + def remove(dependency, options = {}) + name = Dependency.name(dependency) - # Now delete the entry - @graph.delete(name) + if @lockfile.dependency?(name) + return + end - # Recursively try to delete the remaining dependencies for this item - nested_dependencies.each(&method(:remove)) - end + if dependency?(name, options) + return + end - # Update the graph with the given cookbooks. This method destroys the - # existing dependency graph with this new result! - # - # @param [Array<CachedCookbook>] - # the list of cookbooks to populate the graph with - def update(cookbooks) - @graph = {} + # Grab the nested dependencies for this particular entry so we can + # recurse and try to remove them from the graph. + locked = @graph[name] + nested_dependencies = locked && locked.dependencies.keys || [] - cookbooks.each do |cookbook| - @graph[cookbook.cookbook_name.to_s] = GraphItem.new( - cookbook.name, - cookbook.version, - cookbook.dependencies, - ) + # Now delete the entry + @graph.delete(name) + + # Recursively try to delete the remaining dependencies for this item + nested_dependencies.each(&method(:remove)) end - end - # Write the contents of the graph to the lockfile format. - # - # The resulting format looks like: - # - # GRAPH - # apache2 (1.8.14) - # yum-epel (0.2.0) - # yum (~> 3.0) - # - # @example lockfile.graph.to_lock #=> "GRAPH\n apache2 (1.18.14)\n..." - # - # @return [String] - # - def to_lock - out = "#{Lockfile::GRAPH}\n" - @graph.sort.each do |name, item| - out << " #{name} (#{item.version})\n" + # Update the graph with the given cookbooks. This method destroys the + # existing dependency graph with this new result! + # + # @param [Array<CachedCookbook>] + # the list of cookbooks to populate the graph with + def update(cookbooks) + @graph = {} - unless item.dependencies.empty? - item.dependencies.sort.each do |name, constraint| - out << " #{name} (#{constraint})\n" - end + cookbooks.each do |cookbook| + @graph[cookbook.cookbook_name.to_s] = GraphItem.new( + cookbook.name, + cookbook.version, + cookbook.dependencies, + ) end end - out - end - - private - - # A single item inside the graph. - class GraphItem - # The name of the cookbook that corresponds to this graph item. + # Write the contents of the graph to the lockfile format. # - # @return [String] - # the name of the cookbook - attr_reader :name - - # The locked version for this graph item. + # The resulting format looks like: # + # GRAPH + # apache2 (1.8.14) + # yum-epel (0.2.0) + # yum (~> 3.0) + # + # @example lockfile.graph.to_lock #=> "GRAPH\n apache2 (1.18.14)\n..." + # # @return [String] - # the locked version of the graph item (as a string) - attr_reader :version - - # The list of dependencies and their constraints. # - # @return [Hash<String, String>] - # the list of dependencies for this graph item, where the key - # corresponds to the name of the dependency and the value is the - # version constraint. - attr_reader :dependencies + def to_lock + out = "#{Lockfile::GRAPH}\n" + @graph.sort.each do |name, item| + out << " #{name} (#{item.version})\n" - # Create a new graph item. - def initialize(name, version, dependencies = {}) - @name = name.to_s - @version = version.to_s - @dependencies = dependencies - end + unless item.dependencies.empty? + item.dependencies.sort.each do |name, constraint| + out << " #{name} (#{constraint})\n" + end + end + end - # Add a new dependency to the list. - # - # @param [#to_s] name - # the name to use - # @param [#to_s] constraint - # the version constraint to use - def add_dependency(name, constraint) - @dependencies[name.to_s] = constraint.to_s + out end + + private + + # A single item inside the graph. + class GraphItem + # The name of the cookbook that corresponds to this graph item. + # + # @return [String] + # the name of the cookbook + attr_reader :name + + # The locked version for this graph item. + # + # @return [String] + # the locked version of the graph item (as a string) + attr_reader :version + + # The list of dependencies and their constraints. + # + # @return [Hash<String, String>] + # the list of dependencies for this graph item, where the key + # corresponds to the name of the dependency and the value is the + # version constraint. + attr_reader :dependencies + + # Create a new graph item. + def initialize(name, version, dependencies = {}) + @name = name.to_s + @version = version.to_s + @dependencies = dependencies + end + + # Add a new dependency to the list. + # + # @param [#to_s] name + # the name to use + # @param [#to_s] constraint + # the version constraint to use + def add_dependency(name, constraint) + @dependencies[name.to_s] = constraint.to_s + end + end end - end end end