modules/mu/config.rb in cloud-mu-3.1.3 vs modules/mu/config.rb in cloud-mu-3.1.4

- old
+ new

@@ -16,10 +16,14 @@ require 'json' require 'erb' require 'pp' require 'json-schema' require 'net/http' +require 'mu/config/schema_helpers' +require 'mu/config/tail' +require 'mu/config/ref' +require 'mu/config/doc_helpers' autoload :GraphViz, 'graphviz' autoload :ChronicDuration, 'chronic_duration' module MU @@ -33,39 +37,10 @@ end # Exception class for deploy parameter (mu-deploy -p foo=bar) errors class DeployParamError < MuError end - # The default cloud provider for new resources. Must exist in MU.supportedClouds - # return [String] - def self.defaultCloud - configured = {} - MU::Cloud.supportedClouds.each { |cloud| - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud) - if $MU_CFG[cloud.downcase] and !$MU_CFG[cloud.downcase].empty? - configured[cloud] = $MU_CFG[cloud.downcase].size - configured[cloud] += 0.5 if cloudclass.hosted? # tiebreaker - end - } - if configured.size > 0 - return configured.keys.sort { |a, b| - configured[b] <=> configured[a] - }.first - else - MU::Cloud.supportedClouds.each { |cloud| - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud) - return cloud if cloudclass.hosted? - } - return MU::Cloud.supportedClouds.first - end - end - - # The default grooming agent for new resources. Must exist in MU.supportedGroomers. - def self.defaultGroomer - MU.localOnly ? "Ansible" : "Chef" - end - attr_accessor :nat_routes attr_reader :skipinitialupdates @@config_path = nil # The path to the most recently loaded configuration file @@ -73,124 +48,10 @@ # The path to the most recently loaded configuration file def self.config_path @@config_path end - # Accessor for our Basket of Kittens schema definition - def self.schema - @@schema - end - - # Deep merge a configuration hash so we can meld different cloud providers' - # schemas together, while preserving documentation differences - def self.schemaMerge(orig, new, cloud) - if new.is_a?(Hash) - new.each_pair { |k, v| - if cloud and k == "description" and v.is_a?(String) and !v.match(/\b#{Regexp.quote(cloud.upcase)}\b/) and !v.empty? - new[k] = "+"+cloud.upcase+"+: "+v - end - if orig and orig.has_key?(k) - elsif orig - orig[k] = new[k] - else - orig = new - end - schemaMerge(orig[k], new[k], cloud) - } - elsif orig.is_a?(Array) and new - orig.concat(new) - orig.uniq! - elsif new.is_a?(String) - orig ||= "" - orig += "\n" if !orig.empty? - orig += "+#{cloud.upcase}+: "+new - else -# XXX I think this is a NOOP? - end - end - - # Accessor for our Basket of Kittens schema definition, with various - # cloud-specific details merged so we can generate documentation for them. - def self.docSchema - docschema = Marshal.load(Marshal.dump(@@schema)) - only_children = {} - MU::Cloud.resource_types.each_pair { |classname, attrs| - MU::Cloud.supportedClouds.each { |cloud| - begin - require "mu/clouds/#{cloud.downcase}/#{attrs[:cfg_name]}" - rescue LoadError => e - next - end - res_class = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(classname) - required, res_schema = res_class.schema(self) - docschema["properties"][attrs[:cfg_plural]]["items"]["description"] ||= "" - docschema["properties"][attrs[:cfg_plural]]["items"]["description"] += "\n#\n# `#{cloud}`: "+res_class.quality - res_schema.each { |key, cfg| - if !docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] - only_children[attrs[:cfg_plural]] ||= {} - only_children[attrs[:cfg_plural]][key] ||= {} - only_children[attrs[:cfg_plural]][key][cloud] = cfg - end - } - } - } - - # recursively chase down description fields in arrays and objects of our - # schema and prepend stuff to them for documentation - def self.prepend_descriptions(prefix, cfg) - cfg["prefix"] = prefix - if cfg["type"] == "array" and cfg["items"] - cfg["items"] = prepend_descriptions(prefix, cfg["items"]) - elsif cfg["type"] == "object" and cfg["properties"] - cfg["properties"].each_pair { |key, subcfg| - cfg["properties"][key] = prepend_descriptions(prefix, cfg["properties"][key]) - } - end - cfg - end - - MU::Cloud.resource_types.each_pair { |classname, attrs| - MU::Cloud.supportedClouds.each { |cloud| - res_class = nil - begin - res_class = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(classname) - rescue MU::Cloud::MuCloudResourceNotImplemented - next - end - required, res_schema = res_class.schema(self) - next if required.size == 0 and res_schema.size == 0 - res_schema.each { |key, cfg| - cfg["description"] ||= "" - if !cfg["description"].empty? - cfg["description"] = "\n# +"+cloud.upcase+"+: "+cfg["description"] - end - if docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] - schemaMerge(docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key], cfg, cloud) - docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] ||= "" - docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] += "\n"+(cfg["description"].match(/^#/) ? "" : "# ")+cfg["description"] - MU.log "Munging #{cloud}-specific #{classname.to_s} schema into BasketofKittens => #{attrs[:cfg_plural]} => #{key}", MU::DEBUG, details: docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] - else - if only_children[attrs[:cfg_plural]][key] - prefix = only_children[attrs[:cfg_plural]][key].keys.map{ |x| x.upcase }.join(" & ")+" ONLY" - cfg["description"].gsub!(/^\n#/, '') # so we don't leave the description blank in the "optional parameters" section - cfg = prepend_descriptions(prefix, cfg) - end - - docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] = cfg - end - docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"] = {} - docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"][cloud] = cfg - } - - docschema['required'].concat(required) - docschema['required'].uniq! - } - } - - docschema - end - attr_reader :config @@parameters = {} @@user_supplied_parameters = {} attr_reader :parameters @@ -241,499 +102,20 @@ # @return [Hash] def self.stripConfig(config) MU::Config.manxify(Marshal.load(Marshal.dump(MU.structToHash(config.dup))), remove_runtime_keys: true) end - # A wrapper class for resources to refer to other resources, whether they - # be a sibling object in the current deploy, an object in another deploy, - # or a plain cloud id from outside of Mu. - class Ref - attr_reader :name - attr_reader :type - attr_reader :cloud - attr_reader :deploy_id - attr_reader :region - attr_reader :credentials - attr_reader :habitat - attr_reader :mommacat - attr_reader :tag_key - attr_reader :tag_value - attr_reader :obj - - @@refs = [] - @@ref_semaphore = Mutex.new - - # Little bit of a factory pattern... given a hash of options for a {MU::Config::Ref} objects, first see if we have an existing one that matches our more immutable attributes (+cloud+, +id+, etc). If we do, return that. If we do not, create one, add that to our inventory, and return that instead. - # @param cfg [Hash]: - # @return [MU::Config::Ref] - def self.get(cfg) - return cfg if cfg.is_a?(MU::Config::Ref) - checkfields = cfg.keys.map { |k| k.to_sym } - required = [:id, :type] - - @@ref_semaphore.synchronize { - match = nil - @@refs.each { |ref| - saw_mismatch = false - saw_match = false - needed_values = [] - checkfields.each { |field| - next if !cfg[field] - ext_value = ref.instance_variable_get("@#{field.to_s}".to_sym) - if !ext_value - needed_values << field - next - end - if cfg[field] != ext_value - saw_mismatch = true - elsif required.include?(field) and cfg[field] == ext_value - saw_match = true - end - } - if saw_match and !saw_mismatch - # populate empty fields we got from this request - if needed_values.size > 0 - newref = ref.dup - needed_values.each { |field| - newref.instance_variable_set("@#{field.to_s}".to_sym, cfg[field]) - if !newref.respond_to?(field) - newref.singleton_class.instance_eval { attr_reader field.to_sym } - end - } - @@refs << newref - return newref - else - return ref - end - end - } - - } - - # if we get here, there was no match - newref = MU::Config::Ref.new(cfg) - @@ref_semaphore.synchronize { - @@refs << newref - return newref - } - end - - # A way of dynamically defining +attr_reader+ without leaking memory - def self.define_reader(name) - define_method(name) { - instance_variable_get("@#{name.to_s}") - } - end - - # @param cfg [Hash]: A Basket of Kittens configuration hash containing - # lookup information for a cloud object - def initialize(cfg) - cfg.keys.each { |field| - next if field == "tag" - if !cfg[field].nil? - self.instance_variable_set("@#{field}".to_sym, cfg[field]) - elsif !cfg[field.to_sym].nil? - self.instance_variable_set("@#{field.to_s}".to_sym, cfg[field.to_sym]) - end - MU::Config::Ref.define_reader(field) - } - if cfg['tag'] and cfg['tag']['key'] and - !cfg['tag']['key'].empty? and cfg['tag']['value'] - @tag_key = cfg['tag']['key'] - @tag_value = cfg['tag']['value'] - end - - if @deploy_id and !@mommacat - @mommacat = MU::MommaCat.getLitter(@deploy_id, set_context_to_me: false) - elsif @mommacat and !@deploy_id - @deploy_id = @mommacat.deploy_id - end - - kitten(shallow: true) if @mommacat # try to populate the actual cloud object for this - end - - # Comparison operator - def <=>(other) - return 1 if other.nil? - self.to_s <=> other.to_s - end - - # Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}. - # @param aliases [Array<Hash>]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+). - # @return [Hash] - def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: []) - parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1') - desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource" - schema = { - "type" => "object", - "#MU_REFERENCE" => true, - "minProperties" => 1, - "description" => desc, - "properties" => { - "id" => { - "type" => "string", - "description" => "Cloud identifier of a resource we want to reference, typically used when leveraging resources not managed by MU" - }, - "name" => { - "type" => "string", - "description" => "The short (internal Mu) name of a resource we're attempting to reference. Typically used when referring to a sibling resource elsewhere in the same deploy, or in another known Mu deploy in conjunction with +deploy_id+." - }, - "type" => { - "type" => "string", - "description" => "The resource type we're attempting to reference.", - "enum" => MU::Cloud.resource_types.values.map { |t| t[:cfg_plural] } - }, - "deploy_id" => { - "type" => "string", - "description" => "Our target resource should be found in this Mu deploy." - }, - "credentials" => MU::Config.credentials_primitive, - "region" => MU::Config.region_primitive, - "cloud" => MU::Config.cloud_primitive, - "tag" => { - "type" => "object", - "description" => "If the target resource supports tagging and our resource implementations +find+ method supports it, we can attempt to locate it by tag.", - "properties" => { - "key" => { - "type" => "string", - "description" => "The tag or label key to search against" - }, - "value" => { - "type" => "string", - "description" => "The tag or label value to match" - } - } - } - } - } - if !["folders", "habitats"].include?(type) - schema["properties"]["habitat"] = MU::Config::Habitat.reference - end - - if omit_fields - omit_fields.each { |f| - schema["properties"].delete(f) - } - end - - if !type.nil? - schema["required"] = ["type"] - schema["properties"]["type"]["default"] = type - schema["properties"]["type"]["enum"] = [type] - end - - aliases.each { |a| - a.each_pair { |k, v| - if schema["properties"][v] - schema["properties"][k] = schema["properties"][v].dup - schema["properties"][k]["description"] = "Alias for <tt>#{v}</tt>" - else - MU.log "Reference schema alias #{k} wants to alias #{v}, but no such attribute exists", MU::WARN, details: caller[4] - end - } - } - - schema - end - - # Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment, - # of the sort that would have been used to declare this reference in the - # first place. - def to_h - me = { } - - self.instance_variables.each { |var| - next if [:@obj, :@mommacat, :@tag_key, :@tag_value].include?(var) - val = self.instance_variable_get(var) - next if val.nil? - val = val.to_h if val.is_a?(MU::Config::Ref) - me[var.to_s.sub(/^@/, '')] = val - } - if @tag_key and !@tag_key.empty? - me['tag'] = { - 'key' => @tag_key, - 'value' => @tag_value - } - end - me - end - - # Getter for the #{id} instance variable that attempts to populate it if - # it's not set. - # @return [String,nil] - def id - return @id if @id - kitten # if it's not defined, attempt to define it - @id - end - - # Alias for {id} - # @return [String,nil] - def cloud_id - id - end - - # Return a {MU::Cloud} object for this reference. This is only meant to be - # called in a live deploy, which is to say that if called during initial - # configuration parsing, results may be incorrect. - # @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches. - def kitten(mommacat = @mommacat, shallow: false) - return nil if !@cloud or !@type - - if @obj - @deploy_id ||= @obj.deploy_id - @id ||= @obj.cloud_id - @name ||= @obj.config['name'] - return @obj - end - - if mommacat - @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: false) - if @obj # initialize missing attributes, if we can - @id ||= @obj.cloud_id - @mommacat ||= mommacat - @obj.intoDeploy(@mommacat) # make real sure these are set - @deploy_id ||= mommacat.deploy_id - if !@name - if @obj.config and @obj.config['name'] - @name = @obj.config['name'] - elsif @obj.mu_name -if @type == "folders" -MU.log "would assign name '#{@obj.mu_name}' in ref to this folder if I were feeling aggressive", MU::WARN, details: self.to_h -end -# @name = @obj.mu_name - end - end - return @obj - else -# MU.log "Failed to find a live '#{@type.to_s}' object named #{@name}#{@id ? " (#{@id})" : "" }#{ @habitat ? " in habitat #{@habitat}" : "" }", MU::WARN, details: self - end - end - - if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud::Google::User.cannedServiceAcctName?(@id)) and !shallow - - begin - hab_arg = if @habitat.nil? - [nil] - elsif @habitat.is_a?(MU::Config::Ref) - [@habitat.id] - elsif @habitat.is_a?(Hash) - [@habitat["id"]] - else - [@habitat.to_s] - end - - found = MU::MommaCat.findStray( - @cloud, - @type, - name: @name, - cloud_id: @id, - deploy_id: @deploy_id, - region: @region, - habitats: hab_arg, - credentials: @credentials, - dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type)) - ) - @obj ||= found.first if found - rescue ThreadError => e - # Sometimes MommaCat calls us in a potential deadlock situation; - # don't be the cause of a fatal error if so, we don't need this - # object that badly. - raise e if !e.message.match(/recursive locking/) -rescue SystemExit => e -# XXX this is temporary, to cope with some debug stuff that's in findStray -# for the nonce -return - end - end - - if @obj - @deploy_id ||= @obj.deploy_id - @id ||= @obj.cloud_id - @name ||= @obj.config['name'] - end - - @obj - end - - end - - # A wrapper for config leaves that came from ERB parameters instead of raw - # YAML or JSON. Will behave like a string for things that expect that - # sort of thing. Code that needs to know that this leaf was the result of - # a parameter will be able to tell by the object class being something - # other than a plain string, array, or hash. - class Tail - @value = nil - @name = nil - @prettyname = nil - @description = nil - @prefix = "" - @suffix = "" - @is_list_element = false - @pseudo = false - @runtimecode = nil - @valid_values = [] - @index = 0 - attr_reader :description - attr_reader :pseudo - attr_reader :index - attr_reader :value - attr_reader :runtimecode - attr_reader :valid_values - attr_reader :is_list_element - - def initialize(name, value, prettyname = nil, cloudtype = "String", valid_values = [], description = "", is_list_element = false, prefix: "", suffix: "", pseudo: false, runtimecode: nil, index: 0) - @name = name - @bindings = {} - @value = value - @valid_values = valid_values - @pseudo = pseudo - @index = index - @runtimecode = runtimecode - @cloudtype = cloudtype - @is_list_element = is_list_element - @description ||= - if !description.nil? - description - else - "" - end - @prettyname ||= - if !prettyname.nil? - prettyname - else - @name.capitalize.gsub(/[^a-z0-9]/i, "") - end - @prefix = prefix if !prefix.nil? - @suffix = suffix if !suffix.nil? - end - - # Return the parameter name of this Tail - def getName - @name - end - # Return the platform-specific cloud type of this Tail - def getCloudType - @cloudtype - end - # Return the human-friendly name of this Tail - def getPrettyName - @prettyname - end - # Walk like a String - def to_s - @prefix.to_s+@value.to_s+@suffix.to_s - end - # Quack like a String - def to_str - to_s - end - # Upcase like a String - def upcase - to_s.upcase - end - # Downcase like a String - def downcase - to_s.downcase - end - # Check for emptiness like a String - def empty? - to_s.empty? - end - # Match like a String - def match(*args) - to_s.match(*args) - end - # Check for equality like a String - def ==(o) - (o.class == self.class or o.class == "String") && o.to_s == to_s - end - # Concatenate like a string - def +(o) - return to_s if o.nil? - to_s + o.to_s - end - # Perform global substitutions like a String - def gsub(*args) - to_s.gsub(*args) - end - end - - # Wrapper method for creating a {MU::Config::Tail} object as a reference to - # a parameter that's valid in the loaded configuration. - # @param param [<String>]: The name of the parameter to which this should be tied. - # @param value [<String>]: The value of the parameter to return when asked - # @param prettyname [<String>]: A human-friendly parameter name to be used when generating CloudFormation templates and the like - # @param cloudtype [<String>]: A platform-specific identifier used by cloud layers to identify a parameter's type, e.g. AWS::EC2::VPC::Id - # @param valid_values [Array<String>]: A list of acceptable String values for the given parameter. - # @param description [<String>]: A long-form description of what the parameter does. - # @param list_of [<String>]: Indicates that the value should be treated as a member of a list (array) by the cloud layer. - # @param prefix [<String>]: A static String that should be prefixed to the stored value when queried - # @param suffix [<String>]: A static String that should be appended to the stored value when queried - # @param pseudo [<Boolean>]: This is a pseudo-parameter, automatically provided, and not available as user input. - # @param runtimecode [<String>]: Actual code to allow the cloud layer to interpret literally in its own idiom, e.g. '"Ref" : "AWS::StackName"' for CloudFormation - def getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil) - if value.nil? - if @@parameters.nil? or !@@parameters.has_key?(param) - MU.log "Parameter '#{param}' (#{param.class.name}) referenced in config but not provided (#{caller[0]})", MU::DEBUG, details: @@parameters - return nil -# raise DeployParamError - else - value = @@parameters[param] - end - end - if !prettyname.nil? - prettyname.gsub!(/[^a-z0-9]/i, "") # comply with CloudFormation restrictions - end - if value.is_a?(MU::Config::Tail) - MU.log "Parameter #{param} is using a nested parameter as a value. This rarely works, depending on the target cloud. YMMV.", MU::WARN - tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode) - elsif !list_of.nil? or (@@tails.has_key?(param) and @@tails[param].is_a?(Array)) - tail = [] - count = 0 - value.split(/\s*,\s*/).each { |subval| - if @@tails.has_key?(param) and !@@tails[param][count].nil? - subval = @@tails[param][count].values.first.to_s if subval.nil? - list_of = @@tails[param][count].values.first.getName if list_of.nil? - prettyname = @@tails[param][count].values.first.getPrettyName if prettyname.nil? - description = @@tails[param][count].values.first.description if description.nil? - valid_values = @@tails[param][count].values.first.valid_values if valid_values.nil? or valid_values.empty? - cloudtype = @@tails[param][count].values.first.getCloudType if @@tails[param][count].values.first.getCloudType != "String" - end - prettyname = param.capitalize if prettyname.nil? - tail << { list_of => MU::Config::Tail.new(list_of, subval, prettyname, cloudtype, valid_values, description, true, pseudo: pseudo, index: count) } - count = count + 1 - } - else - if @@tails.has_key?(param) - pseudo = @@tails[param].pseudo - value = @@tails[param].to_s if value.nil? - prettyname = @@tails[param].getPrettyName if prettyname.nil? - description = @@tails[param].description if description.nil? - valid_values = @@tails[param].valid_values if valid_values.nil? or valid_values.empty? - cloudtype = @@tails[param].getCloudType if @@tails[param].getCloudType != "String" - end - tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode) - end - - if valid_values and valid_values.size > 0 and value - if !valid_values.include?(value) - raise DeployParamError, "Invalid parameter value '#{value}' supplied for '#{param}'" - end - end - @@tails[param] = tail - - tail - end - # Load up our YAML or JSON and parse it through ERB, optionally substituting # externally-supplied parameters. def resolveConfig(path: @@config_path, param_pass: false, cloud: nil) config = nil @param_pass = param_pass + if cloud + MU.log "Exposing cloud variable to ERB with value of #{cloud}", MU::DEBUG + end + # Catch calls to missing variables in Basket of Kittens files when being # parsed by ERB, and replace with placeholders for parameters. This # method_missing is only defined innside {MU::Config.resolveConfig} def method_missing(var_name) if @param_pass @@ -860,21 +242,19 @@ # @param skipinitialupdates [Boolean]: Whether to forcibly apply the *skipinitialupdates* flag to nodes created by this configuration. # @param params [Hash]: Optional name-value parameter pairs, which will be passed to our configuration files as ERB variables. # @param cloud [String]: Sets a parameter named 'cloud', and insert it as the default cloud platform if not already declared # @return [Hash]: The complete validated configuration for a deployment. def initialize(path, skipinitialupdates = false, params: {}, updating: nil, default_credentials: nil, cloud: nil) - $myPublicIp = MU::Cloud::AWS.getAWSMetaData("public-ipv4") - $myRoot = MU.myRoot + $myPublicIp ||= MU.mu_public_ip + $myRoot ||= MU.myRoot $myRoot.freeze - $myAZ = MU.myAZ.freeze + $myAZ ||= MU.myAZ.freeze $myAZ.freeze - $myRegion = MU.curRegion.freeze + $myRegion ||= MU.curRegion.freeze $myRegion.freeze - $myAppName = nil - @kittens = {} @kittencfg_semaphore = Mutex.new @@config_path = path @admin_firewall_rules = [] @skipinitialupdates = skipinitialupdates @@ -918,11 +298,11 @@ # for the rest of the config to reference. # XXX Figure out how to make include() add parameters for us. Right now # you can't specify parameters in an included file, because ERB is what's # doing the including, and parameters need to already be resolved so that # ERB can use them. - param_cfg, raw_erb_params_only = resolveConfig(path: @@config_path, param_pass: true, cloud: cloud) + param_cfg, _raw_erb_params_only = resolveConfig(path: @@config_path, param_pass: true, cloud: cloud) if param_cfg.has_key?("parameters") param_cfg["parameters"].each { |param| if param.has_key?("default") and param["default"].nil? param["default"] = "" end @@ -974,11 +354,11 @@ } raise DeployParamError, "One or more invalid parameters specified" if !ok $parameters = @@parameters.dup $parameters.freeze - tmp_cfg, raw_erb = resolveConfig(path: @@config_path, cloud: cloud) + tmp_cfg, _raw_erb = resolveConfig(path: @@config_path, cloud: cloud) # Convert parameter entries that constitute whole config keys into # {MU::Config::Tail} objects. def resolveTails(tree, indent= "") if tree.is_a?(Hash) @@ -1020,12 +400,10 @@ MU.log "Cloud provider #{@config['cloud']} is not supported", MU::ERR, details: MU::Cloud.supportedClouds end exit 1 end - types = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] } - MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }.each { |type| if @config[type] @config[type].each { |k| next if !k.is_a?(Hash) applyInheritedDefaults(k, type) @@ -1039,176 +417,18 @@ #pp @config #raise "DERP" @config.freeze end - # Output the dependencies of this BoK stack as a directed acyclic graph. - # Very useful for debugging. - def visualizeDependencies - # GraphViz won't like MU::Config::Tail, pare down to plain Strings - config = MU::Config.stripConfig(@config) - begin - g = GraphViz.new(:G, :type => :digraph) - # Generate a GraphViz node for each resource in this stack - nodes = {} - MU::Cloud.resource_types.each_pair { |classname, attrs| - nodes[attrs[:cfg_name]] = {} - if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]] - config[attrs[:cfg_plural]].each { |resource| - nodes[attrs[:cfg_name]][resource['name']] = g.add_nodes("#{classname}: #{resource['name']}") - } - end - } - # Now add edges corresponding to the dependencies they list - MU::Cloud.resource_types.each_pair { |classname, attrs| - if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]] - config[attrs[:cfg_plural]].each { |resource| - if resource.has_key?("dependencies") - me = nodes[attrs[:cfg_name]][resource['name']] - resource["dependencies"].each { |dep| - parent = nodes[dep['type']][dep['name']] - g.add_edges(me, parent) - } - end - } - end - } - # Spew some output? - MU.log "Emitting dependency graph as /tmp/#{config['appname']}.jpg", MU::NOTICE - g.output(:jpg => "/tmp/#{config['appname']}.jpg") - rescue Exception => e - MU.log "Failed to generate GraphViz dependency tree: #{e.inspect}. This should only matter to developers.", MU::WARN, details: e.backtrace - end - end - - # Generate a documentation-friendly dummy Ruby class for our mu.yaml main - # config. - def self.emitConfigAsRuby - example = %Q{--- -public_address: 1.2.3.4 -mu_admin_email: egtlabs@eglobaltech.com -mu_admin_name: Joe Schmoe -mommacat_port: 2260 -banner: My Example Mu Master -mu_repository: git://github.com/cloudamatic/mu.git -repos: -- https://github.com/cloudamatic/mu_demo_platform -allow_invade_foreign_vpcs: true -ansible_dir: -aws: - egtdev: - region: us-east-1 - log_bucket_name: egt-mu-log-bucket - default: true - name: egtdev - personal: - region: us-east-2 - log_bucket_name: my-mu-log-bucket - name: personal - google: - egtlabs: - project: egt-labs-admin - credentials_file: /opt/mu/etc/google.json - region: us-east4 - log_bucket_name: hexabucket-761234 - default: true -} - mu_yaml_schema = eval(%Q{ -$NOOP = true -load "#{MU.myRoot}/bin/mu-configure" -$CONFIGURABLES -}) - return if mu_yaml_schema.nil? or !mu_yaml_schema.is_a?(Hash) - muyamlpath = "#{MU.myRoot}/modules/mu/mu.yaml.rb" - MU.log "Converting mu.yaml schema to Ruby objects in #{muyamlpath}" - muyaml_rb = File.new(muyamlpath, File::CREAT|File::TRUNC|File::RDWR, 0644) - muyaml_rb.puts "# Configuration schema for mu.yaml. See also {https://github.com/cloudamatic/mu/wiki/Configuration the Mu wiki}." - muyaml_rb.puts "#" - muyaml_rb.puts "# Example:" - muyaml_rb.puts "#" - muyaml_rb.puts "# <pre>" - example.split(/\n/).each { |line| - muyaml_rb.puts "# "+line+" " # markdooooown - } - muyaml_rb.puts "# </pre>" - muyaml_rb.puts "module MuYAML" - muyaml_rb.puts "\t# The configuration file format for Mu's main config file." - self.printMuYamlSchema(muyaml_rb, [], { "subtree" => mu_yaml_schema }) - muyaml_rb.puts "end" - muyaml_rb.close - end - - # Take the schema we've defined and create a dummy Ruby class tree out of - # it, basically so we can leverage Yard to document it. - def self.emitSchemaAsRuby - kittenpath = "#{MU.myRoot}/modules/mu/kittens.rb" - MU.log "Converting Basket of Kittens schema to Ruby objects in #{kittenpath}" - kitten_rb = File.new(kittenpath, File::CREAT|File::TRUNC|File::RDWR, 0644) - kitten_rb.puts "### THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT ###" - kitten_rb.puts "#" - kitten_rb.puts "#" - kitten_rb.puts "#" - kitten_rb.puts "module MU" - kitten_rb.puts "class Config" - kitten_rb.puts "\t# The configuration file format for Mu application stacks." - self.printSchema(kitten_rb, ["BasketofKittens"], MU::Config.docSchema) - kitten_rb.puts "end" - kitten_rb.puts "end" - kitten_rb.close - - end - - # Take an IP block and split it into a more-or-less arbitrary number of - # subnets. - # @param ip_block [String]: CIDR of the network to subdivide - # @param subnets_desired [Integer]: Number of subnets we want back - # @param max_mask [Integer]: The highest netmask we're allowed to use for a subnet (various by cloud provider) - # @return [MU::Config::Tail]: Resulting subnet tails, or nil if an error occurred. - def divideNetwork(ip_block, subnets_desired, max_mask = 28) - cidr = NetAddr::IPv4Net.parse(ip_block.to_s) - - # Ugly but reliable method of landing on the right subnet size - subnet_bits = cidr.netmask.prefix_len - begin - subnet_bits += 1 - if subnet_bits > max_mask - MU.log "Can't subdivide #{cidr.to_s} into #{subnets_desired.to_s}", MU::ERR - raise MuError, "Subnets smaller than /#{max_mask} not permitted" - end - end while cidr.subnet_count(subnet_bits) < subnets_desired - - if cidr.subnet_count(subnet_bits) > subnets_desired - MU.log "Requested #{subnets_desired.to_s} subnets from #{cidr.to_s}, leaving #{(cidr.subnet_count(subnet_bits)-subnets_desired).to_s} unused /#{subnet_bits.to_s}s available", MU::NOTICE - end - - begin - subnets = [] - (0..subnets_desired).each { |x| - subnets << cidr.nth_subnet(subnet_bits, x).to_s - } - rescue RuntimeError => e - if e.message.match(/exceeds subnets available for allocation/) - MU.log e.message, MU::ERR - MU.log "I'm attempting to create #{subnets_desired} subnets (one public and one private for each Availability Zone), of #{subnet_size} addresses each, but that's too many for a /#{cidr.netmask.prefix_len} network. Either declare a larger network, or explicitly declare a list of subnets with few enough entries to fit.", MU::ERR - return nil - else - raise e - end - end - - subnets = getTail("subnetblocks", value: subnets.join(","), cloudtype: "CommaDelimitedList", description: "IP Address ranges to be used for VPC subnets", prettyname: "SubnetIpBlocks", list_of: "ip_block").map { |tail| tail["ip_block"] } - subnets - end - # See if a given resource is configured in the current stack # @param name [String]: The name of the resource being checked # @param type [String]: The type of resource being checked # @return [Boolean] def haveLitterMate?(name, type, has_multiple: false) @kittencfg_semaphore.synchronize { matches = [] - shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) + _shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type) if @kittens[cfg_plural] @kittens[cfg_plural].each { |kitten| if kitten['name'].to_s == name.to_s or kitten['virtual_name'].to_s == name.to_s or (has_multiple and name.nil?) @@ -1231,11 +451,11 @@ # Remove a resource from the current stack # @param name [String]: The name of the resource being removed # @param type [String]: The type of resource being removed def removeKitten(name, type) @kittencfg_semaphore.synchronize { - shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) + _shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type) deletia = nil if @kittens[cfg_plural] @kittens[cfg_plural].each { |kitten| if kitten['name'] == name deletia = kitten @@ -1245,54 +465,18 @@ @kittens[type].delete(deletia) if !deletia.nil? end } end - # FirewallRules can reference other FirewallRules, which means we need to do - # an extra pass to make sure we get all intra-stack dependencies correct. - # @param acl [Hash]: The configuration hash for the FirewallRule to check - # @return [Hash] - def resolveIntraStackFirewallRefs(acl, delay_validation = false) - acl["rules"].each { |acl_include| - if acl_include['sgs'] - acl_include['sgs'].each { |sg_ref| - if haveLitterMate?(sg_ref, "firewall_rules") - acl["dependencies"] ||= [] - found = false - acl["dependencies"].each { |dep| - if dep["type"] == "firewall_rule" and dep["name"] == sg_ref - dep["no_create_wait"] = true - found = true - end - } - if !found - acl["dependencies"] << { - "type" => "firewall_rule", - "name" => sg_ref, - "no_create_wait" => true - } - end - siblingfw = haveLitterMate?(sg_ref, "firewall_rules") - if !siblingfw["#MU_VALIDATED"] -# XXX raise failure somehow - insertKitten(siblingfw, "firewall_rules", delay_validation: delay_validation) - end - end - } - end - } - acl - end - # Insert a resource into the current stack # @param descriptor [Hash]: The configuration description, as from a Basket of Kittens # @param type [String]: The type of resource being added # @param delay_validation [Boolean]: Whether to hold off on calling the resource's validateConfig method # @param ignore_duplicates [Boolean]: Do not raise an exception if we attempt to insert a resource with a +name+ field that's already in use def insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false, overwrite: false) append = false - start = Time.now +# start = Time.now shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) MU.log "insertKitten on #{cfg_name} #{descriptor['name']} (delay_validation: #{delay_validation.to_s})", MU::DEBUG, details: caller[0] if overwrite removeKitten(descriptor['name'], type) @@ -1615,11 +799,11 @@ plain_cfg.delete("#TARGETCLASS") plain_cfg.delete("#TARGETNAME") plain_cfg.delete("parent_block") if cfg_plural == "vpcs" begin JSON::Validator.validate!(myschema, plain_cfg) - rescue JSON::Schema::ValidationError => e + rescue JSON::Schema::ValidationError pp plain_cfg # Use fully_validate to get the complete error list, save some time errors = JSON::Validator.fully_validate(myschema, plain_cfg) realerrors = [] errors.each { |err| @@ -1663,169 +847,67 @@ } ok end - @@allregions = [] - MU::Cloud.availableClouds.each { |cloud| - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud) - regions = cloudclass.listRegions() - @@allregions.concat(regions) if regions - } + # For our resources which specify intra-stack dependencies, make sure those + # dependencies are actually declared. + # TODO check for loops + def self.check_dependencies(config) + ok = true - # Configuration chunk for choosing a provider region - # @return [Hash] - def self.region_primitive - if !@@allregions or @@allregions.empty? - @@allregions = [] - MU::Cloud.availableClouds.each { |cloud| - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud) - return @allregions if !cloudclass.listRegions() - @@allregions.concat(cloudclass.listRegions()) - } - end - { - "type" => "string", - "enum" => @@allregions - } - end - - # Configuration chunk for choosing a set of cloud credentials - # @return [Hash] - def self.credentials_primitive - { - "type" => "string", - "description" => "Specify a non-default set of credentials to use when authenticating to cloud provider APIs, as listed in `mu.yaml` under each provider's subsection. If " - } - end - - # Configuration chunk for creating resource tags as an array of key/value - # pairs. - # @return [Hash] - def self.optional_tags_primitive - { - "type" => "boolean", - "description" => "Tag the resource with our optional tags (+MU-HANDLE+, +MU-MASTER-NAME+, +MU-OWNER+).", - "default" => true - } - end - - # Configuration chunk for creating resource tags as an array of key/value - # pairs. - # @return [Hash] - def self.tags_primitive - { - "type" => "array", - "minItems" => 1, - "items" => { - "description" => "Tags to apply to this resource. Will apply at the cloud provider level and in node groomers, where applicable.", - "type" => "object", - "title" => "tags", - "required" => ["key", "value"], - "additionalProperties" => false, - "properties" => { - "key" => { - "type" => "string", - }, - "value" => { - "type" => "string", - } + config.each_pair { |type, values| + if values.instance_of?(Array) + values.each { |resource| + if resource.kind_of?(Hash) and !resource["dependencies"].nil? + append = [] + delete = [] + resource["dependencies"].each { |dependency| + _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(dependency["type"]) + found = false + names_seen = [] + if !config[cfg_plural].nil? + config[cfg_plural].each { |service| + names_seen << service["name"].to_s + found = true if service["name"].to_s == dependency["name"].to_s + if service["virtual_name"] + names_seen << service["virtual_name"].to_s + if service["virtual_name"].to_s == dependency["name"].to_s + found = true + append_me = dependency.dup + append_me['name'] = service['name'] + append << append_me + delete << dependency + end + end + } + end + if !found + MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR, details: names_seen + ok = false + end + } + if append.size > 0 + append.uniq! + resource["dependencies"].concat(append) + end + if delete.size > 0 + delete.each { |delete_me| + resource["dependencies"].delete(delete_me) + } + end + end } - } + end } + return ok end - # Configuration chunk for choosing a cloud provider - # @return [Hash] - def self.cloud_primitive - { - "type" => "string", -# "default" => MU::Config.defaultCloud, # applyInheritedDefaults does this better - "enum" => MU::Cloud.supportedClouds - } - end - - # Generate configuration for the general-pursose ADMIN firewall rulesets - # (security groups in AWS). Note that these are unique to regions and - # individual VPCs (as well as Classic, which is just a degenerate case of - # a VPC for our purposes. - # @param vpc [Hash]: A VPC reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC). - # @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource. - # @param cloud [String]: The parent resource's cloud plugin identifier - # @param region [String]: Cloud provider region, if applicable. - # @return [Hash<String>]: A dependency description that the calling resource can then add to itself. - def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false) - if !cloud or (cloud == "AWS" and !region) - raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider" - end - hosts = Array.new - hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip - hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip - hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip - hosts << "#{admin_ip}/32" if admin_ip - hosts.uniq! - - rules = [] - if cloud == "Google" - rules = [ - { "ingress" => true, "proto" => "all", "hosts" => hosts }, - { "egress" => true, "proto" => "all", "hosts" => hosts } - ] - else - rules = [ - { "proto" => "tcp", "port_range" => "0-65535", "hosts" => hosts }, - { "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts }, - { "proto" => "icmp", "port_range" => "-1", "hosts" => hosts } - ] - end - - resclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("FirewallRule") - - if rules_only - return rules - end - - name = "admin" - name += credentials.to_s if credentials - realvpc = nil - if vpc - realvpc = {} - ['vpc_name', 'vpc_id'].each { |p| - if vpc[p] - vpc[p.sub(/^vpc_/, '')] = vpc[p] - vpc.delete(p) - end - } - ['cloud', 'id', 'name', 'deploy_id', 'habitat', 'credentials'].each { |field| - realvpc[field] = vpc[field] if !vpc[field].nil? - } - if !realvpc['id'].nil? and !realvpc['id'].empty? - # Stupid kludge for Google cloud_ids which are sometimes URLs and - # sometimes not. Requirements are inconsistent from scenario to - # scenario. - name = name + "-" + realvpc['id'].gsub(/.*\//, "") - realvpc['id'] = getTail("id", value: realvpc['id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC", cloudtype: "AWS::EC2::VPC::Id") if realvpc["id"].is_a?(String) - elsif !realvpc['name'].nil? - name = name + "-" + realvpc['name'] - end - end - - - acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials } - if cloud == "Google" and acl["vpc"] and acl["vpc"]["habitat"] - acl['project'] = acl["vpc"]["habitat"]["id"] || acl["vpc"]["habitat"]["name"] - end - acl.delete("vpc") if !acl["vpc"] - if !resclass.isGlobal? and !region.nil? and !region.empty? - acl["region"] = region - end - @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl) - return {"type" => "firewall_rule", "name" => name} - end - - private - + # Ugly text-manipulation to recursively resolve some placeholder strings + # we put in for ERB include() directives. + # @param lines [String] + # @return [String] def self.resolveYAMLAnchors(lines) new_text = "" lines.each_line { |line| if line.match(/# MU::Config\.include PLACEHOLDER /) $yaml_refs.each_pair { |anchor, data| @@ -1841,23 +923,22 @@ new_text = new_text + line } return new_text end - # Given a path to a config file, try to guess whether it's YAML or JSON. # @param path [String]: The path to the file to check. def self.guessFormat(path) raw = File.read(path) # Rip out ERB references that will bollocks parser syntax, first. stripped = raw.gsub(/<%.*?%>,?/, "").gsub(/,[\n\s]*([\]\}])/, '\1') begin JSON.parse(stripped) - rescue JSON::ParserError => e + rescue JSON::ParserError begin YAML.load(raw.gsub(/<%.*?%>/, "")) - rescue Psych::SyntaxError => e + rescue Psych::SyntaxError # Ok, well neither of those worked, let's assume that filenames are # meaningful. if path.match(/\.(yaml|yml)$/i) MU.log "Guessing that #{path} is YAML based on filename", MU::DEBUG return :yaml @@ -1933,11 +1014,11 @@ elsif file.match(/(yaml|yml)$/i) assume_type = :yaml end begin erb = ERB.new(File.read(file), nil, "<>") - rescue Errno::ENOENT => e + rescue Errno::ENOENT retries = retries + 1 if retries == 1 file = File.dirname(MU::Config.config_path)+"/"+orig_filename retry elsif retries == 2 @@ -1958,16 +1039,16 @@ # differing file formats, or work around YAML's annoying dependence # on indentation. parsed_cfg = nil begin parsed_cfg = JSON.parse(erb.result(binding)) - parsed_as = :json +# parsed_as = :json rescue JSON::ParserError => e MU.log e.inspect, MU::DEBUG begin parsed_cfg = YAML.load(MU::Config.resolveYAMLAnchors(erb.result(binding))) - parsed_as = :yaml +# parsed_as = :yaml rescue Psych::SyntaxError => e MU.log e.inspect, MU::DEBUG MU.log "#{file} parsed neither as JSON nor as YAML, including as raw text", MU::WARN if @param_pass return erb.result(binding) end @@ -1978,28 +1059,30 @@ else MU.log "Including #{file} as interpreted YAML", loglevel $yaml_refs[file] = ""+YAML.dump(parsed_cfg).sub(/^---\n/, "") return "# MU::Config.include PLACEHOLDER #{file} REDLOHECALP" end - rescue SyntaxError => e + rescue SyntaxError raise ValidationError, "ERB in #{file} threw a syntax error" end end - # (see #include) - def include(file) - MU::Config.include(file, get_binding(@@tails.keys.sort), param_pass = @param_pass) - end - @@bindings = {} # Keep a cache of bindings we've created as sandbox contexts for ERB # processing, so we don't keep reloading the entire Mu library inside new # ones. def self.global_bindings @@bindings end + private + + # (see #include) + def include(file) + MU::Config.include(file, get_binding(@@tails.keys.sort), @param_pass) + end + # Namespace magic to pass to ERB's result method. def get_binding(keyset) environment = $environment myPublicIp = $myPublicIp myRoot = $myRoot @@ -2010,246 +1093,10 @@ # return MU::Config.global_bindings[keyset] if MU::Config.global_bindings[keyset] MU::Config.global_bindings[keyset] = binding MU::Config.global_bindings[keyset] end - def applySchemaDefaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil, type: nil) - return if schema_chunk.nil? - - if conf_chunk != nil and schema_chunk["properties"].kind_of?(Hash) and conf_chunk.is_a?(Hash) - - if schema_chunk["properties"]["creation_style"].nil? or - schema_chunk["properties"]["creation_style"] != "existing" - schema_chunk["properties"].each_pair { |key, subschema| - shortclass = if conf_chunk[key] - shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(key) - shortclass - else - nil - end - - new_val = applySchemaDefaults(conf_chunk[key], subschema, depth+1, conf_chunk, type: shortclass).dup - - conf_chunk[key] = Marshal.load(Marshal.dump(new_val)) if !new_val.nil? - } - end - elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array) - conf_chunk.map! { |item| - # If we're working on a resource type, go get implementation-specific - # schema information so that we set those defaults correctly. - realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud']) - - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(item["cloud"]).const_get(type) - toplevel_required, cloudschema = cloudclass.schema(self) - - newschema = schema_chunk["items"].dup - newschema["properties"].merge!(cloudschema) - newschema - else - schema_chunk["items"].dup - end - - applySchemaDefaults(item, realschema, depth+1, conf_chunk, type: type).dup - } - else - if conf_chunk.nil? and !schema_chunk["default_if"].nil? and !siblings.nil? - schema_chunk["default_if"].each { |cond| - if siblings[cond["key_is"]] == cond["value_is"] - return Marshal.load(Marshal.dump(cond["set"])) - end - } - end - if conf_chunk.nil? and schema_chunk["default"] != nil - return Marshal.load(Marshal.dump(schema_chunk["default"])) - end - end - - return conf_chunk - end - - # For our resources which specify intra-stack dependencies, make sure those - # dependencies are actually declared. - # TODO check for loops - def self.check_dependencies(config) - ok = true - - config.each_pair { |type, values| - if values.instance_of?(Array) - values.each { |resource| - if resource.kind_of?(Hash) and !resource["dependencies"].nil? - append = [] - delete = [] - resource["dependencies"].each { |dependency| - shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(dependency["type"]) - found = false - names_seen = [] - if !config[cfg_plural].nil? - config[cfg_plural].each { |service| - names_seen << service["name"].to_s - found = true if service["name"].to_s == dependency["name"].to_s - if service["virtual_name"] - names_seen << service["virtual_name"].to_s - if service["virtual_name"].to_s == dependency["name"].to_s - found = true - append_me = dependency.dup - append_me['name'] = service['name'] - append << append_me - delete << dependency - end - end - } - end - if !found - MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR, details: names_seen - ok = false - end - } - if append.size > 0 - append.uniq! - resource["dependencies"].concat(append) - end - if delete.size > 0 - delete.each { |delete_me| - resource["dependencies"].delete(delete_me) - } - end - end - } - end - } - return ok - end - - - # Verify that a server or server_pool has a valid AD config referencing - # valid Vaults for credentials. - def self.check_vault_refs(server) - ok = true - server['vault_access'] = [] if server['vault_access'].nil? - server['groomer'] ||= self.defaultGroomer - groomclass = MU::Groomer.loadGroomer(server['groomer']) - - begin - if !server['active_directory'].nil? - ["domain_admin_vault", "domain_join_vault"].each { |vault_class| - server['vault_access'] << { - "vault" => server['active_directory'][vault_class]['vault'], - "item" => server['active_directory'][vault_class]['item'] - } - item = groomclass.getSecret( - vault: server['active_directory'][vault_class]['vault'], - item: server['active_directory'][vault_class]['item'], - ) - ["username_field", "password_field"].each { |field| - if !item.has_key?(server['active_directory'][vault_class][field]) - ok = false - MU.log "I don't see a value named #{field} in Chef Vault #{server['active_directory'][vault_class]['vault']}:#{server['active_directory'][vault_class]['item']}", MU::ERR - end - } - } - end - - if !server['windows_auth_vault'].nil? - server['use_cloud_provider_windows_password'] = false - - server['vault_access'] << { - "vault" => server['windows_auth_vault']['vault'], - "item" => server['windows_auth_vault']['item'] - } - item = groomclass.getSecret( - vault: server['windows_auth_vault']['vault'], - item: server['windows_auth_vault']['item'] - ) - ["password_field", "ec2config_password_field", "sshd_password_field"].each { |field| - if !item.has_key?(server['windows_auth_vault'][field]) - MU.log "No value named #{field} in Chef Vault #{server['windows_auth_vault']['vault']}:#{server['windows_auth_vault']['item']}, will use a generated password.", MU::NOTICE - server['windows_auth_vault'].delete(field) - end - } - end - # Check all of the non-special ones while we're at it - server['vault_access'].each { |v| - next if v['vault'] == "splunk" and v['item'] == "admin_user" - item = groomclass.getSecret(vault: v['vault'], item: v['item']) - } - rescue MuError - MU.log "Can't load a Chef Vault I was configured to use. Does it exist?", MU::ERR - ok = false - end - return ok - end - - - # Given a bare hash describing a resource, insert default values which can - # be inherited from its parent or from the root of the BoK. - # @param kitten [Hash]: A resource descriptor - # @param type [String]: The type of resource this is ("servers" etc) - def applyInheritedDefaults(kitten, type) - return if !kitten.is_a?(Hash) - kitten['cloud'] ||= @config['cloud'] - kitten['cloud'] ||= MU::Config.defaultCloud - - if !MU::Cloud.supportedClouds.include?(kitten['cloud']) - return - end - - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']) - shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) - resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass) - - schema_fields = ["us_only", "scrub_mu_isms", "credentials", "billing_acct"] - if !resclass.isGlobal? - kitten['region'] ||= @config['region'] - kitten['region'] ||= cloudclass.myRegion(kitten['credentials']) - schema_fields << "region" - end - - kitten['credentials'] ||= @config['credentials'] - kitten['credentials'] ||= cloudclass.credConfig(name_only: true) - - kitten['us_only'] ||= @config['us_only'] - kitten['us_only'] ||= false - - kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms'] - kitten['scrub_mu_isms'] ||= false - - if kitten['cloud'] == "Google" -# TODO this should be cloud-generic (handle AWS accounts, Azure subscriptions) - if resclass.canLiveIn.include?(:Habitat) - kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials']) - schema_fields << "project" - end - if kitten['region'].nil? and !kitten['#MU_CLOUDCLASS'].nil? and - !resclass.isGlobal? and - ![MU::Cloud::VPC, MU::Cloud::FirewallRule].include?(kitten['#MU_CLOUDCLASS']) - if MU::Cloud::Google.myRegion((kitten['credentials'])).nil? - raise ValidationError, "Google '#{type}' resource '#{kitten['name']}' declared without a region, but no default Google region declared in mu.yaml under #{kitten['credentials'].nil? ? "default" : kitten['credentials']} credential set" - end - kitten['region'] ||= MU::Cloud::Google.myRegion - end - elsif kitten["cloud"] == "AWS" and !resclass.isGlobal? and !kitten['region'] - if MU::Cloud::AWS.myRegion.nil? - raise ValidationError, "AWS resource declared without a region, but no default AWS region found" - end - kitten['region'] ||= MU::Cloud::AWS.myRegion - end - - - kitten['billing_acct'] ||= @config['billing_acct'] if @config['billing_acct'] - - kitten["dependencies"] ||= [] - - # Make sure the schema knows about these "new" fields, so that validation - # doesn't trip over them. - schema_fields.each { |field| - if @@schema["properties"][field] - MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG, details: @@schema["properties"][field] - @@schema["properties"][type]["items"]["properties"][field] ||= @@schema["properties"][field] - end - } - end - def validate(config = @config) ok = true count = 0 @kittens ||= {} @@ -2275,13 +1122,15 @@ @kittens[type].each { |descriptor| ok = false if !insertKitten(descriptor, type) } } + newrules = [] @kittens["firewall_rules"].each { |acl| - acl = resolveIntraStackFirewallRefs(acl) + newrules << resolveIntraStackFirewallRefs(acl) } + @kittens["firewall_rules"] = newrules # VPCs do complex things in their cloud-layer validation that other # resources tend to need, like subnet allocation, so hit them early. @kittens["vpcs"].each { |vpc| ok = false if !insertKitten(vpc, "vpcs") @@ -2319,11 +1168,11 @@ end ruleset = haveLitterMate?("database"+db['name'], "firewall_rules") if ruleset ["server_pools", "servers"].each { |type| - shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) + _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type) @kittens[cfg_plural].each { |server| server["dependencies"].each { |dep| if dep["type"] == "database" and dep["name"] == db["name"] # XXX this is AWS-specific, I think. We need to use source_tags to make this happen in Google. This logic probably needs to be dumped into the database layer. ruleset["rules"] << { @@ -2376,520 +1225,22 @@ # raise ValidationError, "Validation error in #{@@config_path}!\n"+realerrors.join("\n") # end # end end - # Emit our mu.yaml schema in a format that YARD can comprehend and turn into - # documentation. - def self.printMuYamlSchema(muyaml_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil) - return if schema.nil? - if schema["subtree"] - printme = Array.new - # order sub-elements by whether they're required, so we can use YARD's - # grouping tags on them - have_required = schema["subtree"].keys.any? { |k| schema["subtree"][k]["required"] } - prop_list = schema["subtree"].keys.sort { |a, b| - if schema["subtree"][a]["required"] and !schema["subtree"][b]["required"] - -1 - elsif !schema["subtree"][a]["required"] and schema["subtree"][b]["required"] - 1 - else - a <=> b - end - } - - req = false - printme << "# @!group Optional parameters" if !have_required - prop_list.each { |name| - prop = schema["subtree"][name] - if prop["required"] - printme << "# @!group Required parameters" if !req - req = true - else - if req - printme << "# @!endgroup" - printme << "# @!group Optional parameters" - end - req = false - end - - printme << self.printMuYamlSchema(muyaml_rb, class_hierarchy+ [name], prop, false, req) - } - printme << "# @!endgroup" - - desc = (schema['desc'] || schema['title']) - - tabs = 1 - class_hierarchy.each { |classname| - if classname == class_hierarchy.last and desc - muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{desc}\n" - end - muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}" - tabs = tabs + 1 - } - printme.each { |lines| - if !lines.nil? and lines.is_a?(String) - lines.lines.each { |line| - muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + line - } - end - } - - class_hierarchy.each { |classname| - tabs = tabs - 1 - muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end" - } - - # And now that we've dealt with our children, pass our own rendered - # commentary back up to our caller. - name = class_hierarchy.last - if in_array - type = "Array<#{class_hierarchy.join("::")}>" - else - type = class_hierarchy.join("::") - end - - docstring = "\n" - docstring = docstring + "# **REQUIRED**\n" if required -# docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"] - docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if desc - docstring = docstring + "#\n" - docstring = docstring + "# @return [#{type}]\n" - docstring = docstring + "# @see #{class_hierarchy.join("::")}\n" - docstring = docstring + "attr_accessor :#{name}" - return docstring - - else - in_array = schema["array"] - name = class_hierarchy.last - type = if schema['boolean'] - "Boolean" - else - "String" - end - if in_array - type = "Array<#{type}>" - end - docstring = "\n" - - prefixes = [] - prefixes << "# **REQUIRED**" if schema["required"] and schema['default'].nil? -# prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"] - prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil? - if !schema['pattern'].nil? - # XXX unquoted regex chars confuse the hell out of YARD. How do we - # quote {}[] etc in YARD-speak? - prefixes << "# **Must match pattern `#{schema['pattern'].to_s.gsub(/\n/, "\n#")}`**" - end - - desc = (schema['desc'] || schema['title']) - if prefixes.size > 0 - docstring += prefixes.join(",\n") - if desc and desc.size > 1 - docstring += " - " - end - docstring += "\n" - end - - docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if !desc.nil? - docstring = docstring + "#\n" - docstring = docstring + "# @return [#{type}]\n" - docstring = docstring + "attr_accessor :#{name}" - - return docstring - end - - end - - # Emit our Basket of Kittens schema in a format that YARD can comprehend - # and turn into documentation. - def self.printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil) - return if schema.nil? - - if schema["type"] == "object" - printme = [] - - if !schema["properties"].nil? - # order sub-elements by whether they're required, so we can use YARD's - # grouping tags on them - if !schema["required"].nil? and schema["required"].size > 0 - prop_list = schema["properties"].keys.sort_by { |name| - schema["required"].include?(name) ? 0 : 1 - } - else - prop_list = schema["properties"].keys - end - req = false - printme << "# @!group Optional parameters" if schema["required"].nil? or schema["required"].size == 0 - prop_list.each { |name| - prop = schema["properties"][name] - - if class_hierarchy.size == 1 - - _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name) - if cfg_name - example_path = MU.myRoot+"/modules/mu/config/"+cfg_name+".yml" - if File.exist?(example_path) - example = "#\n# Examples:\n#\n" - # XXX these variables are all parameters from the BoKs in - # modules/tests. A really clever implementation would read - # and parse them to get default values, perhaps, instead of - # hard-coding them here. - instance_type = "t2.medium" - db_size = "db.t2.medium" - vpc_name = "some_vpc" - logs_name = "some_loggroup" - queues_name = "some_queue" - server_pools_name = "some_server_pool" - ["simple", "complex"].each { |complexity| - erb = ERB.new(File.read(example_path), nil, "<>") - example += "# !!!yaml\n" - example += "# ---\n" - example += "# appname: #{complexity}\n" - example += "# #{cfg_plural}:\n" - firstline = true - erb.result(binding).split(/\n/).each { |l| - l.chomp! - l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/) - next if l.empty? or l.match(/^\s+$/) - if firstline - l = "- "+l - firstline = false - else - l = " "+l - end - example += "# "+l+" "+"\n" - } - example += "# &nbsp;\n#\n" if complexity == "simple" - } - schema["properties"][name]["items"]["description"] ||= "" - if !schema["properties"][name]["items"]["description"].empty? - schema["properties"][name]["items"]["description"] += "\n" - end - schema["properties"][name]["items"]["description"] += example - end - end - end - - if !schema["required"].nil? and schema["required"].include?(name) - printme << "# @!group Required parameters" if !req - req = true - else - if req - printme << "# @!endgroup" - printme << "# @!group Optional parameters" - end - req = false - end - - printme << self.printSchema(kitten_rb, class_hierarchy+ [name], prop, false, req, prefix: schema["prefix"]) - } - printme << "# @!endgroup" - end - - tabs = 1 - class_hierarchy.each { |classname| - if classname == class_hierarchy.last and !schema['description'].nil? - kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{schema['description']}\n" - end - kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}" - tabs = tabs + 1 - } - printme.each { |lines| - if !lines.nil? and lines.is_a?(String) - lines.lines.each { |line| - kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + line - } - end - } - - class_hierarchy.each { |classname| - tabs = tabs - 1 - kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end" - } - - # And now that we've dealt with our children, pass our own rendered - # commentary back up to our caller. - name = class_hierarchy.last - if in_array - type = "Array<#{class_hierarchy.join("::")}>" - else - type = class_hierarchy.join("::") - end - - docstring = "\n" - docstring = docstring + "# **REQUIRED**\n" if required - docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"] - docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil? - docstring = docstring + "#\n" - docstring = docstring + "# @return [#{type}]\n" - docstring = docstring + "# @see #{class_hierarchy.join("::")}\n" - docstring = docstring + "attr_accessor :#{name}" - return docstring - - elsif schema["type"] == "array" - return self.printSchema(kitten_rb, class_hierarchy, schema['items'], true, required, prefix: prefix) - else - name = class_hierarchy.last - if schema['type'].nil? - MU.log "Couldn't determine schema type in #{class_hierarchy.join(" => ")}", MU::WARN, details: schema - return nil - end - if in_array - type = "Array<#{schema['type'].capitalize}>" - else - type = schema['type'].capitalize - end - docstring = "\n" - - prefixes = [] - prefixes << "# **REQUIRED**" if required and schema['default'].nil? - prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"] - prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil? - if !schema['enum'].nil? and !schema["enum"].empty? - prefixes << "# **Must be one of: `#{schema['enum'].join(', ')}`**" - elsif !schema['pattern'].nil? - # XXX unquoted regex chars confuse the hell out of YARD. How do we - # quote {}[] etc in YARD-speak? - prefixes << "# **Must match pattern `#{schema['pattern'].gsub(/\n/, "\n#")}`**" - end - - if prefixes.size > 0 - docstring += prefixes.join(",\n") - if schema['description'] and schema['description'].size > 1 - docstring += " - " - end - docstring += "\n" - end - - docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil? - docstring = docstring + "#\n" - docstring = docstring + "# @return [#{type}]\n" - docstring = docstring + "attr_accessor :#{name}" - - return docstring - end - - end - - def self.dependencies_primitive - { - "type" => "array", - "items" => { - "type" => "object", - "description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.", - "required" => ["name", "type"], - "additionalProperties" => false, - "properties" => { - "name" => {"type" => "string"}, - "type" => { - "type" => "string", - "enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] } - }, - "phase" => { - "type" => "string", - "description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.", - "enum" => ["create", "groom"] - }, - "no_create_wait" => { - "type" => "boolean", - "default" => false, - "description" => "By default, it's assumed that we want to wait on our parents' creation phase, in addition to whatever is declared in this stanza. Setting this flag will bypass waiting on our parent resource's creation, so that our create or groom phase can instead depend only on the parent's groom phase. " - } - } - } - } - end - - CIDR_PATTERN = "^\\d+\\.\\d+\\.\\d+\\.\\d+\/[0-9]{1,2}$" - CIDR_DESCRIPTION = "CIDR-formatted IP block, e.g. 1.2.3.4/32" - CIDR_PRIMITIVE = { - "type" => "string", - "pattern" => CIDR_PATTERN, - "description" => CIDR_DESCRIPTION - } - - # Have a default value available for config schema elements that take an - # email address. - # @return [String] - def self.notification_email - if MU.chef_user == "mu" - ENV['MU_ADMIN_EMAIL'] - else - MU.userEmail - end - end - - # Load and validate the schema for an individual resource class, optionally - # merging cloud-specific schema components. - # @param type [String]: The resource type to load - # @param cloud [String]: A specific cloud, whose implementation's schema of this resource we will merge - # @return [Hash] - def self.loadResourceSchema(type, cloud: nil) - valid = true - shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) - schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass) - - [:schema, :validate].each { |method| - if !schemaclass.respond_to?(method) - MU.log "MU::Config::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR - return [nil, false] if method == :schema - valid = false - end - } - - schema = schemaclass.schema.dup - - schema["properties"]["virtual_name"] = { - "description" => "Internal use.", - "type" => "string" - } - schema["properties"]["dependencies"] = MU::Config.dependencies_primitive - schema["properties"]["cloud"] = MU::Config.cloud_primitive - schema["properties"]["credentials"] = MU::Config.credentials_primitive - schema["title"] = type.to_s - - if cloud - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(shortclass) - - if cloudclass.respond_to?(:schema) - reqd, cloudschema = cloudclass.schema - cloudschema.each { |key, cfg| - if schema["properties"][key] - schemaMerge(schema["properties"][key], cfg, cloud) - else - schema["properties"][key] = cfg.dup - end - } - else - MU.log "MU::Cloud::#{cloud}::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR - valid = false - end - - end - - return [schema, valid] - end - - @@schema = { - "$schema" => "http://json-schema.org/draft-04/schema#", - "title" => "MU Application", - "type" => "object", - "description" => "A MU application stack, consisting of at least one resource.", - "required" => ["admins", "appname"], - "properties" => { - "appname" => { - "type" => "string", - "description" => "A name for your application stack. Should be short, but easy to differentiate from other applications.", - }, - "scrub_mu_isms" => { - "type" => "boolean", - "description" => "When 'cloud' is set to 'CloudFormation,' use this flag to strip out Mu-specific artifacts (tags, standard userdata, naming conventions, etc) to yield a clean, source-agnostic template. Setting this flag here will override declarations in individual resources." - }, - "project" => { - "type" => "string", - "description" => "**GOOGLE ONLY**: The project into which to deploy resources" - }, - "billing_acct" => { - "type" => "string", - "description" => "**GOOGLE ONLY**: Billing account ID to associate with a newly-created Google Project. If not specified, will attempt to locate a billing account associated with the default project for our credentials.", - }, - "region" => MU::Config.region_primitive, - "credentials" => MU::Config.credentials_primitive, - "us_only" => { - "type" => "boolean", - "description" => "For resources which span regions, restrict to regions inside the United States", - "default" => false - }, - "conditions" => { - "type" => "array", - "items" => { - "type" => "object", - "required" => ["name", "cloudcode"], - "description" => "CloudFormation-specific. Define Conditions as in http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/conditions-section-structure.html. Arguments must use the cloudCode() macro.", - "properties" => { - "name" => { "required" => true, "type" => "string" }, - "cloudcode" => { "required" => true, "type" => "string" }, - } - } - }, - "parameters" => { - "type" => "array", - "items" => { - "type" => "object", - "title" => "parameter", - "description" => "Parameters to be substituted elsewhere in this Basket of Kittens as ERB variables (<%= varname %>)", - "additionalProperties" => false, - "properties" => { - "name" => { "required" => true, "type" => "string" }, - "default" => { "type" => "string" }, - "list_of" => { - "type" => "string", - "description" => "Treat the value as a comma-separated list of values with this key name, equivalent to CloudFormation's various List<> types. For example, set to 'subnet_id' to pass values as an array of subnet identifiers as the 'subnets' argument of a VPC stanza." - }, - "prettyname" => { - "type" => "string", - "description" => "An alternative name to use when generating parameter fields in, for example, CloudFormation templates" - }, - "description" => {"type" => "string"}, - "cloudtype" => { - "type" => "string", - "description" => "A platform-specific string describing the type of validation to use for this parameter. E.g. when generating a CloudFormation template, set to AWS::EC2::Image::Id to validate input as an AMI identifier." - }, - "required" => { - "type" => "boolean", - "default" => true - }, - "valid_values" => { - "type" => "array", - "description" => "List of valid values for this parameter. Can only be a list of static strings, for now.", - "items" => { - "type" => "string" - } - } - } - } - }, - # TODO availability zones (or an array thereof) - - "admins" => { - "type" => "array", - "items" => { - "type" => "object", - "title" => "admin", - "description" => "Administrative contacts for this application stack. Will be automatically set to invoking Mu user, if not specified.", - "required" => ["name", "email"], - "additionalProperties" => false, - "properties" => { - "name" => {"type" => "string"}, - "email" => {"type" => "string"}, - "public_key" => { - "type" => "string", - "description" => "An OpenSSH-style public key string. This will be installed on all instances created in this deployment." - } - } - }, - "minItems" => 1, - "uniqueItems" => true - } - }, - "additionalProperties" => false - } - failed = [] # Load all of the config stub files at the Ruby level MU::Cloud.resource_types.each_pair { |type, cfg| begin require "mu/config/#{cfg[:cfg_name]}" - rescue LoadError => e + rescue LoadError # raise MuError, "MU::Config implemention of #{type} missing from modules/mu/config/#{cfg[:cfg_name]}.rb" MU.log "MU::Config::#{type} stub class is missing", MU::ERR failed << type next end } - MU::Cloud.resource_types.each_pair { |type, cfg| begin schema, valid = loadResourceSchema(type) failed << type if !valid