modules/mu/config.rb in cloud-mu-2.1.0beta vs modules/mu/config.rb in cloud-mu-3.0.0beta
- old
+ new
@@ -26,10 +26,13 @@
# Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
class Config
# Exception class for BoK parse or validation errors
class ValidationError < MU::MuError
end
+ # Exception class for duplicate resource names
+ class DuplicateNameError < MU::MuError
+ end
# Exception class for deploy parameter (mu-deploy -p foo=bar) errors
class DeployParamError < MuError
end
# The default cloud provider for new resources. Must exist in MU.supportedClouds
@@ -39,74 +42,33 @@
MU::Cloud.supportedClouds.each { |cloud|
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
if $MU_CFG[cloud.downcase] and !$MU_CFG[cloud.downcase].empty?
configured[cloud] = $MU_CFG[cloud.downcase].size
configured[cloud] += 0.5 if cloudclass.hosted? # tiebreaker
- elsif cloudclass.hosted?
- configured[cloud] = 1
end
}
if configured.size > 0
return configured.keys.sort { |a, b|
configured[b] <=> configured[a]
}.first
else
+ MU::Cloud.supportedClouds.each { |cloud|
+ cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+ return cloud if cloudclass.hosted?
+ }
return MU::Cloud.supportedClouds.first
end
end
# The default grooming agent for new resources. Must exist in MU.supportedGroomers.
def self.defaultGroomer
- "Chef"
+ MU.localOnly ? "Ansible" : "Chef"
end
attr_accessor :nat_routes
attr_reader :skipinitialupdates
- attr_reader :google_images
- @@google_images = YAML.load(File.read("#{MU.myRoot}/modules/mu/defaults/google_images.yaml"))
- if File.exists?("#{MU.etcDir}/google_images.yaml")
- custom = YAML.load(File.read("#{MU.etcDir}/google_images.yaml"))
- @@google_images.merge!(custom) { |key, oldval, newval|
- if !oldval.is_a?(Hash) and !newval.nil?
- if !newval.nil?
- newval
- else
- oldval
- end
- else
- oldval.merge(newval)
- end
- }
- end
- # The list of known Google Images which we can use for a given platform
- def self.google_images
- @@google_images
- end
-
- attr_reader :amazon_images
- @@amazon_images = YAML.load(File.read("#{MU.myRoot}/modules/mu/defaults/amazon_images.yaml"))
- if File.exists?("#{MU.etcDir}/amazon_images.yaml")
- custom = YAML.load(File.read("#{MU.etcDir}/amazon_images.yaml"))
- @@amazon_images.merge!(custom) { |key, oldval, newval|
- if !oldval.is_a?(Hash) and !newval.nil?
- if !newval.nil?
- newval
- else
- oldval
- end
- else
- oldval.merge(newval)
- end
- }
- end
- # The list of known Amazon AMIs, by region, which we can use for a given
- # platform.
- def self.amazon_images
- @@amazon_images
- end
-
@@config_path = nil
# The path to the most recently loaded configuration file
attr_reader :config_path
# The path to the most recently loaded configuration file
def self.config_path
@@ -121,24 +83,28 @@
# Deep merge a configuration hash so we can meld different cloud providers'
# schemas together, while preserving documentation differences
def self.schemaMerge(orig, new, cloud)
if new.is_a?(Hash)
new.each_pair { |k, v|
+ if cloud and k == "description" and v.is_a?(String) and !v.match(/\b#{Regexp.quote(cloud.upcase)}\b/) and !v.empty?
+ new[k] = "+"+cloud.upcase+"+: "+v
+ end
if orig and orig.has_key?(k)
- schemaMerge(orig[k], new[k], cloud)
elsif orig
orig[k] = new[k]
else
orig = new
end
+ schemaMerge(orig[k], new[k], cloud)
}
elsif orig.is_a?(Array) and new
orig.concat(new)
orig.uniq!
elsif new.is_a?(String)
orig ||= ""
- orig += "\n#{cloud.upcase}: "+new
+ orig += "\n" if !orig.empty?
+ orig += "+#{cloud.upcase}+: "+new
else
# XXX I think this is a NOOP?
end
end
@@ -169,12 +135,10 @@
}
# recursively chase down description fields in arrays and objects of our
# schema and prepend stuff to them for documentation
def self.prepend_descriptions(prefix, cfg)
-# cfg["description"] ||= ""
-# cfg["description"] = prefix+cfg["description"]
cfg["prefix"] = prefix
if cfg["type"] == "array" and cfg["items"]
cfg["items"] = prepend_descriptions(prefix, cfg["items"])
elsif cfg["type"] == "object" and cfg["properties"]
cfg["properties"].each_pair { |key, subcfg|
@@ -194,19 +158,22 @@
end
required, res_schema = res_class.schema(self)
next if required.size == 0 and res_schema.size == 0
res_schema.each { |key, cfg|
cfg["description"] ||= ""
- cfg["description"] = "+"+cloud.upcase+"+: "+cfg["description"]
+ if !cfg["description"].empty?
+ cfg["description"] = "\n# +"+cloud.upcase+"+: "+cfg["description"]
+ end
if docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
schemaMerge(docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key], cfg, cloud)
docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] ||= ""
docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] += "\n"+(cfg["description"].match(/^#/) ? "" : "# ")+cfg["description"]
MU.log "Munging #{cloud}-specific #{classname.to_s} schema into BasketofKittens => #{attrs[:cfg_plural]} => #{key}", MU::DEBUG, details: docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
else
if only_children[attrs[:cfg_plural]][key]
prefix = only_children[attrs[:cfg_plural]][key].keys.map{ |x| x.upcase }.join(" & ")+" ONLY"
+ cfg["description"].gsub!(/^\n#/, '') # so we don't leave the description blank in the "optional parameters" section
cfg = prepend_descriptions(prefix, cfg)
end
docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] = cfg
end
@@ -245,23 +212,338 @@
# layers that don't care about the metadata in Tails.
# @param config [Hash]: The configuration tree to convert
# @return [Hash]: The modified configuration
def self.manxify(config)
if config.is_a?(Hash)
+ newhash = {}
config.each_pair { |key, val|
- config[key] = self.manxify(val)
+ newhash[key] = self.manxify(val)
}
+ config = newhash
elsif config.is_a?(Array)
+ newarray = []
config.each { |val|
- val = self.manxify(val)
+ newarray << self.manxify(val)
}
+ config = newarray
elsif config.is_a?(MU::Config::Tail)
return config.to_s
+ elsif config.is_a?(MU::Config::Ref)
+ return config.to_h
end
return config
end
+ # Make a deep copy of a config hash and pare it down to only primitive
+ # types, even at the leaves.
+ # @param config [Hash]
+ # @return [Hash]
+ def self.stripConfig(config)
+ MU::Config.manxify(Marshal.load(Marshal.dump(MU.structToHash(config.dup))))
+ end
+
+ # A wrapper class for resources to refer to other resources, whether they
+ # be a sibling object in the current deploy, an object in another deploy,
+ # or a plain cloud id from outside of Mu.
+ class Ref
+ attr_reader :name
+ attr_reader :type
+ attr_reader :cloud
+ attr_reader :deploy_id
+ attr_reader :region
+ attr_reader :credentials
+ attr_reader :habitat
+ attr_reader :mommacat
+ attr_reader :tag_key
+ attr_reader :tag_value
+ attr_reader :obj
+
+ @@refs = []
+ @@ref_semaphore = Mutex.new
+
+ # Little bit of a factory pattern... given a hash of options for a {MU::Config::Ref} objects, first see if we have an existing one that matches our more immutable attributes (+cloud+, +id+, etc). If we do, return that. If we do not, create one, add that to our inventory, and return that instead.
+ # @param cfg [Hash]:
+ # @return [MU::Config::Ref]
+ def self.get(cfg)
+ return cfg if cfg.is_a?(MU::Config::Ref)
+ checkfields = cfg.keys.map { |k| k.to_sym }
+ required = [:id, :type]
+
+ @@ref_semaphore.synchronize {
+ match = nil
+ @@refs.each { |ref|
+ saw_mismatch = false
+ saw_match = false
+ needed_values = []
+ checkfields.each { |field|
+ next if !cfg[field]
+ ext_value = ref.instance_variable_get("@#{field.to_s}".to_sym)
+ if !ext_value
+ needed_values << field
+ next
+ end
+ if cfg[field] != ext_value
+ saw_mismatch = true
+ elsif required.include?(field) and cfg[field] == ext_value
+ saw_match = true
+ end
+ }
+ if saw_match and !saw_mismatch
+ # populate empty fields we got from this request
+ if needed_values.size > 0
+ newref = ref.dup
+ needed_values.each { |field|
+ newref.instance_variable_set("@#{field.to_s}".to_sym, cfg[field])
+ if !newref.respond_to?(field)
+ newref.singleton_class.instance_eval { attr_reader field.to_sym }
+ end
+ }
+ @@refs << newref
+ return newref
+ else
+ return ref
+ end
+ end
+ }
+
+ }
+
+ # if we get here, there was no match
+ newref = MU::Config::Ref.new(cfg)
+ @@ref_semaphore.synchronize {
+ @@refs << newref
+ return newref
+ }
+ end
+
+ # @param cfg [Hash]: A Basket of Kittens configuration hash containing
+ # lookup information for a cloud object
+ def initialize(cfg)
+ cfg.keys.each { |field|
+ next if field == "tag"
+ if !cfg[field].nil?
+ self.instance_variable_set("@#{field}".to_sym, cfg[field])
+ elsif !cfg[field.to_sym].nil?
+ self.instance_variable_set("@#{field.to_s}".to_sym, cfg[field.to_sym])
+ end
+ self.singleton_class.instance_eval { attr_reader field.to_sym }
+ }
+ if cfg['tag'] and cfg['tag']['key'] and
+ !cfg['tag']['key'].empty? and cfg['tag']['value']
+ @tag_key = cfg['tag']['key']
+ @tag_value = cfg['tag']['value']
+ end
+
+ if @deploy_id and !@mommacat
+ @mommacat = MU::MommaCat.new(@deploy_id, set_context_to_me: false, create: false)
+ elsif @mommacat and !@deploy_id
+ @deploy_id = @mommacat.deploy_id
+ end
+
+ kitten if @mommacat # try to populate the actual cloud object for this
+ end
+
+ # Comparison operator
+ def <=>(other)
+ return 1 if other.nil?
+ self.to_s <=> other.to_s
+ end
+
+ # Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}.
+ # @param aliases [Array<Hash>]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+).
+ # @return [Hash]
+ def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil)
+ parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1')
+ desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource"
+ schema = {
+ "type" => "object",
+ "#MU_REFERENCE" => true,
+ "minProperties" => 1,
+ "description" => desc,
+ "properties" => {
+ "id" => {
+ "type" => "string",
+ "description" => "Cloud identifier of a resource we want to reference, typically used when leveraging resources not managed by MU"
+ },
+ "name" => {
+ "type" => "string",
+ "description" => "The short (internal Mu) name of a resource we're attempting to reference. Typically used when referring to a sibling resource elsewhere in the same deploy, or in another known Mu deploy in conjunction with +deploy_id+."
+ },
+ "type" => {
+ "type" => "string",
+ "description" => "The resource type we're attempting to reference.",
+ "enum" => MU::Cloud.resource_types.values.map { |t| t[:cfg_plural] }
+ },
+ "deploy_id" => {
+ "type" => "string",
+ "description" => "Our target resource should be found in this Mu deploy."
+ },
+ "credentials" => MU::Config.credentials_primitive,
+ "region" => MU::Config.region_primitive,
+ "cloud" => MU::Config.cloud_primitive,
+ "tag" => {
+ "type" => "object",
+ "description" => "If the target resource supports tagging and our resource implementations +find+ method supports it, we can attempt to locate it by tag.",
+ "properties" => {
+ "key" => {
+ "type" => "string",
+ "description" => "The tag or label key to search against"
+ },
+ "value" => {
+ "type" => "string",
+ "description" => "The tag or label value to match"
+ }
+ }
+ }
+ }
+ }
+ if !["folders", "habitats"].include?(type)
+ schema["properties"]["habitat"] = MU::Config::Habitat.reference
+ end
+
+ if !type.nil?
+ schema["required"] = ["type"]
+ schema["properties"]["type"]["default"] = type
+ schema["properties"]["type"]["enum"] = [type]
+ end
+
+ aliases.each { |a|
+ a.each_pair { |k, v|
+ if schema["properties"][v]
+ schema["properties"][k] = schema["properties"][v].dup
+ schema["properties"][k]["description"] = "Alias for <tt>#{v}</tt>"
+ else
+ MU.log "Reference schema alias #{k} wants to alias #{v}, but no such attribute exists", MU::WARN, details: caller[4]
+ end
+ }
+ }
+
+ schema
+ end
+
+ # Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment,
+ # of the sort that would have been used to declare this reference in the
+ # first place.
+ def to_h
+ me = { }
+
+ self.instance_variables.each { |var|
+ next if [:@obj, :@mommacat, :@tag_key, :@tag_value].include?(var)
+ val = self.instance_variable_get(var)
+ next if val.nil?
+ val = val.to_h if val.is_a?(MU::Config::Ref)
+ me[var.to_s.sub(/^@/, '')] = val
+ }
+ if @tag_key and !@tag_key.empty?
+ me['tag'] = {
+ 'key' => @tag_key,
+ 'value' => @tag_value
+ }
+ end
+ me
+ end
+
+ # Getter for the #{id} instance variable that attempts to populate it if
+ # it's not set.
+ # @return [String,nil]
+ def id
+ return @id if @id
+ kitten # if it's not defined, attempt to define it
+ @id
+ end
+
+ # Alias for {id}
+ # @return [String,nil]
+ def cloud_id
+ id
+ end
+
+ # Return a {MU::Cloud} object for this reference. This is only meant to be
+ # called in a live deploy, which is to say that if called during initial
+ # configuration parsing, results may be incorrect.
+ # @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches.
+ def kitten(mommacat = @mommacat)
+ return nil if !@cloud or !@type
+
+ if @obj
+ @deploy_id ||= @obj.deploy_id
+ @id ||= @obj.cloud_id
+ @name ||= @obj.config['name']
+ return @obj
+ end
+
+ if mommacat
+ @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: false)
+ if @obj # initialize missing attributes, if we can
+ @id ||= @obj.cloud_id
+ @mommacat ||= mommacat
+ @obj.intoDeploy(@mommacat) # make real sure these are set
+ @deploy_id ||= mommacat.deploy_id
+ if !@name
+ if @obj.config and @obj.config['name']
+ @name = @obj.config['name']
+ elsif @obj.mu_name
+if @type == "folders"
+MU.log "would assign name '#{@obj.mu_name}' in ref to this folder if I were feeling aggressive", MU::WARN, details: self.to_h
+end
+# @name = @obj.mu_name
+ end
+ end
+ return @obj
+ else
+# MU.log "Failed to find a live '#{@type.to_s}' object named #{@name}#{@id ? " (#{@id})" : "" }#{ @habitat ? " in habitat #{@habitat}" : "" }", MU::WARN, details: self
+ end
+ end
+
+ if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud::Google::User.cannedServiceAcctName?(@id))
+
+ begin
+ hab_arg = if @habitat.nil?
+ [nil]
+ elsif @habitat.is_a?(MU::Config::Ref)
+ [@habitat.id]
+ elsif @habitat.is_a?(Hash)
+ [@habitat["id"]]
+ else
+ [@habitat.to_s]
+ end
+
+ found = MU::MommaCat.findStray(
+ @cloud,
+ @type,
+ name: @name,
+ cloud_id: @id,
+ deploy_id: @deploy_id,
+ region: @region,
+ habitats: hab_arg,
+ credentials: @credentials,
+ dummy_ok: (["habitats", "folders", "users", "groups"].include?(@type))
+ )
+ @obj ||= found.first if found
+ rescue ThreadError => e
+ # Sometimes MommaCat calls us in a potential deadlock situation;
+ # don't be the cause of a fatal error if so, we don't need this
+ # object that badly.
+ raise e if !e.message.match(/recursive locking/)
+rescue SystemExit => e
+# XXX this is temporary, to cope with some debug stuff that's in findStray
+# for the nonce
+return
+ end
+ end
+
+ if @obj
+ @deploy_id ||= @obj.deploy_id
+ @id ||= @obj.cloud_id
+ @name ||= @obj.config['name']
+ end
+
+ @obj
+ end
+
+ end
+
# A wrapper for config leaves that came from ERB parameters instead of raw
# YAML or JSON. Will behave like a string for things that expect that
# sort of thing. Code that needs to know that this leaf was the result of
# a parameter will be able to tell by the object class being something
# other than a plain string, array, or hash.
@@ -322,11 +604,11 @@
def getPrettyName
@prettyname
end
# Walk like a String
def to_s
- @prefix+@value+@suffix
+ @prefix.to_s+@value.to_s+@suffix.to_s
end
# Quack like a String
def to_str
to_s
end
@@ -348,10 +630,15 @@
end
# Check for equality like a String
def ==(o)
(o.class == self.class or o.class == "String") && o.to_s == to_s
end
+ # Concatenate like a string
+ def +(o)
+ return to_s if o.nil?
+ to_s + o.to_s
+ end
# Perform global substitutions like a String
def gsub(*args)
to_s.gsub(*args)
end
end
@@ -435,20 +722,25 @@
def method_missing(var_name)
if @param_pass
"MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
else
tail = getTail(var_name.to_s)
+
if tail.is_a?(Array)
if @param_pass
return tail.map {|f| f.values.first.to_s }.join(",")
else
# Don't try to jam complex types into a string file format, just
# sub them back in later from a placeholder.
return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
end
else
- return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
+ if @param_pass
+ tail.to_s
+ else
+ return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
+ end
end
end
end
# A check for the existence of a user-supplied parameter value that can
@@ -468,17 +760,32 @@
placeholder = code if placeholder.nil?
getTail(var_name, value: placeholder, runtimecode: code)
"MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
end
+ # Make sure our parameter values are all available in the local namespace
+ # that ERB will be using, minus any that conflict with existing variables
+ erb_binding = get_binding
+ @@tails.each_pair { |key, tail|
+ next if !tail.is_a?(MU::Config::Tail) or tail.is_list_element
+ # XXX figure out what to do with lists
+ begin
+ erb_binding.local_variable_set(key.to_sym, tail.to_s)
+ rescue NameError
+ MU.log "Binding #{key} = #{tail.to_s}", MU::DEBUG
+ erb_binding.local_variable_set(key.to_sym, tail.to_s)
+ end
+ }
+
# Figure out what kind of file we're loading. We handle includes
# differently if YAML is involved. These globals get used inside
# templates. They're globals on purpose. Stop whining.
$file_format = MU::Config.guessFormat(path)
$yaml_refs = {}
erb = ERB.new(File.read(path), nil, "<>")
- raw_text = erb.result(get_binding)
+
+ raw_text = erb.result(erb_binding)
raw_json = nil
# If we're working in YAML, do some magic to make includes work better.
yaml_parse_error = nil
if $file_format == :yaml
@@ -531,11 +838,11 @@
# Load, resolve, and validate a configuration file ("Basket of Kittens").
# @param path [String]: The path to the master config file to load. Note that this can include other configuration files via ERB.
# @param skipinitialupdates [Boolean]: Whether to forcibly apply the *skipinitialupdates* flag to nodes created by this configuration.
# @param params [Hash]: Optional name-value parameter pairs, which will be passed to our configuration files as ERB variables.
# @return [Hash]: The complete validated configuration for a deployment.
- def initialize(path, skipinitialupdates = false, params: params = Hash.new, updating: nil)
+ def initialize(path, skipinitialupdates = false, params: {}, updating: nil, default_credentials: nil)
$myPublicIp = MU::Cloud::AWS.getAWSMetaData("public-ipv4")
$myRoot = MU.myRoot
$myRoot.freeze
$myAZ = MU.myAZ.freeze
@@ -549,10 +856,11 @@
@kittencfg_semaphore = Mutex.new
@@config_path = path
@admin_firewall_rules = []
@skipinitialupdates = skipinitialupdates
@updating = updating
+ @default_credentials = default_credentials
ok = true
params.each_pair { |name, value|
begin
raise DeployParamError, "Parameter must be formatted as name=value" if value.nil? or value.empty?
@@ -599,19 +907,23 @@
if param.has_key?("default")
@@parameters[param['name']] = param['default'].nil? ? "" : param['default']
elsif param["required"] or !param.has_key?("required")
MU.log "Required parameter '#{param['name']}' not supplied", MU::ERR
ok = false
+ next
+ else # not required, no default
+ next
end
- if param.has_key?("cloudtype")
- getTail(param['name'], value: @@parameters[param['name']], cloudtype: param["cloudtype"], valid_values: param['valid_values'], description: param['description'], prettyname: param['prettyname'], list_of: param['list_of'])
- else
- getTail(param['name'], value: @@parameters[param['name']], valid_values: param['valid_values'], description: param['description'], prettyname: param['prettyname'], list_of: param['list_of'])
- end
end
+ if param.has_key?("cloudtype")
+ getTail(param['name'], value: @@parameters[param['name']], cloudtype: param["cloudtype"], valid_values: param['valid_values'], description: param['description'], prettyname: param['prettyname'], list_of: param['list_of'])
+ else
+ getTail(param['name'], value: @@parameters[param['name']], valid_values: param['valid_values'], description: param['description'], prettyname: param['prettyname'], list_of: param['list_of'])
+ end
}
end
+
raise ValidationError if !ok
@@parameters.each_pair { |name, val|
next if @@tails.has_key?(name) and @@tails[name].is_a?(MU::Config::Tail) and @@tails[name].pseudo
# Parameters can have limited parameterization of their own
if @@tails[name].to_s.match(/^(.*?)MU::Config.getTail PLACEHOLDER (.+?) REDLOHECALP(.*)/)
@@ -624,17 +936,17 @@
next
end
MU.log "Passing variable '#{name}' into #{path} with value '#{val}'"
}
raise DeployParamError, "One or more invalid parameters specified" if !ok
- $parameters = @@parameters
+ $parameters = @@parameters.dup
$parameters.freeze
tmp_cfg, raw_erb = resolveConfig(path: @@config_path)
# Convert parameter entries that constitute whole config keys into
- # MU::Config::Tail objects.
+ # {MU::Config::Tail} objects.
def resolveTails(tree, indent= "")
if tree.is_a?(Hash)
tree.each_pair { |key, val|
tree[key] = resolveTails(val, indent+" ")
}
@@ -661,23 +973,36 @@
"name" => MU.chef_user == "mu" ? "Mu Administrator" : MU.userName,
"email" => MU.userEmail
}
]
end
- MU::Config.set_defaults(@config, MU::Config.schema)
+
+ @config['credentials'] ||= @default_credentials
+
+ types = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }
+
+ MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }.each { |type|
+ if @config[type]
+ @config[type].each { |k|
+ applyInheritedDefaults(k, type)
+ }
+ end
+ }
+ applySchemaDefaults(@config, MU::Config.schema)
+
validate # individual resources validate when added now, necessary because the schema can change depending on what cloud they're targeting
# XXX but now we're not validating top-level keys, argh
#pp @config
#raise "DERP"
- return @config.freeze
+ @config.freeze
end
# Output the dependencies of this BoK stack as a directed acyclic graph.
# Very useful for debugging.
def visualizeDependencies
# GraphViz won't like MU::Config::Tail, pare down to plain Strings
- config = MU::Config.manxify(Marshal.load(Marshal.dump(@config)))
+ config = MU::Config.stripConfig(@config)
begin
g = GraphViz.new(:G, :type => :digraph)
# Generate a GraphViz node for each resource in this stack
nodes = {}
MU::Cloud.resource_types.each_pair { |classname, attrs|
@@ -779,11 +1104,13 @@
@kittencfg_semaphore.synchronize {
matches = []
shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
if @kittens[cfg_plural]
@kittens[cfg_plural].each { |kitten|
- if kitten['name'] == name.to_s or kitten['virtual_name'] == name.to_s
+ if kitten['name'].to_s == name.to_s or
+ kitten['virtual_name'].to_s == name.to_s or
+ (has_multiple and name.nil?)
if has_multiple
matches << kitten
else
return kitten
end
@@ -855,102 +1182,149 @@
# Insert a resource into the current stack
# @param descriptor [Hash]: The configuration description, as from a Basket of Kittens
# @param type [String]: The type of resource being added
# @param delay_validation [Boolean]: Whether to hold off on calling the resource's validateConfig method
- def insertKitten(descriptor, type, delay_validation = false)
+ # @param ignore_duplicates [Boolean]: Do not raise an exception if we attempt to insert a resource with a +name+ field that's already in use
+ def insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false)
append = false
+ start = Time.now
+ shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
+ if !ignore_duplicates and haveLitterMate?(descriptor['name'], cfg_name)
+# raise DuplicateNameError, "A #{shortclass} named #{descriptor['name']} has already been inserted into this configuration"
+ end
+
@kittencfg_semaphore.synchronize {
- append = !@kittens[type].include?(descriptor)
+ append = !@kittens[cfg_plural].include?(descriptor)
# Skip if this kitten has already been validated and appended
if !append and descriptor["#MU_VALIDATED"]
return true
end
}
ok = true
- shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
descriptor["#MU_CLOUDCLASS"] = classname
- inheritDefaults(descriptor, cfg_plural)
+
+ applyInheritedDefaults(descriptor, cfg_plural)
+
+ # Meld defaults from our global schema and, if applicable, from our
+ # cloud-specific schema.
schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass)
+ myschema = Marshal.load(Marshal.dump(MU::Config.schema["properties"][cfg_plural]["items"]))
+ more_required, more_schema = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"]).const_get(shortclass.to_s).schema(self)
+ if more_schema
+ MU::Config.schemaMerge(myschema["properties"], more_schema, descriptor["cloud"])
+ end
+ myschema["required"] ||= []
+ if more_required
+ myschema["required"].concat(more_required)
+ myschema["required"].uniq!
+ end
+ descriptor = applySchemaDefaults(descriptor, myschema, type: shortclass)
+ MU.log "Schema check on #{descriptor['cloud']} #{cfg_name} #{descriptor['name']}", MU::DEBUG, details: myschema
+
if (descriptor["region"] and descriptor["region"].empty?) or
(descriptor['cloud'] == "Google" and ["firewall_rule", "vpc"].include?(cfg_name))
descriptor.delete("region")
end
# Make sure a sensible region has been targeted, if applicable
+ classobj = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"])
if descriptor["region"]
- classobj = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"])
valid_regions = classobj.listRegions
if !valid_regions.include?(descriptor["region"])
MU.log "Known regions for cloud '#{descriptor['cloud']}' do not include '#{descriptor["region"]}'", MU::ERR, details: valid_regions
ok = false
end
end
- if descriptor['project']
- if haveLitterMate?(descriptor['project'], "habitats")
+ if descriptor.has_key?('project')
+ if descriptor['project'].nil?
+ descriptor.delete('project')
+ elsif haveLitterMate?(descriptor['project'], "habitats")
descriptor['dependencies'] ||= []
descriptor['dependencies'] << {
"type" => "habitat",
"name" => descriptor['project']
}
end
end
# Does this resource go in a VPC?
if !descriptor["vpc"].nil? and !delay_validation
+ # Quietly fix old vpc reference style
+ if descriptor['vpc']['vpc_id']
+ descriptor['vpc']['id'] ||= descriptor['vpc']['vpc_id']
+ descriptor['vpc'].delete('vpc_id')
+ end
+ if descriptor['vpc']['vpc_name']
+ descriptor['vpc']['name'] = descriptor['vpc']['vpc_name']
+ descriptor['vpc'].delete('vpc_name')
+ end
+
descriptor['vpc']['cloud'] = descriptor['cloud']
if descriptor['credentials']
descriptor['vpc']['credentials'] ||= descriptor['credentials']
end
if descriptor['vpc']['region'].nil? and !descriptor['region'].nil? and !descriptor['region'].empty? and descriptor['vpc']['cloud'] != "Google"
descriptor['vpc']['region'] = descriptor['region']
end
# If we're using a VPC in this deploy, set it as a dependency
- if !descriptor["vpc"]["vpc_name"].nil? and
- haveLitterMate?(descriptor["vpc"]["vpc_name"], "vpcs") and
+ if !descriptor["vpc"]["name"].nil? and
+ haveLitterMate?(descriptor["vpc"]["name"], "vpcs") and
descriptor["vpc"]['deploy_id'].nil? and
- descriptor["vpc"]['vpc_id'].nil?
+ descriptor["vpc"]['id'].nil?
descriptor["dependencies"] << {
"type" => "vpc",
- "name" => descriptor["vpc"]["vpc_name"]
+ "name" => descriptor["vpc"]["name"],
}
+ siblingvpc = haveLitterMate?(descriptor["vpc"]["name"], "vpcs")
- siblingvpc = haveLitterMate?(descriptor["vpc"]["vpc_name"], "vpcs")
+ if siblingvpc and siblingvpc['bastion'] and
+ ["server", "server_pool"].include?(cfg_name) and
+ !descriptor['bastion']
+ if descriptor['name'] != siblingvpc['bastion'].to_h['name']
+ descriptor["dependencies"] << {
+ "type" => "server",
+ "name" => siblingvpc['bastion'].to_h['name']
+ }
+ end
+ end
+
# things that live in subnets need their VPCs to be fully
# resolved before we can proceed
if ["server", "server_pool", "loadbalancer", "database", "cache_cluster", "container_cluster", "storage_pool"].include?(cfg_name)
if !siblingvpc["#MU_VALIDATED"]
ok = false if !insertKitten(siblingvpc, "vpcs")
end
end
if !MU::Config::VPC.processReference(descriptor['vpc'],
cfg_plural,
- shortclass.to_s+" '#{descriptor['name']}'",
+ descriptor,
self,
dflt_region: descriptor['region'],
- is_sibling: true,
credentials: descriptor['credentials'],
+ dflt_project: descriptor['project'],
sibling_vpcs: @kittens['vpcs'])
ok = false
end
# If we're using a VPC from somewhere else, make sure the flippin'
# thing exists, and also fetch its id now so later search routines
# don't have to work so hard.
else
- if !MU::Config::VPC.processReference(descriptor["vpc"], cfg_plural,
- "#{shortclass} #{descriptor['name']}",
+ if !MU::Config::VPC.processReference(descriptor["vpc"],
+ cfg_plural,
+ descriptor,
self,
credentials: descriptor['credentials'],
+ dflt_project: descriptor['project'],
dflt_region: descriptor['region'])
- MU.log "insertKitten was called from #{caller[0]}", MU::ERR
ok = false
end
end
# if we didn't specify credentials but can inherit some from our target
@@ -977,25 +1351,36 @@
if !haveLitterMate?(fwname, "firewall_rules") and
(descriptor['ingress_rules'] or
["server", "server_pool", "database"].include?(cfg_name))
descriptor['ingress_rules'] ||= []
+ fw_classobj = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"]).const_get("FirewallRule")
acl = {
"name" => fwname,
"rules" => descriptor['ingress_rules'],
"region" => descriptor['region'],
"credentials" => descriptor["credentials"]
}
- acl["vpc"] = descriptor['vpc'].dup if descriptor['vpc']
+ if !fw_classobj.isGlobal?
+ acl['region'] = descriptor['region']
+ acl['region'] ||= classobj.myRegion(acl['credentials'])
+ else
+ acl.delete("region")
+ end
+ if descriptor["vpc"]
+ acl["vpc"] = descriptor['vpc'].dup
+ acl["vpc"].delete("subnet_pref")
+ end
+
["optional_tags", "tags", "cloud", "project"].each { |param|
acl[param] = descriptor[param] if descriptor[param]
}
descriptor["add_firewall_rules"] = [] if descriptor["add_firewall_rules"].nil?
- descriptor["add_firewall_rules"] << {"rule_name" => fwname}
+ descriptor["add_firewall_rules"] << {"rule_name" => fwname, "type" => "firewall_rules" } # XXX why the duck is there a type argument required here?
acl = resolveIntraStackFirewallRefs(acl)
- ok = false if !insertKitten(acl, "firewall_rules")
+ ok = false if !insertKitten(acl, "firewall_rules", delay_validation)
end
# Does it declare association with any sibling LoadBalancers?
if !descriptor["loadbalancers"].nil?
descriptor["loadbalancers"].each { |lb|
@@ -1028,11 +1413,11 @@
"type" => "firewall_rule",
"name" => acl_include["rule_name"]
}
siblingfw = haveLitterMate?(acl_include["rule_name"], "firewall_rules")
if !siblingfw["#MU_VALIDATED"]
- ok = false if !insertKitten(siblingfw, "firewall_rules")
+ ok = false if !insertKitten(siblingfw, "firewall_rules", delay_validation)
end
elsif acl_include["rule_name"]
MU.log shortclass.to_s+" #{descriptor['name']} depends on FirewallRule #{acl_include["rule_name"]}, but no such rule declared.", MU::ERR
ok = false
end
@@ -1101,25 +1486,13 @@
# Call the generic validation for the resource type, first and foremost
# XXX this might have to be at the top of this insertKitten instead of
# here
ok = false if !schemaclass.validate(descriptor, self)
- # Merge the cloud-specific JSON schema and validate against it
- myschema = Marshal.load(Marshal.dump(MU::Config.schema["properties"][cfg_plural]["items"]))
- more_required, more_schema = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"]).const_get(shortclass.to_s).schema(self)
-
- if more_schema
- MU::Config.schemaMerge(myschema["properties"], more_schema, descriptor["cloud"])
- MU::Config.set_defaults(descriptor, myschema)
- end
- myschema["required"] ||= []
- myschema["required"].concat(more_required)
- myschema["required"].uniq!
- MU.log "Schema check on #{descriptor['cloud']} #{cfg_name} #{descriptor['name']}", MU::DEBUG, details: myschema
-
- plain_cfg = MU::Config.manxify(Marshal.load(Marshal.dump(descriptor)))
+ plain_cfg = MU::Config.stripConfig(descriptor)
plain_cfg.delete("#MU_CLOUDCLASS")
+ plain_cfg.delete("#MU_VALIDATION_ATTEMPTED")
plain_cfg.delete("#TARGETCLASS")
plain_cfg.delete("#TARGETNAME")
plain_cfg.delete("parent_block") if cfg_plural == "vpcs"
begin
JSON::Validator.validate!(myschema, plain_cfg)
@@ -1141,44 +1514,53 @@
# Run the cloud class's deeper validation, unless we've already failed
# on stuff that will cause spurious alarms further in
if ok
parser = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"]).const_get(shortclass.to_s)
- plain_descriptor = MU::Config.manxify(Marshal.load(Marshal.dump(descriptor)))
- passed = parser.validateConfig(plain_descriptor, self)
+ original_descriptor = MU::Config.stripConfig(descriptor)
+ passed = parser.validateConfig(descriptor, self)
- if passed
- descriptor.merge!(plain_descriptor)
- else
+ if !passed
+ descriptor = original_descriptor
ok = false
end
+
+ # Make sure we've been configured with the right credentials
+ cloudbase = Object.const_get("MU").const_get("Cloud").const_get(descriptor['cloud'])
+ credcfg = cloudbase.credConfig(descriptor['credentials'])
+ if !credcfg or credcfg.empty?
+ raise ValidationError, "#{descriptor['cloud']} #{cfg_name} #{descriptor['name']} declares credential set #{descriptor['credentials']}, but no such credentials exist for that cloud provider"
+ end
+
descriptor['#MU_VALIDATED'] = true
end
-
end
descriptor["dependencies"].uniq!
@kittencfg_semaphore.synchronize {
@kittens[cfg_plural] << descriptor if append
}
+
ok
end
@@allregions = []
- MU::Cloud.supportedClouds.each { |cloud|
+ MU::Cloud.availableClouds.each { |cloud|
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
- @@allregions.concat(cloudclass.listRegions())
+ regions = cloudclass.listRegions()
+ @@allregions.concat(regions) if regions
}
# Configuration chunk for choosing a provider region
# @return [Hash]
def self.region_primitive
if !@@allregions or @@allregions.empty?
@@allregions = []
- MU::Cloud.supportedClouds.each { |cloud|
+ MU::Cloud.availableClouds.each { |cloud|
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+ return @allregions if !cloudclass.listRegions()
@@allregions.concat(cloudclass.listRegions())
}
end
{
"type" => "string",
@@ -1234,11 +1616,11 @@
# Configuration chunk for choosing a cloud provider
# @return [Hash]
def self.cloud_primitive
{
"type" => "string",
- "default" => MU::Config.defaultCloud,
+# "default" => MU::Config.defaultCloud, # applyInheritedDefaults does this better
"enum" => MU::Cloud.supportedClouds
}
end
# Generate configuration for the general-pursose ADMIN firewall rulesets
@@ -1248,42 +1630,21 @@
# @param vpc [Hash]: A VPC reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC).
# @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource.
# @param cloud [String]: The parent resource's cloud plugin identifier
# @param region [String]: Cloud provider region, if applicable.
# @return [Hash<String>]: A dependency description that the calling resource can then add to itself.
- def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil)
+ def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false)
if !cloud or (cloud == "AWS" and !region)
raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider"
end
hosts = Array.new
hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip
hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip
hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip
hosts << "#{admin_ip}/32" if admin_ip
hosts.uniq!
- name = "admin"
- name += credentials.to_s if credentials
- realvpc = nil
- if vpc
- realvpc = {}
- realvpc['vpc_id'] = vpc['vpc_id'] if !vpc['vpc_id'].nil?
- realvpc['vpc_name'] = vpc['vpc_name'] if !vpc['vpc_name'].nil?
- realvpc['deploy_id'] = vpc['deploy_id'] if !vpc['deploy_id'].nil?
- if !realvpc['vpc_id'].nil? and !realvpc['vpc_id'].empty?
- # Stupid kludge for Google cloud_ids which are sometimes URLs and
- # sometimes not. Requirements are inconsistent from scenario to
- # scenario.
- name = name + "-" + realvpc['vpc_id'].gsub(/.*\//, "")
- realvpc['vpc_id'] = getTail("vpc_id", value: realvpc['vpc_id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC", cloudtype: "AWS::EC2::VPC::Id") if realvpc["vpc_id"].is_a?(String)
- elsif !realvpc['vpc_name'].nil?
- name = name + "-" + realvpc['vpc_name']
- end
- end
-
- hosts.uniq!
-
rules = []
if cloud == "Google"
rules = [
{ "ingress" => true, "proto" => "all", "hosts" => hosts },
{ "egress" => true, "proto" => "all", "hosts" => hosts }
@@ -1294,13 +1655,47 @@
{ "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts },
{ "proto" => "icmp", "port_range" => "-1", "hosts" => hosts }
]
end
+ resclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("FirewallRule")
+
+ if rules_only
+ return rules
+ end
+
+ name = "admin"
+ name += credentials.to_s if credentials
+ realvpc = nil
+ if vpc
+ realvpc = {}
+ ['vpc_name', 'vpc_id'].each { |p|
+ if vpc[p]
+ vpc[p.sub(/^vpc_/, '')] = vpc[p]
+ vpc.delete(p)
+ end
+ }
+ ['cloud', 'id', 'name', 'deploy_id', 'habitat', 'credentials'].each { |field|
+ realvpc[field] = vpc[field] if !vpc[field].nil?
+ }
+ if !realvpc['id'].nil? and !realvpc['id'].empty?
+ # Stupid kludge for Google cloud_ids which are sometimes URLs and
+ # sometimes not. Requirements are inconsistent from scenario to
+ # scenario.
+ name = name + "-" + realvpc['id'].gsub(/.*\//, "")
+ realvpc['id'] = getTail("id", value: realvpc['id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC", cloudtype: "AWS::EC2::VPC::Id") if realvpc["id"].is_a?(String)
+ elsif !realvpc['name'].nil?
+ name = name + "-" + realvpc['name']
+ end
+ end
+
+
acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials }
acl.delete("vpc") if !acl["vpc"]
- acl["region"] = region if !region.nil? and !region.empty?
+ if !resclass.isGlobal? and !region.nil? and !region.empty?
+ acl["region"] = region
+ end
@admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl)
return {"type" => "firewall_rule", "name" => name}
end
private
@@ -1474,88 +1869,110 @@
# Namespace magic to pass to ERB's result method.
def get_binding
binding
end
- def self.set_defaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil)
+ def applySchemaDefaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil, type: nil)
return if schema_chunk.nil?
if conf_chunk != nil and schema_chunk["properties"].kind_of?(Hash) and conf_chunk.is_a?(Hash)
+
if schema_chunk["properties"]["creation_style"].nil? or
schema_chunk["properties"]["creation_style"] != "existing"
schema_chunk["properties"].each_pair { |key, subschema|
- new_val = self.set_defaults(conf_chunk[key], subschema, depth+1, conf_chunk)
- conf_chunk[key] = new_val if new_val != nil
+ shortclass = if conf_chunk[key]
+ shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(key)
+ shortclass
+ else
+ nil
+ end
+
+ new_val = applySchemaDefaults(conf_chunk[key], subschema, depth+1, conf_chunk, type: shortclass).dup
+
+ conf_chunk[key] = Marshal.load(Marshal.dump(new_val)) if !new_val.nil?
}
end
elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array)
conf_chunk.map! { |item|
- self.set_defaults(item, schema_chunk["items"], depth+1, conf_chunk)
+ # If we're working on a resource type, go get implementation-specific
+ # schema information so that we set those defaults correctly.
+ realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"]
+
+ cloudclass = Object.const_get("MU").const_get("Cloud").const_get(item["cloud"]).const_get(type)
+ toplevel_required, cloudschema = cloudclass.schema(self)
+
+ newschema = schema_chunk["items"].dup
+ newschema["properties"].merge!(cloudschema)
+ newschema
+ else
+ schema_chunk["items"].dup
+ end
+
+ applySchemaDefaults(item, realschema, depth+1, conf_chunk, type: type).dup
}
else
if conf_chunk.nil? and !schema_chunk["default_if"].nil? and !siblings.nil?
schema_chunk["default_if"].each { |cond|
if siblings[cond["key_is"]] == cond["value_is"]
- return cond["set"]
+ return Marshal.load(Marshal.dump(cond["set"]))
end
}
end
if conf_chunk.nil? and schema_chunk["default"] != nil
- return schema_chunk["default"].dup
+ return Marshal.load(Marshal.dump(schema_chunk["default"]))
end
end
+
return conf_chunk
end
# For our resources which specify intra-stack dependencies, make sure those
# dependencies are actually declared.
# TODO check for loops
def self.check_dependencies(config)
ok = true
- config.each { |type|
- if type.instance_of?(Array)
- type.each { |container|
- if container.instance_of?(Array)
- container.each { |resource|
- if resource.kind_of?(Hash) and resource["dependencies"] != nil
- append = []
- delete = []
- resource["dependencies"].each { |dependency|
- collection = dependency["type"]+"s"
- found = false
- names_seen = []
- if config[collection] != nil
- config[collection].each { |service|
- names_seen << service["name"].to_s
- found = true if service["name"].to_s == dependency["name"].to_s
- if service["virtual_name"]
- names_seen << service["virtual_name"].to_s
- found = true if service["virtual_name"].to_s == dependency["name"].to_s
- append_me = dependency.dup
- append_me['name'] = service['name']
- append << append_me
- delete << dependency
- end
- }
+ config.each_pair { |type, values|
+ if values.instance_of?(Array)
+ values.each { |resource|
+ if resource.kind_of?(Hash) and !resource["dependencies"].nil?
+ append = []
+ delete = []
+ resource["dependencies"].each { |dependency|
+ shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(dependency["type"])
+ found = false
+ names_seen = []
+ if !config[cfg_plural].nil?
+ config[cfg_plural].each { |service|
+ names_seen << service["name"].to_s
+ found = true if service["name"].to_s == dependency["name"].to_s
+ if service["virtual_name"]
+ names_seen << service["virtual_name"].to_s
+ if service["virtual_name"].to_s == dependency["name"].to_s
+ found = true
+ append_me = dependency.dup
+ append_me['name'] = service['name']
+ append << append_me
+ delete << dependency
+ end
end
- if !found
- MU.log "Missing dependency: #{type[0]}{#{resource['name']}} needs #{collection}{#{dependency['name']}}", MU::ERR, details: names_seen
- ok = false
- end
}
- if append.size > 0
- append.uniq!
- resource["dependencies"].concat(append)
- end
- if delete.size > 0
- delete.each { |delete_me|
- resource["dependencies"].delete(delete_me)
- }
- end
end
+ if !found
+ MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR, details: names_seen
+ ok = false
+ end
}
+ if append.size > 0
+ append.uniq!
+ resource["dependencies"].concat(append)
+ end
+ if delete.size > 0
+ delete.each { |delete_me|
+ resource["dependencies"].delete(delete_me)
+ }
+ end
end
}
end
}
return ok
@@ -1620,51 +2037,61 @@
return ok
end
# Given a bare hash describing a resource, insert default values which can
- # be inherited from the current live parent configuration.
+ # be inherited from its parent or from the root of the BoK.
# @param kitten [Hash]: A resource descriptor
# @param type [String]: The type of resource this is ("servers" etc)
- def inheritDefaults(kitten, type)
+ def applyInheritedDefaults(kitten, type)
+ kitten['cloud'] ||= @config['cloud']
kitten['cloud'] ||= MU::Config.defaultCloud
+
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud'])
shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass)
- schema_fields = ["us_only", "scrub_mu_isms", "credentials"]
+ schema_fields = ["us_only", "scrub_mu_isms", "credentials", "billing_acct"]
if !resclass.isGlobal?
+ kitten['region'] ||= @config['region']
+ kitten['region'] ||= cloudclass.myRegion(kitten['credentials'])
schema_fields << "region"
end
+ kitten['credentials'] ||= @config['credentials']
+ kitten['credentials'] ||= cloudclass.credConfig(name_only: true)
+
+ kitten['us_only'] ||= @config['us_only']
+ kitten['us_only'] ||= false
+
+ kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms']
+ kitten['scrub_mu_isms'] ||= false
+
if kitten['cloud'] == "Google"
- kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials'])
- schema_fields << "project"
+# TODO this should be cloud-generic (handle AWS accounts, Azure subscriptions)
+ if resclass.canLiveIn.include?(:Habitat)
+ kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials'])
+ schema_fields << "project"
+ end
if kitten['region'].nil? and !kitten['#MU_CLOUDCLASS'].nil? and
!resclass.isGlobal? and
![MU::Cloud::VPC, MU::Cloud::FirewallRule].include?(kitten['#MU_CLOUDCLASS'])
if MU::Cloud::Google.myRegion((kitten['credentials'])).nil?
raise ValidationError, "Google '#{type}' resource '#{kitten['name']}' declared without a region, but no default Google region declared in mu.yaml under #{kitten['credentials'].nil? ? "default" : kitten['credentials']} credential set"
end
kitten['region'] ||= MU::Cloud::Google.myRegion
end
- elsif kitten["cloud"] == "AWS" and !resclass.isGlobal?
+ elsif kitten["cloud"] == "AWS" and !resclass.isGlobal? and !kitten['region']
if MU::Cloud::AWS.myRegion.nil?
raise ValidationError, "AWS resource declared without a region, but no default AWS region found"
end
kitten['region'] ||= MU::Cloud::AWS.myRegion
end
- kitten['us_only'] ||= @config['us_only']
- kitten['us_only'] ||= false
- kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms']
- kitten['scrub_mu_isms'] ||= false
+ kitten['billing_acct'] ||= @config['billing_acct'] if @config['billing_acct']
- kitten['credentials'] ||= @config['credentials']
- kitten['credentials'] ||= cloudclass.credConfig(name_only: true)
-
kitten["dependencies"] ||= []
# Make sure the schema knows about these "new" fields, so that validation
# doesn't trip over them.
schema_fields.each { |field|
@@ -1675,21 +2102,20 @@
}
end
def validate(config = @config)
ok = true
- plain_cfg = MU::Config.manxify(Marshal.load(Marshal.dump(config)))
count = 0
@kittens ||= {}
types = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }
types.each { |type|
@kittens[type] = config[type]
@kittens[type] ||= []
@kittens[type].each { |k|
- inheritDefaults(k, type)
+ applyInheritedDefaults(k, type)
}
count = count + @kittens[type].size
}
if count == 0
@@ -1706,20 +2132,27 @@
@kittens["firewall_rules"].each { |acl|
acl = resolveIntraStackFirewallRefs(acl)
}
+ # VPCs do complex things in their cloud-layer validation that other
+ # resources tend to need, like subnet allocation, so hit them early.
+ @kittens["vpcs"].each { |vpc|
+ ok = false if !insertKitten(vpc, "vpcs")
+ }
+
# Make sure validation has been called for all on-the-fly generated
# resources.
validated_something_new = false
begin
validated_something_new = false
types.each { |type|
@kittens[type].each { |descriptor|
- if !descriptor["#MU_VALIDATED"]
+ if !descriptor["#MU_VALIDATION_ATTEMPTED"]
validated_something_new = true
ok = false if !insertKitten(descriptor, type)
+ descriptor["#MU_VALIDATION_ATTEMPTED"] = true
end
}
}
end while validated_something_new
@@ -1916,11 +2349,10 @@
docstring = docstring + "attr_accessor :#{name}"
return docstring
end
- return nil
end
def self.dependencies_primitive
{
"type" => "array",
@@ -1967,10 +2399,61 @@
else
MU.userEmail
end
end
+ # Load and validate the schema for an individual resource class, optionally
+ # merging cloud-specific schema components.
+ # @param type [String]: The resource type to load
+ # @param cloud [String]: A specific cloud, whose implementation's schema of this resource we will merge
+ # @return [Hash]
+ def self.loadResourceSchema(type, cloud: nil)
+ valid = true
+ shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
+ schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass)
+
+ [:schema, :validate].each { |method|
+ if !schemaclass.respond_to?(method)
+ MU.log "MU::Config::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
+ return [nil, false] if method == :schema
+ valid = false
+ end
+ }
+
+ schema = schemaclass.schema.dup
+
+ schema["properties"]["virtual_name"] = {
+ "description" => "Internal use.",
+ "type" => "string"
+ }
+ schema["properties"]["dependencies"] = MU::Config.dependencies_primitive
+ schema["properties"]["cloud"] = MU::Config.cloud_primitive
+ schema["properties"]["credentials"] = MU::Config.credentials_primitive
+ schema["title"] = type.to_s
+
+ if cloud
+ cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(shortclass)
+
+ if cloudclass.respond_to?(:schema)
+ reqd, cloudschema = cloudclass.schema
+ cloudschema.each { |key, cfg|
+ if schema["properties"][key]
+ schemaMerge(schema["properties"][key], cfg, cloud)
+ else
+ schema["properties"][key] = cfg.dup
+ end
+ }
+ else
+ MU.log "MU::Cloud::#{cloud}::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
+ valid = false
+ end
+
+ end
+
+ return [schema, valid]
+ end
+
@@schema = {
"$schema" => "http://json-schema.org/draft-04/schema#",
"title" => "MU Application",
"type" => "object",
"description" => "A MU application stack, consisting of at least one resource.",
@@ -1984,12 +2467,16 @@
"type" => "boolean",
"description" => "When 'cloud' is set to 'CloudFormation,' use this flag to strip out Mu-specific artifacts (tags, standard userdata, naming conventions, etc) to yield a clean, source-agnostic template. Setting this flag here will override declarations in individual resources."
},
"project" => {
"type" => "string",
- "description" => "GOOGLE: The project into which to deploy resources"
+ "description" => "**GOOGLE ONLY**: The project into which to deploy resources"
},
+ "billing_acct" => {
+ "type" => "string",
+ "description" => "**GOOGLE ONLY**: Billing account ID to associate with a newly-created Google Project. If not specified, will attempt to locate a billing account associated with the default project for our credentials.",
+ },
"region" => MU::Config.region_primitive,
"credentials" => MU::Config.credentials_primitive,
"us_only" => {
"type" => "boolean",
"description" => "For resources which span regions, restrict to regions inside the United States",
@@ -2082,31 +2569,19 @@
failed << type
next
end
}
+
MU::Cloud.resource_types.each_pair { |type, cfg|
begin
- schemaclass = Object.const_get("MU").const_get("Config").const_get(type)
- [:schema, :validate].each { |method|
- if !schemaclass.respond_to?(method)
- MU.log "MU::Config::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
- failed << type
- end
- }
+ schema, valid = loadResourceSchema(type)
+ failed << type if !valid
next if failed.include?(type)
@@schema["properties"][cfg[:cfg_plural]] = {
"type" => "array",
- "items" => schemaclass.schema
+ "items" => schema
}
- @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["virtual_name"] = {
- "description" => "Internal use.",
- "type" => "string"
- }
- @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["dependencies"] = MU::Config.dependencies_primitive
- @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["cloud"] = MU::Config.cloud_primitive
- @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["credentials"] = MU::Config.credentials_primitive
- @@schema["properties"][cfg[:cfg_plural]]["items"]["title"] = type.to_s
rescue NameError => e
failed << type
MU.log "Error loading #{type} schema from mu/config/#{cfg[:cfg_name]}", MU::ERR, details: "\t"+e.inspect+"\n\t"+e.backtrace[0]
end
}