modules/mu/config.rb in cloud-mu-1.9.0.pre.beta vs modules/mu/config.rb in cloud-mu-2.0.0.pre.alpha

- old
+ new

@@ -37,13 +37,14 @@ begin MU.myCloud rescue NoMethodError "AWS" end - if MU::Cloud::Google.hosted +# XXX this can be more generic (loop through supportedClouds and try this) + if MU::Cloud::Google.hosted? "Google" - elsif MU::Cloud::AWS.hosted + elsif MU::Cloud::AWS.hosted? "AWS" end end # The default grooming agent for new resources. Must exist in MU.supportedGroomers. @@ -763,34 +764,49 @@ # See if a given resource is configured in the current stack # @param name [String]: The name of the resource being checked # @param type [String]: The type of resource being checked # @return [Boolean] - def haveLitterMate?(name, type) + def haveLitterMate?(name, type, has_multiple: false) @kittencfg_semaphore.synchronize { + matches = [] shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) - @kittens[cfg_plural].each { |kitten| - return kitten if kitten['name'] == name.to_s - } + if @kittens[cfg_plural] + @kittens[cfg_plural].each { |kitten| + if kitten['name'] == name.to_s or kitten['virtual_name'] == name.to_s + if has_multiple + matches << kitten + else + return kitten + end + end + } + end + if has_multiple + return matches + else + return false + end } - false end # Remove a resource from the current stack # @param name [String]: The name of the resource being removed # @param type [String]: The type of resource being removed def removeKitten(name, type) @kittencfg_semaphore.synchronize { shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) deletia = nil - @kittens[cfg_plural].each { |kitten| - if kitten['name'] == name - deletia = kitten - break - end - } - @kittens[type].delete(deletia) if !deletia.nil? + if @kittens[cfg_plural] + @kittens[cfg_plural].each { |kitten| + if kitten['name'] == name + deletia = kitten + break + end + } + @kittens[type].delete(deletia) if !deletia.nil? + end } end # FirewallRules can reference other FirewallRules, which means we need to do # an extra pass to make sure we get all intra-stack dependencies correct. @@ -866,10 +882,13 @@ end # Does this resource go in a VPC? if !descriptor["vpc"].nil? and !delay_validation descriptor['vpc']['cloud'] = descriptor['cloud'] + if descriptor['credentials'] + descriptor['vpc']['credentials'] ||= descriptor['credentials'] + end if descriptor['vpc']['region'].nil? and !descriptor['region'].nil? and !descriptor['region'].empty? and descriptor['vpc']['cloud'] != "Google" descriptor['vpc']['region'] = descriptor['region'] end # If we're using a VPC in this deploy, set it as a dependency @@ -894,10 +913,11 @@ cfg_plural, shortclass.to_s+" '#{descriptor['name']}'", self, dflt_region: descriptor['region'], is_sibling: true, + credentials: descriptor['credentials'], sibling_vpcs: @kittens['vpcs']) ok = false end # If we're using a VPC from somewhere else, make sure the flippin' @@ -905,15 +925,23 @@ # don't have to work so hard. else if !MU::Config::VPC.processReference(descriptor["vpc"], cfg_plural, "#{shortclass} #{descriptor['name']}", self, + credentials: descriptor['credentials'], dflt_region: descriptor['region']) MU.log "insertKitten was called from #{caller[0]}", MU::ERR ok = false end end + + # if we didn't specify credentials but can inherit some from our target + # VPC, do so + if descriptor["vpc"]["credentials"] + descriptor["credentials"] ||= descriptor["vpc"]["credentials"] + end + # Clean crud out of auto-created VPC declarations so they don't trip # the schema validator when it's invoked later. if !["server", "server_pool", "database"].include?(cfg_name) descriptor['vpc'].delete("nat_ssh_user") end @@ -931,11 +959,16 @@ if !haveLitterMate?(fwname, "firewall_rules") and (descriptor['ingress_rules'] or ["server", "server_pool", "database"].include?(cfg_name)) descriptor['ingress_rules'] ||= [] - acl = {"name" => fwname, "rules" => descriptor['ingress_rules'], "region" => descriptor['region'] } + acl = { + "name" => fwname, + "rules" => descriptor['ingress_rules'], + "region" => descriptor['region'], + "credentials" => descriptor["credentials"] + } acl["vpc"] = descriptor['vpc'].dup if descriptor['vpc'] ["optional_tags", "tags", "cloud", "project"].each { |param| acl[param] = descriptor[param] if descriptor[param] } descriptor["add_firewall_rules"] = [] if descriptor["add_firewall_rules"].nil? @@ -990,10 +1023,11 @@ # Does it declare some alarms? if descriptor["alarms"] && !descriptor["alarms"].empty? descriptor["alarms"].each { |alarm| alarm["name"] = "#{cfg_name}-#{descriptor["name"]}-#{alarm["name"]}" alarm['dimensions'] = [] if !alarm['dimensions'] + alarm["credentials"] = descriptor["credentials"] alarm["#TARGETCLASS"] = cfg_name alarm["#TARGETNAME"] = descriptor['name'] alarm['cloud'] = descriptor['cloud'] ok = false if !insertKitten(alarm, "alarms", true) @@ -1089,13 +1123,17 @@ # Run the cloud class's deeper validation, unless we've already failed # on stuff that will cause spurious alarms further in if ok parser = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"]).const_get(shortclass.to_s) plain_descriptor = MU::Config.manxify(Marshal.load(Marshal.dump(descriptor))) - return false if !parser.validateConfig(plain_descriptor, self) + passed = parser.validateConfig(plain_descriptor, self) - descriptor.merge!(plain_descriptor) + if passed + descriptor.merge!(plain_descriptor) + else + ok = false + end descriptor['#MU_VALIDATED'] = true end end @@ -1121,10 +1159,19 @@ "type" => "string", "enum" => allregions } end + # Configuration chunk for choosing a set of cloud credentials + # @return [Hash] + def self.credentials_primitive + { + "type" => "string", + "description" => "Specify a non-default set of credentials to use when authenticating to cloud provider APIs, as listed in `mu.yaml` under each provider's subsection. If " + } + end + # Configuration chunk for creating resource tags as an array of key/value # pairs. # @return [Hash] def self.optional_tags_primitive { @@ -1176,21 +1223,22 @@ # @param vpc [Hash]: A VPC reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC). # @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource. # @param cloud [String]: The parent resource's cloud plugin identifier # @param region [String]: Cloud provider region, if applicable. # @return [Hash<String>]: A dependency description that the calling resource can then add to itself. - def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil) + def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil) if !cloud or (cloud == "AWS" and !region) raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider" end hosts = Array.new hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip hosts << "#{admin_ip}/32" if admin_ip hosts.uniq! name = "admin" + name += credentials.to_s if credentials realvpc = nil if vpc realvpc = {} realvpc['vpc_id'] = vpc['vpc_id'] if !vpc['vpc_id'].nil? @@ -1221,13 +1269,13 @@ { "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts }, { "proto" => "icmp", "port_range" => "-1", "hosts" => hosts } ] end - acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true} + acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials } acl.delete("vpc") if !acl["vpc"] - acl["region"] == region if !region.nil? and !region.empty? + acl["region"] = region if !region.nil? and !region.empty? @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl) return {"type" => "firewall_rule", "name" => name} end private @@ -1436,31 +1484,51 @@ # For our resources which specify intra-stack dependencies, make sure those # dependencies are actually declared. # TODO check for loops def self.check_dependencies(config) ok = true + config.each { |type| if type.instance_of?(Array) type.each { |container| if container.instance_of?(Array) container.each { |resource| if resource.kind_of?(Hash) and resource["dependencies"] != nil + append = [] + delete = [] resource["dependencies"].each { |dependency| collection = dependency["type"]+"s" found = false names_seen = [] if config[collection] != nil config[collection].each { |service| names_seen << service["name"].to_s found = true if service["name"].to_s == dependency["name"].to_s + if service["virtual_name"] + names_seen << service["virtual_name"].to_s + found = true if service["virtual_name"].to_s == dependency["name"].to_s + append_me = dependency.dup + append_me['name'] = service['name'] + append << append_me + delete << dependency + end } end if !found MU.log "Missing dependency: #{type[0]}{#{resource['name']}} needs #{collection}{#{dependency['name']}}", MU::ERR, details: names_seen ok = false end } + if append.size > 0 + append.uniq! + resource["dependencies"].concat(append) + end + if delete.size > 0 + delete.each { |delete_me| + resource["dependencies"].delete(delete_me) + } + end end } end } end @@ -1532,42 +1600,53 @@ # be inherited from the current live parent configuration. # @param kitten [Hash]: A resource descriptor # @param type [String]: The type of resource this is ("servers" etc) def inheritDefaults(kitten, type) kitten['cloud'] ||= MU::Config.defaultCloud + cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']) + shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) + resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass) - schema_fields = ["region", "us_only", "scrub_mu_isms"] + schema_fields = ["us_only", "scrub_mu_isms", "credentials"] + if !resclass.isGlobal? + schema_fields << "region" + end + if kitten['cloud'] == "Google" - kitten["project"] ||= MU::Cloud::Google.defaultProject + kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials']) schema_fields << "project" if kitten['region'].nil? and !kitten['#MU_CLOUDCLASS'].nil? and + !resclass.isGlobal? and ![MU::Cloud::VPC, MU::Cloud::FirewallRule].include?(kitten['#MU_CLOUDCLASS']) - if !$MU_CFG['google'] or !$MU_CFG['google']['region'] - raise ValidationError, "Google resource declared without a region, but no default Google region declared in mu.yaml" + if MU::Cloud::Google.myRegion((kitten['credentials'])).nil? + raise ValidationError, "Google '#{type}' resource '#{kitten['name']}' declared without a region, but no default Google region declared in mu.yaml under #{kitten['credentials'].nil? ? "default" : kitten['credentials']} credential set" end - kitten['region'] ||= $MU_CFG['google']['region'] + kitten['region'] ||= MU::Cloud::Google.myRegion(kitten['credentials']) end - else - if !$MU_CFG['aws'] or !$MU_CFG['aws']['region'] - raise ValidationError, "AWS resource declared without a region, but no default AWS region declared in mu.yaml" + elsif !resclass.isGlobal? + if MU::Cloud::AWS.myRegion.nil? + raise ValidationError, "AWS resource declared without a region, but no default AWS region found" end - kitten['region'] ||= $MU_CFG['aws']['region'] + kitten['region'] ||= MU::Cloud::AWS.myRegion end kitten['us_only'] ||= @config['us_only'] kitten['us_only'] ||= false kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms'] kitten['scrub_mu_isms'] ||= false + kitten['credentials'] ||= @config['credentials'] + kitten['credentials'] ||= cloudclass.credConfig(name_only: true) + kitten["dependencies"] ||= [] # Make sure the schema knows about these "new" fields, so that validation # doesn't trip over them. schema_fields.each { |field| if @@schema["properties"][field] - MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG + MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG, details: @@schema["properties"][field] @@schema["properties"][type]["items"]["properties"][field] ||= @@schema["properties"][field] end } end @@ -1604,16 +1683,27 @@ acl = resolveIntraStackFirewallRefs(acl) } # Make sure validation has been called for all on-the-fly generated # resources. - types.each { |type| - @kittens[type].each { |descriptor| - if !descriptor["#MU_VALIDATED"] - ok = false if !insertKitten(descriptor, type) - end + validated_something_new = false + begin + validated_something_new = false + types.each { |type| + @kittens[type].each { |descriptor| + if !descriptor["#MU_VALIDATED"] + validated_something_new = true + ok = false if !insertKitten(descriptor, type) + end + } } + end while validated_something_new + + # Do another pass of resolving intra-stack VPC peering, in case an + # early-parsing VPC needs more details from a later-parsing one + @kittens["vpcs"].each { |vpc| + ok = false if !MU::Config::VPC.resolvePeers(vpc, self) } # add some default holes to allow dependent instances into databases @kittens["databases"].each { |db| if db['port'].nil? @@ -1777,11 +1867,11 @@ prefixes = [] prefixes << "# **REQUIRED**" if required and schema['default'].nil? prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"] prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil? - if !schema['enum'].nil? + if !schema['enum'].nil? and !schema["enum"].empty? prefixes << "# **Must be one of: `#{schema['enum'].join(', ')}`**" elsif !schema['pattern'].nil? # XXX unquoted regex chars confuse the hell out of YARD. How do we # quote {}[] etc in YARD-speak? prefixes << "# **Must match pattern `#{schema['pattern'].gsub(/\n/, "\n#")}`**" @@ -1869,14 +1959,14 @@ "type" => "boolean", "description" => "When 'cloud' is set to 'CloudFormation,' use this flag to strip out Mu-specific artifacts (tags, standard userdata, naming conventions, etc) to yield a clean, source-agnostic template. Setting this flag here will override declarations in individual resources." }, "project" => { "type" => "string", - "description" => "GOOGLE: The project into which to deploy resources", - "default" => MU::Cloud::Google.defaultProject + "description" => "GOOGLE: The project into which to deploy resources" }, "region" => MU::Config.region_primitive, + "credentials" => MU::Config.credentials_primitive, "us_only" => { "type" => "boolean", "description" => "For resources which span regions, restrict to regions inside the United States", "default" => false }, @@ -1981,11 +2071,16 @@ next if failed.include?(type) @@schema["properties"][cfg[:cfg_plural]] = { "type" => "array", "items" => schemaclass.schema } + @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["virtual_name"] = { + "description" => "Internal use.", + "type" => "string" + } @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["dependencies"] = MU::Config.dependencies_primitive @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["cloud"] = MU::Config.cloud_primitive + @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["credentials"] = MU::Config.credentials_primitive @@schema["properties"][cfg[:cfg_plural]]["items"]["title"] = type.to_s rescue NameError => e failed << type MU.log "Error loading #{type} schema from mu/config/#{cfg[:cfg_name]}", MU::ERR, details: "\t"+e.inspect+"\n\t"+e.backtrace[0] end