modules/mu/mommacat.rb in cloud-mu-3.1.5 vs modules/mu/mommacat.rb in cloud-mu-3.1.6

- old
+ new

@@ -17,10 +17,11 @@ require 'json' require 'stringio' require 'securerandom' require 'timeout' require 'mu/mommacat/storage' +require 'mu/mommacat/search' require 'mu/mommacat/daemon' require 'mu/mommacat/naming' module MU @@ -152,11 +153,11 @@ # Make sure mu_user and chef_user are sane. if @mu_user == "root" @chef_user = "mu" else - @chef_user = @mu_user.dup.gsub(/\./, "") + @chef_user = @mu_user.dup.delete(".") @mu_user = "root" if @mu_user == "mu" end @kitten_semaphore = Mutex.new @kittens = {} @original_config = MU::Config.manxify(config) @@ -185,69 +186,23 @@ if set_context_to_me MU::MommaCat.setThreadContext(self) end if create and !@no_artifacts - if !Dir.exist?(MU.dataDir+"/deployments") - MU.log "Creating #{MU.dataDir}/deployments", MU::DEBUG - Dir.mkdir(MU.dataDir+"/deployments", 0700) - end - path = File.expand_path(MU.dataDir+"/deployments")+"/"+@deploy_id - if !Dir.exist?(path) - MU.log "Creating #{path}", MU::DEBUG - Dir.mkdir(path, 0700) - end - if @original_config.nil? or !@original_config.is_a?(Hash) - raise DeployInitializeError, "New MommaCat repository requires config hash" - end - credsets = {} - - MU::Cloud.resource_types.values.each { |attrs| - if !@original_config[attrs[:cfg_plural]].nil? and @original_config[attrs[:cfg_plural]].size > 0 - @original_config[attrs[:cfg_plural]].each { |resource| - - credsets[resource['cloud']] ||= [] - credsets[resource['cloud']] << resource['credentials'] - @clouds[resource['cloud']] = 0 if !@clouds.has_key?(resource['cloud']) - @clouds[resource['cloud']] = @clouds[resource['cloud']] + 1 - - } - end - } - - @ssh_key_name, @ssh_private_key, @ssh_public_key = self.SSHKey - if !File.exist?(deploy_dir+"/private_key") - @private_key, @public_key = createDeployKey - end - MU.log "Creating deploy secret for #{MU.deploy_id}" - @deploy_secret = Password.random(256) - if !@original_config['scrub_mu_isms'] and !@no_artifacts - credsets.each_pair { |cloud, creds| - creds.uniq! - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud) - creds.each { |credentials| - cloudclass.writeDeploySecret(@deploy_id, @deploy_secret, credentials: credentials) - } - } - end - if set_context_to_me - MU::MommaCat.setThreadContext(self) - end - + initDeployDirectory + setDeploySecret + MU::MommaCat.setThreadContext(self) if set_context_to_me save! - end @appname ||= MU.appname @timestamp ||= MU.timestamp @environment ||= MU.environment loadDeploy(set_context_to_me: set_context_to_me) - if !deploy_secret.nil? - if !authKey(deploy_secret) - raise DeployInitializeError, "Client request did not include a valid deploy authorization secret. Verify that userdata runs correctly?" - end + if !deploy_secret.nil? and !authKey(deploy_secret) + raise DeployInitializeError, "Client request did not include a valid deploy authorization secret. Verify that userdata runs correctly?" end @@litter_semaphore.synchronize { @@litters[@deploy_id] ||= self @@ -255,90 +210,11 @@ # Initialize a MU::Cloud object for each resource belonging to this # deploy, IF it already exists, which is to say if we're loading an # existing deploy instead of creating a new one. if !create and @deployment and @original_config and !skip_resource_objects - - MU::Cloud.resource_types.each_pair { |res_type, attrs| - type = attrs[:cfg_plural] - if @deployment.has_key?(type) - - @deployment[type].each_pair { |res_name, data| - orig_cfg = nil - if @original_config.has_key?(type) - @original_config[type].each { |resource| - if resource["name"] == res_name - orig_cfg = resource - break - end - } - end - - # Some Server objects originated from ServerPools, get their - # configs from there - if type == "servers" and orig_cfg.nil? and - @original_config.has_key?("server_pools") - @original_config["server_pools"].each { |resource| - if resource["name"] == res_name - orig_cfg = resource - break - end - } - end - - if orig_cfg.nil? - MU.log "Failed to locate original config for #{attrs[:cfg_name]} #{res_name} in #{@deploy_id}", MU::WARN if !["firewall_rules", "databases", "storage_pools", "cache_clusters", "alarms"].include?(type) # XXX shaddap - next - end - - if orig_cfg['vpc'] and orig_cfg['vpc'].is_a?(Hash) - ref = if orig_cfg['vpc']['id'] and orig_cfg['vpc']['id'].is_a?(Hash) - orig_cfg['vpc']['id']['mommacat'] = self - MU::Config::Ref.get(orig_cfg['vpc']['id']) - else - orig_cfg['vpc']['mommacat'] = self - MU::Config::Ref.get(orig_cfg['vpc']) - end - orig_cfg['vpc'].delete('mommacat') - orig_cfg['vpc'] = ref if ref.kitten(shallow: true) - end - - begin - # Load up MU::Cloud objects for all our kittens in this deploy - orig_cfg['environment'] = @environment # not always set in old deploys - if attrs[:has_multiples] - data.keys.each { |mu_name| - attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: mu_name, delay_descriptor_load: delay_descriptor_load) - } - else - # XXX hack for old deployments, this can go away some day - if data['mu_name'].nil? or data['mu_name'].empty? - if res_type.to_s == "LoadBalancer" and !data['awsname'].nil? - data['mu_name'] = data['awsname'].dup - elsif res_type.to_s == "FirewallRule" and !data['group_name'].nil? - data['mu_name'] = data['group_name'].dup - elsif res_type.to_s == "Database" and !data['identifier'].nil? - data['mu_name'] = data['identifier'].dup.upcase - elsif res_type.to_s == "VPC" - # VPC names are deterministic, just generate the things - data['mu_name'] = getResourceName(data['name']) - end - end - if data['mu_name'].nil? - raise MuError, "Unable to find or guess a Mu name for #{res_type}: #{res_name} in #{@deploy_id}" - end - attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: data['mu_name'], cloud_id: data['cloud_id']) - end - rescue StandardError => e - if e.class != MU::Cloud::MuCloudResourceNotImplemented - MU.log "Failed to load an existing resource of type '#{type}' in #{@deploy_id}: #{e.inspect}", MU::WARN, details: e.backtrace - end - end - } - - end - } + loadObjects(delay_descriptor_load) end @initializing = false # XXX this .owned? method may get changed by the Ruby maintainers @@ -347,11 +223,11 @@ # List all the cloud providers declared by resources in our deploy. def cloudsUsed seen = [] seen << @original_config['cloud'] if @original_config['cloud'] - MU::Cloud.resource_types.values.each { |attrs| + MU::Cloud.resource_types.each_value { |attrs| type = attrs[:cfg_plural] if @original_config[type] @original_config[type].each { |resource| seen << resource['cloud'] if resource['cloud'] } @@ -367,22 +243,19 @@ return [] if !@original_config seen = [] # clouds = [] seen << @original_config['credentials'] if @original_config['credentials'] # defaultcloud = @original_config['cloud'] - MU::Cloud.resource_types.values.each { |attrs| + MU::Cloud.resource_types.each_value { |attrs| type = attrs[:cfg_plural] if @original_config[type] @original_config[type].each { |resource| if resource['credentials'] seen << resource['credentials'] else - cloudclass = if @original_config['cloud'] - Object.const_get("MU").const_get("Cloud").const_get(@original_config['cloud']) - else - Object.const_get("MU").const_get("Cloud").const_get(MU::Config.defaultCloud) - end + cloudconst = @original_config['cloud'] ? @original_config['cloud'] : MU::Config.defaultCloud + Object.const_get("MU").const_get("Cloud").const_get(cloudconst) seen << cloudclass.credConfig(name_only: true) end } end } @@ -402,11 +275,11 @@ if hab_ref and hab_ref.id habitats << hab_ref.id end end - MU::Cloud.resource_types.values.each { |attrs| + MU::Cloud.resource_types.each_value { |attrs| type = attrs[:cfg_plural] if @original_config[type] @original_config[type].each { |resource| if resource['project'] habitats << resource['project'] @@ -479,11 +352,11 @@ realtypes << cfg_plural } end count = 0 - MU::Cloud.resource_types.values.each { |data| + MU::Cloud.resource_types.each_value { |data| next if @original_config[data[:cfg_plural]].nil? next if realtypes.size > 0 and (!negate and !realtypes.include?(data[:cfg_plural])) @original_config[data[:cfg_plural]].each { |resource| if clouds.nil? or clouds.size == 0 or (!negate and clouds.include?(resource["cloud"])) or (negate and !clouds.include?(resource["cloud"])) count = count + 1 @@ -497,17 +370,17 @@ def removeKitten(object) if !object raise MuError, "Nil arguments to removeKitten are not allowed" end @kitten_semaphore.synchronize { - MU::Cloud.resource_types.values.each { |attrs| + MU::Cloud.resource_types.each_value { |attrs| type = attrs[:cfg_plural] next if !@kittens.has_key?(type) tmplitter = @kittens[type].values.dup tmplitter.each { |nodeclass, data| if data.is_a?(Hash) - data.keys.each { |mu_name| + data.each_key { |mu_name| if data == object @kittens[type][nodeclass].delete(mu_name) return end } @@ -532,17 +405,16 @@ if !type or !name or !object or !object.mu_name raise MuError, "Nil arguments to addKitten are not allowed (got type: #{type}, name: #{name}, and '#{object}' to add)" end _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type) - has_multiples = attrs[:has_multiples] object.intoDeploy(self) @kitten_semaphore.synchronize { @kittens[type] ||= {} @kittens[type][object.habitat] ||= {} - if has_multiples + if attrs[:has_multiples] @kittens[type][object.habitat][name] ||= {} @kittens[type][object.habitat][name][object.mu_name] = object else @kittens[type][object.habitat][name] = object end @@ -575,11 +447,11 @@ end MU::MommaCat.lock("deployment-notification") loadDeploy(true) # make sure we're not trampling deployment data @secret_semaphore.synchronize { if @secrets[type].nil? - raise SecretError, "'#{type}' is not a valid secret type (valid types: #{@secrets.keys.to_s})" + raise SecretError, "'#{type}' is not a valid secret type (valid types: #{@secrets.keys.join(", ")})" end @secrets[type][instance_id] = encryptWithDeployKey(raw_secret) } save! MU::MommaCat.unlock("deployment-notification") @@ -591,11 +463,11 @@ # @param quiet [Boolean]: Do not log errors for non-existent secrets def fetchSecret(instance_id, type, quiet: false) @secret_semaphore.synchronize { if @secrets[type].nil? return nil if quiet - raise SecretError, "'#{type}' is not a valid secret type (valid types: #{@secrets.keys.to_s})" + raise SecretError, "'#{type}' is not a valid secret type (valid types: #{@secrets.keys.join(", ")})" end if @secrets[type][instance_id].nil? return nil if quiet raise SecretError, "No '#{type}' secret known for instance #{instance_id}" end @@ -652,649 +524,92 @@ return [@ssh_key_name, @ssh_private_key, @ssh_public_key] end @@dummy_cache = {} - # Locate a resource that's either a member of another deployment, or of no - # deployment at all, and return a {MU::Cloud} object for it. - # @param cloud [String]: The Cloud provider to use. - # @param type [String]: The resource type. Can be the full class name, symbolic name, or Basket of Kittens configuration shorthand for the resource type. - # @param deploy_id [String]: The identifier of an outside deploy to search. - # @param name [String]: The name of the resource as defined in its 'name' Basket of Kittens field, typically used in conjunction with deploy_id. - # @param mu_name [String]: The fully-resolved and deployed name of the resource, typically used in conjunction with deploy_id. - # @param cloud_id [String]: A cloud provider identifier for this resource. - # @param region [String]: The cloud provider region - # @param tag_key [String]: A cloud provider tag to help identify the resource, used in conjunction with tag_value. - # @param tag_value [String]: A cloud provider tag to help identify the resource, used in conjunction with tag_key. - # @param allow_multi [Boolean]: Permit an array of matching resources to be returned (if applicable) instead of just one. - # @param dummy_ok [Boolean]: Permit return of a faked {MU::Cloud} object if we don't have enough information to identify a real live one. - # @param flags [Hash]: Other cloud or resource type specific options to pass to that resource's find() method - # @return [Array<MU::Cloud>] - def self.findStray( - cloud, - type, - deploy_id: nil, - name: nil, - mu_name: nil, - cloud_id: nil, - credentials: nil, - region: nil, - tag_key: nil, - tag_value: nil, - allow_multi: false, - calling_deploy: MU.mommacat, - flags: {}, - habitats: [], - dummy_ok: false, - debug: false, - no_deploy_search: false - ) - start = Time.now - callstr = "findStray(cloud: #{cloud}, type: #{type}, deploy_id: #{deploy_id}, calling_deploy: #{calling_deploy.deploy_id if !calling_deploy.nil?}, name: #{name}, cloud_id: #{cloud_id}, tag_key: #{tag_key}, tag_value: #{tag_value}, credentials: #{credentials}, habitats: #{habitats ? habitats.to_s : "[]"}, dummy_ok: #{dummy_ok.to_s}, flags: #{flags.to_s}) from #{caller[0]}" -# callstack = caller.dup - - return nil if cloud == "CloudFormation" and !cloud_id.nil? - shortclass, _cfg_name, cfg_plural, classname, _attrs = MU::Cloud.getResourceNames(type) - if !MU::Cloud.supportedClouds.include?(cloud) or shortclass.nil? - MU.log "findStray was called with bogus cloud argument '#{cloud}'", MU::WARN, details: callstr - return nil - end - - begin - # TODO this is dumb as hell, clean this up.. and while we're at it - # .dup everything so we don't mangle referenced values from the caller - deploy_id = deploy_id.to_s if deploy_id.class.to_s == "MU::Config::Tail" - name = name.to_s if name.class.to_s == "MU::Config::Tail" - cloud_id = cloud_id.to_s if !cloud_id.nil? - mu_name = mu_name.to_s if mu_name.class.to_s == "MU::Config::Tail" - tag_key = tag_key.to_s if tag_key.class.to_s == "MU::Config::Tail" - tag_value = tag_value.to_s if tag_value.class.to_s == "MU::Config::Tail" - type = cfg_plural - resourceclass = MU::Cloud.loadCloudType(cloud, shortclass) - cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud) - - credlist = if credentials - [credentials] - else - cloudclass.listCredentials - end - - if (tag_key and !tag_value) or (!tag_key and tag_value) - raise MuError, "Can't call findStray with only one of tag_key and tag_value set, must be both or neither" - end - # Help ourselves by making more refined parameters out of mu_name, if - # they weren't passed explicitly - if mu_name - if !tag_key and !tag_value - # XXX "Name" is an AWS-ism, perhaps those plugins should do this bit? - tag_key="Name" - tag_value=mu_name - end - # We can extract a deploy_id from mu_name if we don't have one already - if !deploy_id and mu_name - deploy_id = mu_name.sub(/^(\w+-\w+-\d{10}-[A-Z]{2})-/, '\1') - end - end - loglevel = debug ? MU::NOTICE : MU::DEBUG - - MU.log callstr, loglevel, details: caller - - # See if the thing we're looking for is a member of the deploy that's - # asking after it. - if !deploy_id.nil? and !calling_deploy.nil? and - calling_deploy.deploy_id == deploy_id and (!name.nil? or !mu_name.nil?) - handle = calling_deploy.findLitterMate(type: type, name: name, mu_name: mu_name, cloud_id: cloud_id, credentials: credentials) - return [handle] if !handle.nil? - end - - kittens = {} - # Search our other deploys for matching resources - if !no_deploy_search and (deploy_id or name or mu_name or cloud_id) - MU.log "findStray: searching my deployments (#{cfg_plural}, name: #{name}, deploy_id: #{deploy_id}, mu_name: #{mu_name}) - #{sprintf("%.2fs", (Time.now-start))}", loglevel - - # Check our in-memory cache of live deploys before resorting to - # metadata - littercache = nil - # Sometimes we're called inside a locked thread, sometimes not. Deal - # with locking gracefully. - begin - @@litter_semaphore.synchronize { - littercache = @@litters.dup - } - rescue ThreadError => e - raise e if !e.message.match(/recursive locking/) - littercache = @@litters.dup - end - - littercache.each_pair { |cur_deploy, momma| - next if deploy_id and deploy_id != cur_deploy - - straykitten = momma.findLitterMate(type: type, cloud_id: cloud_id, name: name, mu_name: mu_name, credentials: credentials, created_only: true) - if straykitten - MU.log "Found matching kitten #{straykitten.mu_name} in-memory - #{sprintf("%.2fs", (Time.now-start))}", loglevel - # Peace out if we found the exact resource we want - if cloud_id and straykitten.cloud_id.to_s == cloud_id.to_s - return [straykitten] - elsif mu_name and straykitten.mu_name == mu_name - return [straykitten] - else - kittens[straykitten.cloud_id] ||= straykitten - end - end - } - - mu_descs = MU::MommaCat.getResourceMetadata(cfg_plural, name: name, deploy_id: deploy_id, mu_name: mu_name) - MU.log "findStray: #{mu_descs.size.to_s} deploys had matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel - - mu_descs.each_pair { |cur_deploy_id, matches| - MU.log "findStray: #{cur_deploy_id} had #{matches.size.to_s} initial matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel - next if matches.nil? or matches.size == 0 - - momma = MU::MommaCat.getLitter(cur_deploy_id) - - straykitten = nil - - # If we found exactly one match in this deploy, use its metadata to - # guess at resource names we weren't told. - if matches.size > 1 and cloud_id - MU.log "findStray: attempting to narrow down multiple matches with cloud_id #{cloud_id} - #{sprintf("%.2fs", (Time.now-start))}", loglevel - straykitten = momma.findLitterMate(type: type, cloud_id: cloud_id, credentials: credentials, created_only: true) - elsif matches.size == 1 and name.nil? and mu_name.nil? - if cloud_id.nil? - straykitten = momma.findLitterMate(type: type, name: matches.first["name"], cloud_id: matches.first["cloud_id"], credentials: credentials) - else - MU.log "findStray: fetching single match with cloud_id #{cloud_id} - #{sprintf("%.2fs", (Time.now-start))}", loglevel - straykitten = momma.findLitterMate(type: type, name: matches.first["name"], cloud_id: cloud_id, credentials: credentials) - end -# elsif !flags.nil? and !flags.empty? # XXX eh, maybe later -# # see if we can narrow it down further with some flags -# filtered = [] -# matches.each { |m| -# f = resourceclass.find(cloud_id: m['mu_name'], flags: flags) -# filtered << m if !f.nil? and f.size > 0 -# MU.log "RESULT FROM find(cloud_id: #{m['mu_name']}, flags: #{flags})", MU::WARN, details: f -# } -# if filtered.size == 1 -# straykitten = momma.findLitterMate(type: type, name: matches.first["name"], cloud_id: filtered.first['cloud_id']) -# end - else - # There's more than one of this type of resource in the target - # deploy, so see if findLitterMate can narrow it down for us - straykitten = momma.findLitterMate(type: type, name: name, mu_name: mu_name, cloud_id: cloud_id, credentials: credentials) - end - - next if straykitten.nil? - straykitten.intoDeploy(momma) - - if straykitten.cloud_id.nil? - MU.log "findStray: kitten #{straykitten.mu_name} came back with nil cloud_id", MU::WARN - next - end - - kittens[straykitten.cloud_id] ||= straykitten - - # Peace out if we found the exact resource we want - if cloud_id and straykitten.cloud_id.to_s == cloud_id.to_s - return [straykitten] - # ...or if we've validated our one possible match - elsif !cloud_id and mu_descs.size == 1 and matches.size == 1 - return [straykitten] - elsif credentials and credlist.size == 1 and straykitten.credentials == credentials - return [straykitten] - end - } - - -# if !mu_descs.nil? and mu_descs.size > 0 and !deploy_id.nil? and !deploy_id.empty? and !mu_descs.first.empty? -# MU.log "I found descriptions that might match #{resourceclass.cfg_plural} name: #{name}, deploy_id: #{deploy_id}, mu_name: #{mu_name}, but couldn't isolate my target kitten", MU::WARN, details: caller -# puts File.read(deploy_dir(deploy_id)+"/deployment.json") -# end - - # We can't refine any further by asking the cloud provider... - if !cloud_id and !tag_key and !tag_value and kittens.size > 1 - if !allow_multi - raise MuError, "Multiple matches in MU::MommaCat.findStray where none allowed from deploy_id: '#{deploy_id}', name: '#{name}', mu_name: '#{mu_name}' (#{caller[0]})" - else - return kittens.values - end - end - end - - matches = [] - - found_the_thing = false - credlist.each { |creds| - break if found_the_thing - if cloud_id or (tag_key and tag_value) or !flags.empty? or allow_multi - - regions = begin - region ? [region] : cloudclass.listRegions(credentials: creds) - rescue NoMethodError # Not all cloud providers have regions - [nil] - end - - # ..not all resource types care about regions either - if resourceclass.isGlobal? - regions = [nil] - end - - # Decide what habitats (accounts/projects/subscriptions) we'll - # search, if applicable for this resource type. - habitats ||= [] - begin - if flags["project"] # backwards-compat - habitats << flags["project"] - end - if habitats.empty? - if resourceclass.canLiveIn.include?(nil) - habitats << nil - end - if resourceclass.canLiveIn.include?(:Habitat) - habitats.concat(cloudclass.listProjects(creds)) - end - end - rescue NoMethodError # we only expect this to work on Google atm - end - - if habitats.empty? - habitats << nil - end - habitats.uniq! - - habitat_threads = [] - desc_semaphore = Mutex.new - - cloud_descs = {} - habitats.each { |hab| - begin - habitat_threads.each { |t| t.join(0.1) } - habitat_threads.reject! { |t| t.nil? or !t.status } - sleep 1 if habitat_threads.size > 5 - end while habitat_threads.size > 5 - habitat_threads << Thread.new(hab) { |p| - MU.log "findStray: Searching #{p} (#{habitat_threads.size.to_s} habitat threads running) - #{sprintf("%.2fs", (Time.now-start))}", loglevel - cloud_descs[p] = {} - region_threads = [] - regions.each { |reg| region_threads << Thread.new(reg) { |r| - MU.log "findStray: Searching #{r} in #{p} (#{region_threads.size.to_s} region threads running) - #{sprintf("%.2fs", (Time.now-start))}", loglevel - MU.log "findStray: calling #{classname}.find(cloud_id: #{cloud_id}, region: #{r}, tag_key: #{tag_key}, tag_value: #{tag_value}, flags: #{flags}, credentials: #{creds}, project: #{p}) - #{sprintf("%.2fs", (Time.now-start))}", loglevel - found = resourceclass.find(cloud_id: cloud_id, region: r, tag_key: tag_key, tag_value: tag_value, flags: flags, credentials: creds, habitat: p) - MU.log "findStray: #{found ? found.size.to_s : "nil"} results - #{sprintf("%.2fs", (Time.now-start))}", loglevel - - if found - desc_semaphore.synchronize { - cloud_descs[p][r] = found - } - end - # Stop if you found the thing by a specific cloud_id - if cloud_id and found and !found.empty? - found_the_thing = true - Thread.exit - end - } } - begin - region_threads.each { |t| t.join(0.1) } - region_threads.reject! { |t| t.nil? or !t.status } - if region_threads.size > 0 - MU.log "#{region_threads.size.to_s} regions still running in #{p}", loglevel - sleep 3 - end - end while region_threads.size > 0 - } - } - begin - habitat_threads.each { |t| t.join(0.1) } - habitat_threads.reject! { |t| t.nil? or !t.status } - if habitat_threads.size > 0 - MU.log "#{habitat_threads.size.to_s} habitats still running", loglevel - sleep 3 - end - end while habitat_threads.size > 0 - - habitat_threads = [] - habitats.each { |hab| habitat_threads << Thread.new(hab) { |p| - region_threads = [] - regions.each { |reg| region_threads << Thread.new(reg) { |r| - next if cloud_descs[p][r].nil? - cloud_descs[p][r].each_pair { |kitten_cloud_id, descriptor| - - # We already have a MU::Cloud object for this guy, use it - if kittens.has_key?(kitten_cloud_id) - desc_semaphore.synchronize { - matches << kittens[kitten_cloud_id] - } - elsif kittens.size == 0 - if !dummy_ok - next - end - - # If we don't have a MU::Cloud object, manufacture a dummy - # one. Give it a fake name if we have to and have decided - # that's ok. Wild inferences from the cloud descriptor are - # ok to try here. - use_name = if (name.nil? or name.empty?) - if !dummy_ok - nil - elsif !mu_name.nil? - mu_name - # AWS-style tags - elsif descriptor.respond_to?(:tags) and - descriptor.tags.is_a?(Array) and - descriptor.tags.first.respond_to?(:key) and - descriptor.tags.map { |t| t.key }.include?("Name") - descriptor.tags.select { |t| t.key == "Name" }.first.value - else - try = nil - # Various GCP fields - [:display_name, :name, (resourceclass.cfg_name+"_name").to_sym].each { |field| - if descriptor.respond_to?(field) and descriptor.send(field).is_a?(String) - try = descriptor.send(field) - break - end - - } - try ||= if !tag_value.nil? - tag_value - else - kitten_cloud_id - end - try - end - else - name - end - if use_name.nil? - MU.log "Found cloud provider data for #{cloud} #{type} #{kitten_cloud_id}, but without a name I can't manufacture a proper #{type} object to return - #{sprintf("%.2fs", (Time.now-start))}", loglevel, details: caller - next - end - cfg = { - "name" => use_name, - "cloud" => cloud, - "credentials" => creds - } - if !r.nil? and !resourceclass.isGlobal? - cfg["region"] = r - end - - if !p.nil? and resourceclass.canLiveIn.include?(:Habitat) - cfg["project"] = p - end - # If we can at least find the config from the deploy this will - # belong with, use that, even if it's an ungroomed resource. - if !calling_deploy.nil? and - !calling_deploy.original_config.nil? and - !calling_deploy.original_config[type+"s"].nil? - calling_deploy.original_config[type+"s"].each { |s| - if s["name"] == use_name - cfg = s.dup - break - end - } - - newkitten = resourceclass.new(mommacat: calling_deploy, kitten_cfg: cfg, cloud_id: kitten_cloud_id) - desc_semaphore.synchronize { - matches << newkitten - } - else - if !@@dummy_cache[cfg_plural] or !@@dummy_cache[cfg_plural][cfg.to_s] - MU.log "findStray: Generating dummy '#{resourceclass.to_s}' cloudobj with name: #{use_name}, cloud_id: #{kitten_cloud_id.to_s} - #{sprintf("%.2fs", (Time.now-start))}", loglevel, details: cfg - resourceclass.new(mu_name: use_name, kitten_cfg: cfg, cloud_id: kitten_cloud_id.to_s, from_cloud_desc: descriptor) - desc_semaphore.synchronize { - @@dummy_cache[cfg_plural] ||= {} - @@dummy_cache[cfg_plural][cfg.to_s] = resourceclass.new(mu_name: use_name, kitten_cfg: cfg, cloud_id: kitten_cloud_id.to_s, from_cloud_desc: descriptor) - MU.log "findStray: Finished generating dummy '#{resourceclass.to_s}' cloudobj - #{sprintf("%.2fs", (Time.now-start))}", loglevel - } - end - desc_semaphore.synchronize { - matches << @@dummy_cache[cfg_plural][cfg.to_s] - } - end - end - } - } } - MU.log "findStray: tying up #{region_threads.size.to_s} region threads - #{sprintf("%.2fs", (Time.now-start))}", loglevel - region_threads.each { |t| - t.join - } - } } - MU.log "findStray: tying up #{habitat_threads.size.to_s} habitat threads - #{sprintf("%.2fs", (Time.now-start))}", loglevel - habitat_threads.each { |t| - t.join - } - end - } - rescue StandardError => e - MU.log e.inspect, MU::ERR, details: e.backtrace - end - MU.log "findStray: returning #{matches ? matches.size.to_s : "0"} matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel - - matches - end - - # Return the resource object of another member of this deployment - # @param type [String,Symbol]: The type of resource - # @param name [String]: The name of the resource as defined in its 'name' Basket of Kittens field - # @param mu_name [String]: The fully-resolved and deployed name of the resource - # @param cloud_id [String]: The cloud provider's unique identifier for this resource - # @param created_only [Boolean]: Only return the littermate if its cloud_id method returns a value - # @param return_all [Boolean]: Return a Hash of matching objects indexed by their mu_name, instead of a single match. Only valid for resource types where has_multiples is true. - # @return [MU::Cloud] - def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, debug: false, indent: "") - shortclass, cfg_name, cfg_plural, classname, attrs = MU::Cloud.getResourceNames(type) - type = cfg_plural - has_multiples = attrs[:has_multiples] - - loglevel = debug ? MU::NOTICE : MU::DEBUG - - argstring = [:type, :name, :mu_name, :cloud_id, :created_only, :credentials, :habitat, :has_multiples].reject { |a| - binding.local_variable_get(a).nil? - }.map { |v| - v.to_s+": "+binding.local_variable_get(v).to_s - }.join(", ") - - # Fun times: if we specified a habitat, which we may also have done by - # its shorthand sibling name, let's... call ourselves first to make sure - # we're fishing for the right thing. - if habitat - if habitat.is_a?(MU::Config::Ref) and habitat.id - habitat = habitat.id - else - MU.log indent+"findLitterMate(#{argstring}): Attempting to resolve habitat name #{habitat}", loglevel - realhabitat = findLitterMate(type: "habitat", name: habitat, debug: debug, credentials: credentials, indent: indent+" ") - if realhabitat and realhabitat.mu_name - MU.log indent+"findLitterMate: Resolved habitat name #{habitat} to #{realhabitat.mu_name}", loglevel, details: [realhabitat.mu_name, realhabitat.cloud_id, realhabitat.config.keys] - habitat = realhabitat.cloud_id - elsif debug - MU.log indent+"findLitterMate(#{argstring}): Failed to resolve habitat name #{habitat}", MU::WARN - end - end - end - - - @kitten_semaphore.synchronize { - if !@kittens.has_key?(type) - if debug - MU.log indent+"NO SUCH KEY #{type} findLitterMate(#{argstring})", MU::WARN, details: @kittens.keys - end - return nil - end - MU.log indent+"START findLitterMate(#{argstring}), caller: #{caller[2]}", loglevel, details: @kittens[type].keys.map { |hab| hab.to_s+": "+@kittens[type][hab].keys.join(", ") } - matches = [] - - @kittens[type].each { |habitat_group, sib_classes| - next if habitat and habitat_group != habitat and !habitat_group.nil? - sib_classes.each_pair { |sib_class, data| - virtual_name = nil - - if !has_multiples and data and !data.is_a?(Hash) and data.config and data.config.is_a?(Hash) and data.config['virtual_name'] and name == data.config['virtual_name'] - virtual_name = data.config['virtual_name'] - elsif !name.nil? and name != sib_class - next - end - if has_multiples - if !name.nil? - if return_all - MU.log indent+"MULTI-MATCH RETURN_ALL findLitterMate(#{argstring})", loglevel, details: data.keys - return data.dup - end - if data.size == 1 and (cloud_id.nil? or data.values.first.cloud_id == cloud_id) - return data.values.first - elsif mu_name.nil? and cloud_id.nil? - MU.log indent+"#{@deploy_id}: Found multiple matches in findLitterMate based on #{type}: #{name}, and not enough info to narrow down further. Returning an arbitrary result. Caller: #{caller[2]}", MU::WARN, details: data.keys - return data.values.first - end - end - data.each_pair { |sib_mu_name, obj| - if (!mu_name.nil? and mu_name == sib_mu_name) or - (!cloud_id.nil? and cloud_id == obj.cloud_id) or - (!credentials.nil? and credentials == obj.credentials) - if !created_only or !obj.cloud_id.nil? - if return_all - MU.log indent+"MULTI-MATCH RETURN_ALL findLitterMate(#{argstring})", loglevel, details: data.keys - return data.dup - else - MU.log indent+"MULTI-MATCH findLitterMate(#{argstring})", loglevel, details: data.keys - return obj - end - end - end - } - else - - MU.log indent+"CHECKING AGAINST findLitterMate #{habitat_group}/#{type}/#{sib_class} data.cloud_id: #{data.cloud_id}, data.credentials: #{data.credentials}, sib_class: #{sib_class}, virtual_name: #{virtual_name}", loglevel, details: argstring - - data_cloud_id = data.cloud_id.nil? ? nil : data.cloud_id.to_s - - MU.log indent+"(name.nil? or sib_class == name or virtual_name == name)", loglevel, details: (name.nil? or sib_class == name or virtual_name == name).to_s - MU.log indent+"(cloud_id.nil? or cloud_id[#{cloud_id.class.name}:#{cloud_id.to_s}] == data_cloud_id[#{data_cloud_id.class.name}:#{data_cloud_id}])", loglevel, details: (cloud_id.nil? or cloud_id == data_cloud_id).to_s - MU.log indent+"(credentials.nil? or data.credentials.nil? or credentials[#{credentials.class.name}:#{credentials}] == data.credentials[#{data.credentials.class.name}:#{data.credentials}])", loglevel, details: (credentials.nil? or data.credentials.nil? or credentials == data.credentials).to_s - - if (name.nil? or sib_class == name.to_s or virtual_name == name.to_s) and - (cloud_id.nil? or cloud_id.to_s == data_cloud_id) and - (credentials.nil? or data.credentials.nil? or credentials.to_s == data.credentials.to_s) - MU.log indent+"OUTER MATCH PASSED, NEED !created_only (#{created_only.to_s}) or !data_cloud_id.nil? (#{data_cloud_id})", loglevel, details: (cloud_id.nil? or cloud_id == data_cloud_id).to_s - if !created_only or !data_cloud_id.nil? - MU.log indent+"SINGLE MATCH findLitterMate(#{argstring})", loglevel, details: [data.mu_name, data_cloud_id, data.config.keys] - matches << data - end - end - end - } - } - - return matches.first if matches.size == 1 - if return_all and matches.size > 1 - return matches - end - } - - MU.log indent+"NO MATCH findLitterMate(#{argstring})", loglevel - - return nil - end - # Add or remove a resource's metadata to this deployment's structure and # flush it to disk. # @param type [String]: The type of resource (e.g. *server*, *database*). # @param key [String]: The name field of this resource. + # @param mu_name [String]: The mu_name of this resource. # @param data [Hash]: The resource's metadata. + # @param triggering_node [MU::Cloud]: A cloud object calling this notify, usually on behalf of itself # @param remove [Boolean]: Remove this resource from the deploy structure, instead of adding it. # @return [void] def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, delayed_save: false) return if @no_artifacts - MU::MommaCat.lock("deployment-notification") - if !@need_deploy_flush or @deployment.nil? or @deployment.empty? - loadDeploy(true) # make sure we're saving the latest and greatest - end + begin + MU::MommaCat.lock("deployment-notification") - _shortclass, _cfg_name, cfg_plural, _classname, attrs = MU::Cloud.getResourceNames(type) - has_multiples = false + if !@need_deploy_flush or @deployment.nil? or @deployment.empty? + loadDeploy(true) # make sure we're saving the latest and greatest + end - # it's not always the case that we're logging data for a legal resource - # type, though that's what we're usually for - if cfg_plural - type = cfg_plural - has_multiples = attrs[:has_multiples] - end + _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type, false) + has_multiples = attrs[:has_multiples] ? true : false - if mu_name.nil? - if !data.nil? and !data["mu_name"].nil? - mu_name = data["mu_name"] + mu_name ||= if !data.nil? and !data["mu_name"].nil? + data["mu_name"] elsif !triggering_node.nil? and !triggering_node.mu_name.nil? - mu_name = triggering_node.mu_name + triggering_node.mu_name end if mu_name.nil? and has_multiples - MU.log "MU::MommaCat.notify called to modify deployment struct for a type (#{type}) with :has_multiples, but no mu_name available to look under #{key}. Call was #{caller[0]}", MU::WARN, details: data - MU::MommaCat.unlock("deployment-notification") + MU.log "MU::MommaCat.notify called to modify deployment struct for a type (#{type}) with :has_multiples, but no mu_name available to look under #{key}. Call was #{caller(1..1)}", MU::WARN, details: data return end - end - @need_deploy_flush = true + @need_deploy_flush = true - if !remove - if data.nil? - MU.log "MU::MommaCat.notify called to modify deployment struct, but no data provided", MU::WARN - MU::MommaCat.unlock("deployment-notification") - return - end - @notify_semaphore.synchronize { - @deployment[type] ||= {} - } - if has_multiples + if !remove + if data.nil? + MU.log "MU::MommaCat.notify called to modify deployment struct, but no data provided", MU::WARN + return + end @notify_semaphore.synchronize { - @deployment[type][key] ||= {} + @deployment[type] ||= {} } - # fix has_multiples classes that weren't tiered correctly - if @deployment[type][key].is_a?(Hash) and @deployment[type][key].has_key?("mu_name") - olddata = @deployment[type][key].dup - @deployment[type][key][olddata["mu_name"]] = olddata - end - @deployment[type][key][mu_name] = data - MU.log "Adding to @deployment[#{type}][#{key}][#{mu_name}]", MU::DEBUG, details: data - else - @deployment[type][key] = data - MU.log "Adding to @deployment[#{type}][#{key}]", MU::DEBUG, details: data - end - save!(key) if !delayed_save - else - have_deploy = true - if @deployment[type].nil? or @deployment[type][key].nil? - if has_multiples - MU.log "MU::MommaCat.notify called to remove #{type} #{key} #{mu_name} deployment struct, but no such data exist", MU::DEBUG + @notify_semaphore.synchronize { + @deployment[type][key] ||= {} + } + @deployment[type][key][mu_name] = data + MU.log "Adding to @deployment[#{type}][#{key}][#{mu_name}]", MU::DEBUG, details: data else - MU.log "MU::MommaCat.notify called to remove #{type} #{key} deployment struct, but no such data exist", MU::DEBUG + @deployment[type][key] = data + MU.log "Adding to @deployment[#{type}][#{key}]", MU::DEBUG, details: data end - MU::MommaCat.unlock("deployment-notification") + save!(key) if !delayed_save + else + have_deploy = true + if @deployment[type].nil? or @deployment[type][key].nil? + MU.log "MU::MommaCat.notify called to remove #{type} #{key}#{has_multiples ? " "+mu_name : ""} deployment struct, but no such data exist", MU::DEBUG + return + end - return - end + if have_deploy + @notify_semaphore.synchronize { + if has_multiples + MU.log "Removing @deployment[#{type}][#{key}][#{mu_name}]", MU::DEBUG, details: @deployment[type][key][mu_name] + @deployment[type][key].delete(mu_name) + end - if have_deploy - @notify_semaphore.synchronize { - if has_multiples - MU.log "Removing @deployment[#{type}][#{key}][#{mu_name}]", MU::DEBUG, details: @deployment[type][key][mu_name] - @deployment[type][key].delete(mu_name) - if @deployment[type][key].size == 0 + if @deployment[type][key].empty? or !has_multiples + MU.log "Removing @deployment[#{type}][#{key}]", MU::DEBUG, details: @deployment[type][key] @deployment[type].delete(key) end - else - MU.log "Removing @deployment[#{type}][#{key}]", MU::DEBUG, details: @deployment[type][key] - @deployment[type].delete(key) - end - if @deployment[type].size == 0 - @deployment.delete(type) - end - } - end - save! if !delayed_save + if @deployment[type].empty? + @deployment.delete(type) + end + } + end + save! if !delayed_save + end + ensure + MU::MommaCat.unlock("deployment-notification") end - - MU::MommaCat.unlock("deployment-notification") end # Send a Slack notification to a deployment's administrators. # @param subject [String]: The subject line of the message. # @param msg [String]: The message body. @@ -1329,17 +644,17 @@ if !@original_config.nil? @original_config['admins'].each { |admin| to << "#{admin['name']} <#{admin['email']}>" } end - message = <<MESSAGE_END + message = <<MAIL_HEAD_END From: #{MU.handle} <root@localhost> To: #{to.join(",")} Subject: #{subject} #{msg} -MESSAGE_END +MAIL_HEAD_END if !kitten.nil? and kitten.kind_of?(MU::Cloud) message = message + "\n\n**** #{kitten}:\n" if !kitten.report.nil? kitten.report.each { |line| message = message + line @@ -1423,127 +738,61 @@ # @param csr_path [String]: The CSR to sign, as a file. def signSSLCert(csr_path, sans = []) MU::Master::SSL.sign(csr_path, sans, for_user: MU.mu_user) end - # Make sure deployment data is synchronized to/from each node in the + # Make sure deployment data is synchronized to/from each +Server+ in the # currently-loaded deployment. + # @param nodeclasses [Array<String>] + # @param triggering_node [String,MU::Cloud::Server] + # @param save_only [Boolean] def syncLitter(nodeclasses = [], triggering_node: nil, save_only: false) -# XXX take some config logic to decide what nodeclasses to hit? like, make -# inferences from dependencies or something? - - return if MU.syncLitterThread + return if MU.syncLitterThread # don't run recursively by accident return if !Dir.exist?(deploy_dir) - svrs = MU::Cloud.resource_types[:Server][:cfg_plural] # legibility shorthand - if !triggering_node.nil? and nodeclasses.size > 0 - nodeclasses.reject! { |n| n == triggering_node.to_s } - return if nodeclasses.size == 0 + + if !triggering_node.nil? and triggering_node.is_a?(MU::Cloud::Server) + triggering_node = triggering_node.mu_name end - @kitten_semaphore.synchronize { - if @kittens.nil? or - @kittens[svrs].nil? - MU.log "No #{svrs} as yet available in #{@deploy_id}", MU::DEBUG, details: @kittens - return - end + siblings = findLitterMate(type: "server", return_all: true) + return if siblings.nil? or siblings.empty? - - MU.log "Updating these node classes in #{@deploy_id}", MU::DEBUG, details: nodeclasses - } - update_servers = [] - if nodeclasses.nil? or nodeclasses.size == 0 - litter = findLitterMate(type: "server", return_all: true) - return if litter.nil? - litter.each_pair { |mu_name, node| - if !triggering_node.nil? and ( - (triggering_node.is_a?(MU::Cloud::Server) and mu_name == triggering_node.mu_name) or - (triggering_node.is_a?(String) and mu_name == triggering_node) - ) - next - end + siblings.each_pair { |mu_name, node| + next if mu_name == triggering_node or node.groomer.nil? + next if nodeclasses.size > 0 and !nodeclasses.include?(node.config['name']) + if !node.deploydata or !node.deploydata['nodename'] + MU.log "#{mu_name} deploy data is missing (possibly retired or mid-bootstrap), so not syncing it", MU::NOTICE + next + end - if !node.groomer.nil? - update_servers << node - end - } - else - litter = {} - nodeclasses.each { |nodeclass| - mates = findLitterMate(type: "server", name: nodeclass, return_all: true) - litter.merge!(mates) if mates - } - litter.each_pair { |mu_name, node| - if !triggering_node.nil? and ( - (triggering_node.is_a?(MU::Cloud::Server) and mu_name == triggering_node.mu_name) or - (triggering_node.is_a?(String) and mu_name == triggering_node) - ) - next - end - - if !node.deploydata or !node.deploydata.keys.include?('nodename') - details = node.deploydata ? node.deploydata.keys : nil - MU.log "#{mu_name} deploy data is missing (possibly retired or mid-bootstrap), so not syncing it", MU::WARN, details: details - else - update_servers << node - end - } - end - return if update_servers.size == 0 - - MU.log "Updating these nodes in #{@deploy_id}", MU::DEBUG, details: update_servers.map { |n| n.mu_name } - - update_servers.each { |node| - # Not clear where this pollution comes from, but let's stick a temp - # fix in here. - if node.deploydata['nodename'] != node.mu_name and - !node.deploydata['nodename'].nil? and !node.deploydata['nodename'].emty? - MU.log "Node #{node.mu_name} had wrong or missing nodename (#{node.deploydata['nodename']}), correcting", MU::WARN - node.deploydata['nodename'] = node.mu_name - if @deployment[svrs] and @deployment[svrs][node.config['name']] and - @deployment[svrs][node.config['name']][node.mu_name] - @deployment[svrs][node.config['name']][node.mu_name]['nodename'] = node.mu_name - end - save! + if @deployment["servers"][node.config['name']][node.mu_name].nil? or + @deployment["servers"][node.config['name']][node.mu_name] != node.deploydata + @deployment["servers"][node.config['name']][node.mu_name] = node.deploydata + elsif !save_only + # Don't bother running grooms on nodes that don't need to be updated, + # unless we're just going to do a save. + next end + update_servers << node } - # Merge everyone's deploydata together - if !save_only - skip = [] - update_servers.each { |node| - if node.mu_name.nil? or node.deploydata.nil? or node.config.nil? - MU.log "Missing mu_name #{node.mu_name}, deploydata, or config from #{node} in syncLitter", MU::ERR, details: node.deploydata - next - end + return if update_servers.empty? - if !@deployment[svrs][node.config['name']].has_key?(node.mu_name) or @deployment[svrs][node.config['name']][node.mu_name] != node.deploydata - @deployment[svrs][node.config['name']][node.mu_name] = node.deploydata - else - skip << node - end - } - update_servers = update_servers - skip - end + MU.log "Updating nodes in #{@deploy_id}", MU::DEBUG, details: update_servers.map { |n| n.mu_name } - return if MU.inGem? || update_servers.size < 1 threads = [] - parent_thread_id = Thread.current.object_id update_servers.each { |sibling| threads << Thread.new { Thread.abort_on_exception = true - MU.dupGlobals(parent_thread_id) Thread.current.thread_variable_set("name", "sync-"+sibling.mu_name.downcase) MU.setVar("syncLitterThread", true) begin - if sibling.config['groom'].nil? or sibling.config['groom'] - sibling.groomer.saveDeployData - sibling.groomer.run(purpose: "Synchronizing sibling kittens") if !save_only - end + sibling.groomer.saveDeployData + sibling.groomer.run(purpose: "Synchronizing sibling kittens") if !save_only rescue MU::Groomer::RunError => e - MU.log "Sync of #{sibling.mu_name} failed: #{e.inspect}", MU::WARN + MU.log "Sync of #{sibling.mu_name} failed", MU::WARN, details: e.inspect end - MU.purgeGlobals } } threads.each { |t| t.join