modules/mu/mommacat.rb in cloud-mu-3.0.0 vs modules/mu/mommacat.rb in cloud-mu-3.0.1

- old
+ new

@@ -37,10 +37,11 @@ # Failure to groom a node class GroomError < MuError; end @@litters = {} + @@litters_loadtime = {} @@litter_semaphore = Mutex.new # Return a {MU::MommaCat} instance for an existing deploy. Use this instead # of using #initialize directly to avoid loading deploys multiple times or # stepping on the global context for the deployment you're really working @@ -60,28 +61,55 @@ littercache = nil begin @@litter_semaphore.synchronize { littercache = @@litters.dup } + if littercache[deploy_id] and @@litters_loadtime[deploy_id] + deploy_root = File.expand_path(MU.dataDir+"/deployments") + this_deploy_dir = deploy_root+"/"+deploy_id + if File.exist?("#{this_deploy_dir}/deployment.json") + lastmod = File.mtime("#{this_deploy_dir}/deployment.json") + if lastmod > @@litters_loadtime[deploy_id] + MU.log "Deployment metadata for #{deploy_id} was modified on disk, reload", MU::NOTICE + use_cache = false + end + end + end rescue ThreadError => e # already locked by a parent caller and this is a read op, so this is ok raise e if !e.message.match(/recursive locking/) littercache = @@litters.dup end + if !use_cache or littercache[deploy_id].nil? + need_gc = !littercache[deploy_id].nil? newlitter = MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me) # This, we have to synchronize, as it's a write @@litter_semaphore.synchronize { - @@litters[deploy_id] ||= newlitter + @@litters[deploy_id] = newlitter + @@litters_loadtime[deploy_id] = Time.now } + GC.start if need_gc elsif set_context_to_me MU::MommaCat.setThreadContext(@@litters[deploy_id]) end return @@litters[deploy_id] # MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me) end + # Update the in-memory cache of a given deploy. This is intended for use by + # {#save!}, primarily. + # @param deploy_id [String] + # @param litter [MU::MommaCat] + def self.updateLitter(deploy_id, litter) + return if litter.nil? + @@litter_semaphore.synchronize { + @@litters[deploy_id] = litter + @@litters_loadtime[deploy_id] = Time.now + } + end + attr_reader :initializing attr_reader :public_key attr_reader :deploy_secret attr_reader :deployment attr_reader :original_config @@ -631,12 +659,13 @@ # Balancers). # @param name [String]: The shorthand name of the resource, usually the value of the "name" field in an Mu resource declaration. # @param max_length [Integer]: The maximum length of the resulting resource name. # @param need_unique_string [Boolean]: Whether to forcibly append a random three-character string to the name to ensure it's unique. Note that this behavior will be automatically invoked if the name must be truncated. # @param scrub_mu_isms [Boolean]: Don't bother with generating names specific to this deployment. Used to generate generic CloudFormation templates, amongst other purposes. + # @param allowed_chars [Regexp]: A pattern of characters that are legal for this resource name, such as +/[a-zA-Z0-9-]/+ # @return [String]: A full name string for this resource - def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms']) + def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], allowed_chars: nil) if name.nil? raise MuError, "Got no argument to MU::MommaCat.getResourceName" end if @appname.nil? or @environment.nil? or @timestamp.nil? or @seed.nil? MU.log "getResourceName: Missing global deploy variables in thread #{Thread.current.object_id}, using bare name '#{name}' (appname: #{@appname}, environment: #{@environment}, timestamp: #{@timestamp}, seed: #{@seed}, deploy_id: #{@deploy_id}", MU::WARN, details: caller @@ -655,10 +684,23 @@ basename = @appname.upcase + "-" + @environment.upcase + "-" + @timestamp + "-" + @seed.upcase + "-" + name.upcase if scrub_mu_isms basename = @appname.upcase + "-" + @environment.upcase + name.upcase end + subchar = if allowed_chars + if !"-".match(allowed_chars) + if "_".match(allowed_chars) + "_" + else + "" + end + else + "-" + end + end + + basename.gsub!(allowed_chars, subchar) if allowed_chars begin if (basename.length + reserved) > max_length MU.log "Stripping name down from #{basename}[#{basename.length.to_s}] (reserved: #{reserved.to_s}, max_length: #{max_length.to_s})", MU::DEBUG if basename == @appname.upcase + "-" + @seed.upcase + "-" + name.upcase # If we've run out of stuff to strip, truncate what's left and @@ -667,17 +709,19 @@ # hostnames. basename = name.upcase + "-" + @appname.upcase basename.slice!((max_length-(reserved+3))..basename.length) basename.sub!(/-$/, "") basename = basename + "-" + @seed.upcase + basename.gsub!(allowed_chars, subchar) if allowed_chars else # If we have to strip anything, assume we've lost uniqueness and # will have to compensate with #genUniquenessString. need_unique_string = true reserved = 4 basename.sub!(/-[^-]+-#{@seed.upcase}-#{Regexp.escape(name.upcase)}$/, "") basename = basename + "-" + @seed.upcase + "-" + name.upcase + basename.gsub!(allowed_chars, subchar) if allowed_chars end end end while (basename.length + reserved) > max_length # Finally, apply our short random differentiator, if it's needed. @@ -700,10 +744,11 @@ } end else muname = basename end + muname.gsub!(allowed_chars, subchar) if allowed_chars return muname end @@ -899,11 +944,11 @@ end MU::MommaCat.unlock(cloud_id+"-mommagroom") if MU.myCloud == "AWS" MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract end - MU::MommaCat.getLitter(MU.deploy_id, use_cache: false) + MU::MommaCat.getLitter(MU.deploy_id) MU::MommaCat.syncMonitoringConfig(false) MU.log "Grooming complete for '#{name}' mu_name on \"#{MU.handle}\" (#{MU.deploy_id})" FileUtils.touch(MU.dataDir+"/deployments/#{MU.deploy_id}/#{name}_done.txt") MU::MommaCat.unlockAll if first_groom @@ -925,11 +970,11 @@ if !File.directory?(ssh_dir) then MU.log "Creating #{ssh_dir}", MU::DEBUG Dir.mkdir(ssh_dir, 0700) if Process.uid == 0 and @mu_user != "mu" - File.chown(Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid, ssh_dir) + FileUtils.chown Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid, ssh_dir end end if !File.exist?("#{ssh_dir}/#{@ssh_key_name}") MU.log "Generating SSH key #{@ssh_key_name}" %x{/usr/bin/ssh-keygen -N "" -f #{ssh_dir}/#{@ssh_key_name}} @@ -1100,27 +1145,32 @@ @cleanup_threads = [] # Iterate over all known deployments and look for instances that have been # terminated, but not yet cleaned up, then clean them up. - def self.cleanTerminatedInstances + def self.cleanTerminatedInstances(debug = false) + loglevel = debug ? MU::NOTICE : MU::DEBUG MU::MommaCat.lock("clean-terminated-instances", false, true) - MU.log "Checking for harvested instances in need of cleanup", MU::DEBUG + MU.log "Checking for harvested instances in need of cleanup", loglevel parent_thread_id = Thread.current.object_id purged = 0 + MU::MommaCat.listDeploys.each { |deploy_id| next if File.exist?(deploy_dir(deploy_id)+"/.cleanup") - MU.log "Checking for dead wood in #{deploy_id}", MU::DEBUG + MU.log "Checking for dead wood in #{deploy_id}", loglevel need_reload = false @cleanup_threads << Thread.new { MU.dupGlobals(parent_thread_id) deploy = MU::MommaCat.getLitter(deploy_id, set_context_to_me: true) purged_this_deploy = 0 + MU.log "#{deploy_id} has some kittens in it", loglevel, details: deploy.kittens.keys if deploy.kittens.has_key?("servers") + MU.log "#{deploy_id} has some servers declared", loglevel, details: deploy.object_id deploy.kittens["servers"].values.each { |nodeclasses| nodeclasses.each_pair { |nodeclass, servers| deletia = [] + MU.log "Checking status of servers under '#{nodeclass}'", loglevel, details: servers.keys servers.each_pair { |mu_name, server| server.describe if !server.cloud_id MU.log "Checking for presence of #{mu_name}, but unable to fetch its cloud_id", MU::WARN, details: server elsif !server.active? @@ -1143,35 +1193,39 @@ } deletia.each { |mu_name| servers.delete(mu_name) } if purged_this_deploy > 0 - # XXX some kind of filter (obey sync_siblings on nodes' configs) - deploy.syncLitter(servers.keys) + # XXX triggering_node needs to take more than one node name + deploy.syncLitter(servers.keys, triggering_node: deletia.first) end } } end if need_reload + MU.log "Saving modified deploy #{deploy_id}", loglevel deploy.save! - MU::MommaCat.getLitter(deploy_id, use_cache: false) + MU::MommaCat.getLitter(deploy_id) end MU.purgeGlobals } } @cleanup_threads.each { |t| t.join } + MU.log "cleanTerminatedInstances threads complete", loglevel + MU::MommaCat.unlock("clean-terminated-instances", true) @cleanup_threads = [] if purged > 0 if MU.myCloud == "AWS" MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract end MU::MommaCat.syncMonitoringConfig + GC.start end - MU::MommaCat.unlock("clean-terminated-instances", true) + MU.log "cleanTerminatedInstances returning", loglevel end @@dummy_cache = {} # Locate a resource that's either a member of another deployment, or of no @@ -1888,26 +1942,55 @@ "MU-MASTER-NAME" => Socket.gethostname, "MU-OWNER" => MU.mu_user } end + # Clean an IP address out of ~/.ssh/known hosts + # @param ip [String]: The IP to remove + # @return [void] + def self.removeIPFromSSHKnownHosts(ip) + return if ip.nil? + sshdir = "#{@myhome}/.ssh" + knownhosts = "#{sshdir}/known_hosts" + + if File.exist?(knownhosts) and File.open(knownhosts).read.match(/^#{Regexp.quote(ip)} /) + MU.log "Expunging old #{ip} entry from #{knownhosts}", MU::NOTICE + if !@noop + File.open(knownhosts, File::CREAT|File::RDWR, 0600) { |f| + f.flock(File::LOCK_EX) + newlines = Array.new + delete_block = false + f.readlines.each { |line| + next if line.match(/^#{Regexp.quote(ip)} /) + newlines << line + } + f.rewind + f.truncate(0) + f.puts(newlines) + f.flush + f.flock(File::LOCK_UN) + } + end + end + end + # Clean a node's entries out of ~/.ssh/config - # @param node [String]: The node's name + # @param nodename [String]: The node's name # @return [void] - def self.removeHostFromSSHConfig(node) + def self.removeHostFromSSHConfig(nodename) sshdir = "#{@myhome}/.ssh" sshconf = "#{sshdir}/config" - if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{node} /) - MU.log "Expunging old #{node} entry from #{sshconf}", MU::DEBUG + if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{nodename} /) + MU.log "Expunging old #{nodename} entry from #{sshconf}", MU::DEBUG if !@noop File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f| f.flock(File::LOCK_EX) newlines = Array.new delete_block = false f.readlines.each { |line| - if line.match(/^Host #{node}(\s|$)/) + if line.match(/^Host #{nodename}(\s|$)/) delete_block = true elsif line.match(/^Host /) delete_block = false end newlines << line if !delete_block @@ -1984,10 +2067,13 @@ MU::Cloud::DNSZone.createRecordsFromConfig(dnscfg) end end MU::MommaCat.removeHostFromSSHConfig(node) + if server and server.canonicalIP + MU::MommaCat.removeIPFromSSHKnownHosts(server.canonicalIP) + end # XXX add names paramater with useful stuff MU::MommaCat.addHostToSSHConfig( server, ssh_owner: server.deploy.mu_user, ssh_dir: Etc.getpwnam(server.deploy.mu_user).dir+"/.ssh" @@ -2572,11 +2658,11 @@ end } update_servers = update_servers - skip end - return if update_servers.size < 1 + return if MU.inGem? || update_servers.size < 1 threads = [] parent_thread_id = Thread.current.object_id update_servers.each { |sibling| threads << Thread.new { Thread.abort_on_exception = true @@ -2688,13 +2774,20 @@ MU.log "Starting Momma Cat on port #{MU.mommaCatPort}, logging to #{daemonLogFile}, PID file #{daemonPidFile}" origdir = Dir.getwd Dir.chdir(MU.myRoot+"/modules") # XXX what's the safest way to find the 'bundle' executable in both gem and non-gem installs? - cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start} + if MU.inGem? + cmd = %Q{thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start} + else + cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start} + end + MU.log cmd, MU::NOTICE + output = %x{#{cmd}} + Dir.chdir(origdir) retries = 0 begin sleep 1 @@ -2780,10 +2873,11 @@ # @param force [Boolean]: Save even if +no_artifacts+ is set # @param origin [Hash]: Optional blob of data indicating how this deploy was created def save!(triggering_node = nil, force: false, origin: nil) return if @no_artifacts and !force + MU::MommaCat.deploy_struct_semaphore.synchronize { MU.log "Saving deployment #{MU.deploy_id}", MU::DEBUG if !Dir.exist?(deploy_dir) MU.log "Creating #{deploy_dir}", MU::DEBUG @@ -2832,9 +2926,10 @@ raise MuError, "Got #{e.message} at #{e.error_char.dump} (#{e.source_encoding_name} => #{e.destination_encoding_name}) trying to save deployment" end deploy.flock(File::LOCK_UN) deploy.close @need_deploy_flush = false + MU::MommaCat.updateLitter(@deploy_id, self) end if !@original_config.nil? and @original_config.is_a?(Hash) config = File.new("#{deploy_dir}/basket_of_kittens.json", File::CREAT|File::TRUNC|File::RDWR, 0600) config.puts JSON.pretty_generate(MU::Config.manxify(@original_config))