modules/mu/cleanup.rb in cloud-mu-2.1.0beta vs modules/mu/cleanup.rb in cloud-mu-3.0.0beta
- old
+ new
@@ -39,11 +39,11 @@
# @param onlycloud [Boolean]: Purge cloud resources, but skip purging all Mu master metadata, ssh keys, etc.
# @param verbosity [Integer]: Debug level for MU.log output
# @param web [Boolean]: Generate web-friendly output.
# @param ignoremaster [Boolean]: Ignore the tags indicating the originating MU master server when deleting.
# @return [void]
- def self.run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, verbosity: MU::Logger::NORMAL, web: false, ignoremaster: false, skipcloud: false, mommacat: nil)
+ def self.run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, verbosity: MU::Logger::NORMAL, web: false, ignoremaster: false, skipcloud: false, mommacat: nil, credsets: nil, regions: nil)
MU.setLogging(verbosity, web)
@noop = noop
@skipsnapshots = skipsnapshots
@onlycloud = onlycloud
@skipcloud = skipcloud
@@ -59,11 +59,11 @@
else
MU.setVar("dataDir", MU.mainDataDir)
end
- types_in_order = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection", "Habitat", "Folder"]
+ types_in_order = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"]
# Load up our deployment metadata
if !mommacat.nil?
@mommacat = mommacat
else
@@ -73,194 +73,233 @@
# key = OpenSSL::PKey::RSA.new(File.read("#{deploy_dir}/public_key"))
# deploy_secret = key.public_encrypt(File.read("#{deploy_dir}/deploy_secret"))
FileUtils.touch("#{deploy_dir}/.cleanup") if !@noop
else
MU.log "I don't see a deploy named #{deploy_id}.", MU::WARN
- MU.log "Known deployments:\n#{Dir.entries(deploy_dir).reject { |item| item.match(/^\./) or !File.exists?(deploy_dir+"/"+item+"/public_key") }.join("\n")}", MU::WARN
+ MU.log "Known deployments:\n#{Dir.entries(deploy_dir).reject { |item| item.match(/^\./) or !File.exist?(deploy_dir+"/"+item+"/public_key") }.join("\n")}", MU::WARN
MU.log "Searching for remnants of #{deploy_id}, though this may be an invalid MU-ID.", MU::WARN
end
- @mommacat = MU::MommaCat.new(deploy_id, mu_user: MU.mu_user)
+ @mommacat = MU::MommaCat.new(deploy_id, mu_user: MU.mu_user, delay_descriptor_load: true)
rescue Exception => e
MU.log "Can't load a deploy record for #{deploy_id} (#{e.inspect}), cleaning up resources by guesswork", MU::WARN, details: e.backtrace
MU.setVar("deploy_id", deploy_id)
+
end
end
+ regionsused = @mommacat.regionsUsed if @mommacat
+ credsused = @mommacat.credsUsed if @mommacat
+
if !@skipcloud
creds = {}
- MU::Cloud.supportedClouds.each { |cloud|
+ MU::Cloud.availableClouds.each { |cloud|
if $MU_CFG[cloud.downcase] and $MU_CFG[cloud.downcase].size > 0
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
creds[cloud] ||= {}
cloudclass.listCredentials.each { |credset|
+ next if credsets and credsets.size > 0 and !credsets.include?(credset)
+ next if credsused and credsused.size > 0 and !credsused.include?(credset)
+ MU.log "Will scan #{cloud} with credentials #{credset}"
creds[cloud][credset] = cloudclass.listRegions(credentials: credset)
}
end
}
+
parent_thread_id = Thread.current.object_id
deleted_nodes = 0
- @regionthreads = []
+ cloudthreads = []
keyname = "deploy-#{MU.deploy_id}"
-# XXX blindly checking for all of these resources in all clouds is now prohibitively slow. We should only do this when we don't see deployment metadata to work from.
- creds.each_pair { |provider, credsets|
- credsets.each_pair { |credset, regions|
- global_vs_region_semaphore = Mutex.new
- global_done = []
- regions.each { |r|
- @regionthreads << Thread.new {
- MU.dupGlobals(parent_thread_id)
- MU.setVar("curRegion", r)
- projects = []
- if $MU_CFG[provider.downcase][credset]["project"]
-# XXX GCP credential schema needs an array for projects
- projects << $MU_CFG[provider.downcase][credset]["project"]
- end
- if projects == []
- projects << "" # dummy
- MU.log "Checking for #{provider}/#{credset} resources from #{MU.deploy_id} in #{r}", MU::NOTICE
+ creds.each_pair { |provider, credsets_outer|
+ cloudthreads << Thread.new(provider, credsets_outer) { |cloud, credsets_inner|
+ MU.dupGlobals(parent_thread_id)
+ Thread.abort_on_exception = false
+ cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+ habitatclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("Habitat")
+ credsets_inner.each_pair { |credset, acct_regions|
+ next if credsused and !credsused.include?(credset)
+ global_vs_region_semaphore = Mutex.new
+ global_done = {}
+ habitats_done = {}
+ regionthreads = []
+ acct_regions.each { |r|
+ if regionsused
+ if regionsused.size > 0
+ next if !regionsused.include?(r)
+ else
+ next if r != cloudclass.myRegion(credset)
+ end
end
+ if regions and !regions.empty?
+ next if !regions.include?(r)
+ MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}...", MU::NOTICE
+ end
+ regionthreads << Thread.new {
+ MU.dupGlobals(parent_thread_id)
+ Thread.abort_on_exception = false
+ MU.setVar("curRegion", r)
+ projects = []
+ if $MU_CFG[cloud.downcase][credset]["project"]
+# XXX GCP credential schema needs an array for projects
+ projects << $MU_CFG[cloud.downcase][credset]["project"]
+ end
+ begin
+ projects.concat(cloudclass.listProjects(credset))
+ rescue NoMethodError
+ end
- # We do these in an order that unrolls dependent resources
- # sensibly, and we hit :Collection twice because AWS
- # CloudFormation sometimes fails internally.
- projectthreads = []
- projects.each { |project|
- projectthreads << Thread.new {
- MU.dupGlobals(parent_thread_id)
- MU.setVar("curRegion", r)
- if project != ""
- MU.log "Checking for #{provider}/#{credset} resources from #{MU.deploy_id} in #{r}, project #{project}", MU::NOTICE
- end
+ if projects == []
+ projects << "" # dummy
+ MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}", MU::NOTICE
+ end
+ projects.uniq!
- MU.dupGlobals(parent_thread_id)
- flags = {
- "project" => project,
- "onlycloud" => @onlycloud,
- "skipsnapshots" => @skipsnapshots,
- }
- types_in_order.each { |t|
- begin
- skipme = false
- global_vs_region_semaphore.synchronize {
- if Object.const_get("MU").const_get("Cloud").const_get(provider).const_get(t).isGlobal?
- if !global_done.include?(t)
- global_done << t
- flags['global'] = true
- else
- skipme = true
- end
- end
- }
- next if skipme
- rescue MU::Cloud::MuCloudResourceNotImplemented => e
- next
- rescue MU::MuError, NoMethodError => e
- MU.log e.message, MU::WARN
- next
- rescue ::Aws::EC2::Errors::AuthFailure => e
- # AWS has been having transient auth problems with ap-east-1 lately
- MU.log e.message+" in "+r, MU::ERR
- next
+ # We do these in an order that unrolls dependent resources
+ # sensibly, and we hit :Collection twice because AWS
+ # CloudFormation sometimes fails internally.
+ projectthreads = []
+ projects.each { |project|
+ next if !habitatclass.isLive?(project, credset)
+
+ projectthreads << Thread.new {
+ MU.dupGlobals(parent_thread_id)
+ MU.setVar("curRegion", r)
+ Thread.abort_on_exception = false
+ if project != ""
+ MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}, project #{project}", MU::NOTICE
end
- if @mommacat.nil? or @mommacat.numKittens(types: [t]) > 0
- if @mommacat
- found = @mommacat.findLitterMate(type: t, return_all: true, credentials: credset)
- flags['known'] ||= []
- if found.is_a?(Array)
- found.each { |k|
- flags['known'] << k.cloud_id
- }
- elsif found and found.is_a?(Hash)
- flags['known'] << found['cloud_id']
- elsif found
- flags['known'] << found.cloud_id
- end
+ MU.dupGlobals(parent_thread_id)
+ flags = {
+ "project" => project,
+ "onlycloud" => @onlycloud,
+ "skipsnapshots" => @skipsnapshots,
+ }
+ types_in_order.each { |t|
+ shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(t)
+ begin
+ skipme = false
+ global_vs_region_semaphore.synchronize {
+ MU::Cloud.loadCloudType(cloud, t)
+ if Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(t).isGlobal?
+ global_done[project] ||= []
+ if !global_done[project].include?(t)
+ global_done[project] << t
+ flags['global'] = true
+ else
+ skipme = true
+ end
+ end
+ }
+ next if skipme
+ rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented => e
+ next
+ rescue MU::MuError, NoMethodError => e
+ MU.log "While checking mu/clouds/#{cloud.downcase}/#{cloudclass.cfg_name} for global-ness in cleanup: "+e.message, MU::WARN
+ next
+ rescue ::Aws::EC2::Errors::AuthFailure, ::Google::Apis::ClientError => e
+ MU.log e.message+" in "+r, MU::ERR
+ next
end
+
begin
- resclass = Object.const_get("MU").const_get("Cloud").const_get(t)
- resclass.cleanup(
- noop: @noop,
- ignoremaster: @ignoremaster,
- region: r,
- cloud: provider,
- flags: flags,
- credentials: credset
- )
- rescue Seahorse::Client::NetworkingError => e
- MU.log "Service not available in AWS region #{r}, skipping", MU::DEBUG, details: e.message
+ self.call_cleanup(t, credset, cloud, flags, r)
+ rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented => e
+ next
end
- end
+ }
+ } # types_in_order.each { |t|
+ } # projects.each { |project|
+ projectthreads.each do |t|
+ t.join
+ end
+
+ # XXX move to MU::AWS
+ if cloud == "AWS"
+ resp = MU::Cloud::AWS.ec2(region: r, credentials: credset).describe_key_pairs(
+ filters: [{name: "key-name", values: [keyname]}]
+ )
+ resp.data.key_pairs.each { |keypair|
+ MU.log "Deleting key pair #{keypair.key_name} from #{r}"
+ MU::Cloud::AWS.ec2(region: r, credentials: credset).delete_key_pair(key_name: keypair.key_name) if !@noop
}
- }
- }
- projectthreads.each do |t|
- t.join
- end
+ end
+ } # regionthreads << Thread.new {
+ } # acct_regions.each { |r|
+ regionthreads.each do |t|
+ t.join
+ end
- # XXX move to MU::AWS
- if provider == "AWS"
- resp = MU::Cloud::AWS.ec2(region: r, credentials: credset).describe_key_pairs(
- filters: [{name: "key-name", values: [keyname]}]
- )
- resp.data.key_pairs.each { |keypair|
- MU.log "Deleting key pair #{keypair.key_name} from #{r}"
- MU::Cloud::AWS.ec2(region: r, credentials: credset).delete_key_pair(key_name: keypair.key_name) if !@noop
- }
- end
+ } # credsets.each_pair { |credset, acct_regions|
+ } # cloudthreads << Thread.new(provider, credsets) { |cloud, credsets_outer|
+ cloudthreads.each do |t|
+ t.join
+ end
+ } # creds.each_pair { |provider, credsets|
+
+
+ # Knock habitats and folders, which would contain the above resources,
+ # once they're all done.
+ creds.each_pair { |provider, credsets_inner|
+ credsets_inner.keys.each { |credset|
+ next if credsused and !credsused.include?(credset)
+ ["Habitat", "Folder"].each { |t|
+ flags = {
+ "onlycloud" => @onlycloud,
+ "skipsnapshots" => @skipsnapshots
}
+ self.call_cleanup(t, credset, provider, flags, nil)
}
}
}
- @regionthreads.each do |t|
- t.join
- end
- @projectthreads = []
-
-
- @projectthreads.each do |t|
- t.join
- end
-
MU::Cloud::Google.removeDeploySecretsAndRoles(MU.deploy_id)
# XXX port AWS equivalent behavior and add a MU::Cloud wrapper
+
+ creds.each_pair { |provider, credsets_inner|
+ cloudclass = Object.const_get("MU").const_get("Cloud").const_get(provider)
+ credsets_inner.keys.each { |c|
+ cloudclass.cleanDeploy(MU.deploy_id, credentials: c, noop: @noop)
+ }
+ }
end
# Scrub any residual Chef records with matching tags
- if !@onlycloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0)
- MU::Groomer::Chef.loadChefLib
- if File.exists?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
- Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
- end
- deadnodes = []
- Chef::Config[:environment] = MU.environment
- q = Chef::Search::Query.new
+ if !@onlycloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0) and !(Gem.paths and Gem.paths.home and !Dir.exist?("/opt/mu/lib"))
begin
- q.search("node", "tags_MU-ID:#{MU.deploy_id}").each { |item|
- next if item.is_a?(Integer)
- item.each { |node|
- deadnodes << node.name
+ MU::Groomer::Chef.loadChefLib
+ if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
+ Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
+ end
+ deadnodes = []
+ Chef::Config[:environment] = MU.environment
+ q = Chef::Search::Query.new
+ begin
+ q.search("node", "tags_MU-ID:#{MU.deploy_id}").each { |item|
+ next if item.is_a?(Integer)
+ item.each { |node|
+ deadnodes << node.name
+ }
}
- }
- rescue Net::HTTPServerException
- end
+ rescue Net::HTTPServerException
+ end
- begin
- q.search("node", "name:#{MU.deploy_id}-*").each { |item|
- next if item.is_a?(Integer)
- item.each { |node|
- deadnodes << node.name
+ begin
+ q.search("node", "name:#{MU.deploy_id}-*").each { |item|
+ next if item.is_a?(Integer)
+ item.each { |node|
+ deadnodes << node.name
+ }
}
+ rescue Net::HTTPServerException
+ end
+ MU.log "Missed some Chef resources in node cleanup, purging now", MU::NOTICE if deadnodes.size > 0
+ deadnodes.uniq.each { |node|
+ MU::Groomer::Chef.cleanup(node, [], noop)
}
- rescue Net::HTTPServerException
+ rescue LoadError
end
- MU.log "Missed some Chef resources in node cleanup, purging now", MU::NOTICE if deadnodes.size > 0
- deadnodes.uniq.each { |node|
- MU::Groomer::Chef.cleanup(node, [], noop)
- }
end
if !@onlycloud and !@noop and @mommacat
@mommacat.purge!
end
@@ -268,22 +307,22 @@
myhome = Etc.getpwuid(Process.uid).dir
sshdir = "#{myhome}/.ssh"
sshconf = "#{sshdir}/config"
ssharchive = "#{sshdir}/archive"
- Dir.mkdir(sshdir, 0700) if !Dir.exists?(sshdir) and !@noop
- Dir.mkdir(ssharchive, 0700) if !Dir.exists?(ssharchive) and !@noop
+ Dir.mkdir(sshdir, 0700) if !Dir.exist?(sshdir) and !@noop
+ Dir.mkdir(ssharchive, 0700) if !Dir.exist?(ssharchive) and !@noop
keyname = "deploy-#{MU.deploy_id}"
- if File.exists?("#{sshdir}/#{keyname}")
+ if File.exist?("#{sshdir}/#{keyname}")
MU.log "Moving #{sshdir}/#{keyname} to #{ssharchive}/#{keyname}"
if !@noop
File.rename("#{sshdir}/#{keyname}", "#{ssharchive}/#{keyname}")
end
end
- if File.exists?(sshconf) and File.open(sshconf).read.match(/\/deploy\-#{MU.deploy_id}$/)
+ if File.exist?(sshconf) and File.open(sshconf).read.match(/\/deploy\-#{MU.deploy_id}$/)
MU.log "Expunging #{MU.deploy_id} from #{sshconf}"
if !@noop
FileUtils.copy(sshconf, "#{ssharchive}/config-#{MU.deploy_id}")
File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f|
f.flock(File::LOCK_EX)
@@ -307,25 +346,29 @@
end
# XXX refactor with above? They're similar, ish.
hostsfile = "/etc/hosts"
if File.open(hostsfile).read.match(/ #{MU.deploy_id}\-/)
- MU.log "Expunging traces of #{MU.deploy_id} from #{hostsfile}"
- if !@noop
- FileUtils.copy(hostsfile, "#{hostsfile}.cleanup-#{deploy_id}")
- File.open(hostsfile, File::CREAT|File::RDWR, 0644) { |f|
- f.flock(File::LOCK_EX)
- newlines = Array.new
- f.readlines.each { |line|
- newlines << line if !line.match(/ #{MU.deploy_id}\-/)
+ if Process.uid == 0
+ MU.log "Expunging traces of #{MU.deploy_id} from #{hostsfile}"
+ if !@noop
+ FileUtils.copy(hostsfile, "#{hostsfile}.cleanup-#{deploy_id}")
+ File.open(hostsfile, File::CREAT|File::RDWR, 0644) { |f|
+ f.flock(File::LOCK_EX)
+ newlines = Array.new
+ f.readlines.each { |line|
+ newlines << line if !line.match(/ #{MU.deploy_id}\-/)
+ }
+ f.rewind
+ f.truncate(0)
+ f.puts(newlines)
+ f.flush
+ f.flock(File::LOCK_UN)
}
- f.rewind
- f.truncate(0)
- f.puts(newlines)
- f.flush
- f.flock(File::LOCK_UN)
- }
+ end
+ else
+ MU.log "Residual /etc/hosts entries for #{MU.deploy_id} must be removed by root user", MU::WARN
end
end
if !@noop and !@skipcloud
if $MU_CFG['aws'] and $MU_CFG['aws']['account_number']
@@ -351,8 +394,42 @@
if !@noop and !@skipcloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0)
# MU::MommaCat.syncMonitoringConfig
end
+ end
+
+ private
+
+ def self.call_cleanup(type, credset, provider, flags, region)
+ if @mommacat.nil? or @mommacat.numKittens(types: [type]) > 0
+ if @mommacat
+ found = @mommacat.findLitterMate(type: type, return_all: true, credentials: credset)
+ flags['known'] ||= []
+ if found.is_a?(Array)
+ found.each { |k|
+ flags['known'] << k.cloud_id
+ }
+ elsif found and found.is_a?(Hash)
+ flags['known'] << found['cloud_id']
+ elsif found
+ flags['known'] << found.cloud_id
+ end
+ end
+# begin
+ resclass = Object.const_get("MU").const_get("Cloud").const_get(type)
+
+ resclass.cleanup(
+ noop: @noop,
+ ignoremaster: @ignoremaster,
+ region: region,
+ cloud: provider,
+ flags: flags,
+ credentials: credset
+ )
+# rescue ::Seahorse::Client::NetworkingError => e
+# MU.log "Service not available in AWS region #{r}, skipping", MU::DEBUG, details: e.message
+# end
+ end
end
end #class
end #module