modules/mu/clouds/aws/server.rb in cloud-mu-1.9.0.pre.beta vs modules/mu/clouds/aws/server.rb in cloud-mu-2.0.0.pre.alpha
- old
+ new
@@ -208,19 +208,19 @@
# @param device [String]: The OS-level device name of the volume.
# @param tag_name [String]: The name of the tag to attach.
# @param tag_value [String]: The value of the tag to attach.
# @param region [String]: The cloud provider region
# @return [void]
- def self.tagVolumes(instance_id, device: nil, tag_name: "MU-ID", tag_value: MU.deploy_id, region: MU.curRegion)
- MU::Cloud::AWS.ec2(region).describe_volumes(filters: [name: "attachment.instance-id", values: [instance_id]]).each { |vol|
+ def self.tagVolumes(instance_id, device: nil, tag_name: "MU-ID", tag_value: MU.deploy_id, region: MU.curRegion, credentials: nil)
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_volumes(filters: [name: "attachment.instance-id", values: [instance_id]]).each { |vol|
vol.volumes.each { |volume|
volume.attachments.each { |attachment|
vol_parent = attachment.instance_id
vol_id = attachment.volume_id
vol_dev = attachment.device
if vol_parent == instance_id and (vol_dev == device or device.nil?)
- MU::MommaCat.createTag(vol_id, tag_name, tag_value, region: region)
+ MU::MommaCat.createTag(vol_id, tag_name, tag_value, region: region, credentials: credentials)
break
end
}
}
}
@@ -244,23 +244,23 @@
else
MU.log "Node creation complete for #{@config['name']}"
end
MU::MommaCat.unlock(instance.instance_id+"-create")
else
- MU::MommaCat.createStandardTags(instance.instance_id, region: @config['region'])
- MU::MommaCat.createTag(instance.instance_id, "Name", @mu_name, region: @config['region'])
+ MU::MommaCat.createStandardTags(instance.instance_id, region: @config['region'], credentials: @config['credentials'])
+ MU::MommaCat.createTag(instance.instance_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
end
done = true
rescue Exception => e
if !instance.nil? and !done
MU.log "Aborted before I could finish setting up #{@config['name']}, cleaning it up. Stack trace will print once cleanup is complete.", MU::WARN if !@deploy.nocleanup
MU::MommaCat.unlockAll
if !@deploy.nocleanup
parent_thread_id = Thread.current.object_id
Thread.new {
MU.dupGlobals(parent_thread_id)
- MU::Cloud::AWS::Server.cleanup(noop: false, ignoremaster: false, skipsnapshots: true)
+ MU::Cloud::AWS::Server.cleanup(noop: false, ignoremaster: false, region: @config['region'], credentials: @config['credentials'], flags: { "skipsnapshots" => true } )
}
end
end
raise e
end
@@ -284,11 +284,10 @@
:max_count => 1
}
arn = nil
if @config['generate_iam_role']
-# @config['iam_role'], @cfm_role_name, @cfm_prof_name, arn = MU::Cloud::AWS::Server.createIAMProfile(@mu_name, base_profile: @config['iam_role'], extra_policies: @config['iam_policies'])
role = @deploy.findLitterMate(name: @config['name'], type: "roles")
s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file|
'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/'+file
}
role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
@@ -347,14 +346,14 @@
if !@userdata.nil? and !@userdata.empty?
instance_descriptor[:user_data] = Base64.encode64(@userdata)
end
- MU::Cloud::AWS::Server.waitForAMI(@config["ami_id"], region: @config['region'])
+ MU::Cloud::AWS::Server.waitForAMI(@config["ami_id"], region: @config['region'], credentials: @config['credentials'])
# Figure out which devices are embedded in the AMI already.
- image = MU::Cloud::AWS.ec2(@config['region']).describe_images(image_ids: [@config["ami_id"]]).images.first
+ image = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_images(image_ids: [@config["ami_id"]]).images.first
ext_disks = {}
if !image.block_device_mappings.nil?
image.block_device_mappings.each { |disk|
if !disk.device_name.nil? and !disk.device_name.empty? and !disk.ebs.nil? and !disk.ebs.empty?
ext_disks[disk.device_name] = MU.structToHash(disk.ebs)
@@ -389,11 +388,11 @@
# instance_descriptor.delete(:block_device_mappings)
# end
retries = 0
begin
- response = MU::Cloud::AWS.ec2(@config['region']).run_instances(instance_descriptor)
+ response = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).run_instances(instance_descriptor)
rescue Aws::EC2::Errors::InvalidGroupNotFound, Aws::EC2::Errors::InvalidSubnetIDNotFound, Aws::EC2::Errors::InvalidParameterValue => e
if retries < 10
if retries > 7
MU.log "Seeing #{e.inspect} while trying to launch #{node}, retrying a few more times...", MU::WARN, details: instance_descriptor
end
@@ -417,44 +416,44 @@
return if @cloud_id.nil?
if hard
groupname = nil
if !@config['basis'].nil?
- resp = MU::Cloud::AWS.autoscale(@config['region']).describe_auto_scaling_instances(
+ resp = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_instances(
instance_ids: [@cloud_id]
)
groupname = resp.auto_scaling_instances.first.auto_scaling_group_name
MU.log "Pausing Autoscale processes in #{groupname}", MU::NOTICE
- MU::Cloud::AWS.autoscale(@config['region']).suspend_processes(
+ MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).suspend_processes(
auto_scaling_group_name: groupname
)
end
begin
MU.log "Stopping #{@mu_name} (#{@cloud_id})", MU::NOTICE
- MU::Cloud::AWS.ec2(@config['region']).stop_instances(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).stop_instances(
instance_ids: [@cloud_id]
)
- MU::Cloud::AWS.ec2(@config['region']).wait_until(:instance_stopped, instance_ids: [@cloud_id]) do |waiter|
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).wait_until(:instance_stopped, instance_ids: [@cloud_id]) do |waiter|
waiter.before_attempt do |attempts|
MU.log "Waiting for #{@mu_name} to stop for hard reboot"
end
end
MU.log "Starting #{@mu_name} (#{@cloud_id})"
- MU::Cloud::AWS.ec2(@config['region']).start_instances(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).start_instances(
instance_ids: [@cloud_id]
)
ensure
if !groupname.nil?
MU.log "Resuming Autoscale processes in #{groupname}", MU::NOTICE
- MU::Cloud::AWS.autoscale(@config['region']).resume_processes(
+ MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).resume_processes(
auto_scaling_group_name: groupname
)
end
end
else
MU.log "Rebooting #{@mu_name} (#{@cloud_id})"
- MU::Cloud::AWS.ec2(@config['region']).reboot_instances(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).reboot_instances(
instance_ids: [@cloud_id]
)
end
end
@@ -466,11 +465,11 @@
# up in MU::MommaCat's ssh config wangling
ssh_keydir = Etc.getpwuid(Process.uid).dir+"/.ssh"
return nil if @config.nil? or @deploy.nil?
nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
- if !@config["vpc"].nil? and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'])
+ if !@config["vpc"].nil? and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
if !@nat.nil?
if @nat.is_a?(Struct) && @nat.nat_gateway_id && @nat.nat_gateway_id.start_with?("nat-")
raise MuError, "Configured to use NAT Gateway, but I have no route to instance. Either use Bastion, or configure VPC peering"
end
@@ -514,22 +513,22 @@
raise MuError, "Couldn't find instance #{@mu_name} (#{@cloud_id})" if !instance
@cloud_id = instance.instance_id
return false if !MU::MommaCat.lock(instance.instance_id+"-orchestrate", true)
return false if !MU::MommaCat.lock(instance.instance_id+"-groom", true)
- MU::MommaCat.createStandardTags(instance.instance_id, region: @config['region'])
- MU::MommaCat.createTag(instance.instance_id, "Name", node, region: @config['region'])
+ MU::MommaCat.createStandardTags(instance.instance_id, region: @config['region'], credentials: @config['credentials'])
+ MU::MommaCat.createTag(instance.instance_id, "Name", node, region: @config['region'], credentials: @config['credentials'])
if @config['optional_tags']
MU::MommaCat.listOptionalTags.each { |key, value|
- MU::MommaCat.createTag(instance.instance_id, key, value, region: @config['region'])
+ MU::MommaCat.createTag(instance.instance_id, key, value, region: @config['region'], credentials: @config['credentials'])
}
end
if !@config['tags'].nil?
@config['tags'].each { |tag|
- MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: @config['region'])
+ MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
}
end
MU.log "Tagged #{node} (#{instance.instance_id}) with MU-ID=#{MU.deploy_id}", MU::DEBUG
# Make double sure we don't lose a cached mu_windows_name value.
@@ -550,11 +549,11 @@
if retries % 3 == 0
MU.log "Waiting for EC2 instance #{node} (#{@cloud_id}) to be ready...", MU::NOTICE
end
sleep 40
# Get a fresh AWS descriptor
- instance = MU::Cloud::Server.find(cloud_id: @cloud_id, region: @config['region']).values.first
+ instance = MU::Cloud::Server.find(cloud_id: @cloud_id, region: @config['region'], credentials: @config['credentials']).values.first
if instance and instance.state.name == "terminated"
raise MuError, "EC2 instance #{node} (#{@cloud_id}) terminating during bootstrap!"
end
end
rescue Aws::EC2::Errors::ServiceError => e
@@ -570,10 +569,12 @@
punchAdminNAT
# If we came up via AutoScale, the Alarm module won't have had our
# instance ID to associate us with itself. So invoke that here.
+ # XXX might be possible to do this with regular alarm resources and
+ # dependencies now
if !@config['basis'].nil? and @config["alarms"] and !@config["alarms"].empty?
@config["alarms"].each { |alarm|
alarm_obj = MU::MommaCat.findStray(
"AWS",
"alarms",
@@ -582,12 +583,12 @@
name: alarm['name']
).first
alarm["dimensions"] = [{:name => "InstanceId", :value => @cloud_id}]
if alarm["enable_notifications"]
- topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"])
- MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"])
+ topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"], credentials: @config['credentials'])
+ MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"], credentials: @config["credentials"])
alarm["alarm_actions"] = [topic_arn]
alarm["ok_actions"] = [topic_arn]
end
alarm_name = alarm_obj ? alarm_obj.cloud_id : "#{node}-#{alarm['name']}".upcase
@@ -604,11 +605,12 @@
period: alarm["period"],
unit: alarm["unit"],
evaluation_periods: alarm["evaluation_periods"],
threshold: alarm["threshold"],
comparison_operator: alarm["comparison_operator"],
- region: @config["region"]
+ region: @config["region"],
+ credentials: @config['credentials']
)
}
end
# We have issues sometimes where our dns_records are pointing at the wrong node name and IP address.
@@ -633,27 +635,27 @@
@named = true
end
if !@config['src_dst_check'] and !@config["vpc"].nil?
MU.log "Disabling source_dest_check #{node} (making it NAT-worthy)"
- MU::Cloud::AWS.ec2(@config['region']).modify_instance_attribute(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
instance_id: @cloud_id,
source_dest_check: {:value => false}
)
end
# Set console termination protection. Autoscale nodes won't set this
# by default.
- MU::Cloud::AWS.ec2(@config['region']).modify_instance_attribute(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_instance_attribute(
instance_id: @cloud_id,
disable_api_termination: {:value => true}
)
has_elastic_ip = false
if !instance.public_ip_address.nil?
begin
- resp = MU::Cloud::AWS.ec2((@config['region'])).describe_addresses(public_ips: [instance.public_ip_address])
+ resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(public_ips: [instance.public_ip_address])
if resp.addresses.size > 0 and resp.addresses.first.instance_id == @cloud_id
has_elastic_ip = true
end
rescue Aws::EC2::Errors::InvalidAddressNotFound => e
# XXX this is ok to ignore, it means the public IP isn't Elastic
@@ -735,11 +737,11 @@
end
end
end
nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name = getSSHConfig
- if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'])
+ if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
raise MuError, "#{node} is in a private subnet (#{subnet}), but has no NAT host configured, and I have no other route to it"
end
# If we've asked for additional subnets (and this @config is not a
# member of a Server Pool, which has different semantics), create
@@ -747,27 +749,27 @@
if !@config['vpc']['subnets'].nil? and @config['basis'].nil?
device_index = 1
@vpc.subnets { |subnet|
subnet_id = subnet.cloud_id
MU.log "Adding network interface on subnet #{subnet_id} for #{node}"
- iface = MU::Cloud::AWS.ec2(@config['region']).create_network_interface(subnet_id: subnet_id).network_interface
- MU::MommaCat.createStandardTags(iface.network_interface_id, region: @config['region'])
- MU::MommaCat.createTag(iface.network_interface_id, "Name", node+"-ETH"+device_index.to_s, region: @config['region'])
+ iface = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_network_interface(subnet_id: subnet_id).network_interface
+ MU::MommaCat.createStandardTags(iface.network_interface_id, region: @config['region'], credentials: @config['credentials'])
+ MU::MommaCat.createTag(iface.network_interface_id, "Name", node+"-ETH"+device_index.to_s, region: @config['region'], credentials: @config['credentials'])
if @config['optional_tags']
MU::MommaCat.listOptionalTags.each { |key, value|
- MU::MommaCat.createTag(iface.network_interface_id, key, value, region: @config['region'])
+ MU::MommaCat.createTag(iface.network_interface_id, key, value, region: @config['region'], credentials: @config['credentials'])
}
end
if !@config['tags'].nil?
@config['tags'].each { |tag|
- MU::MommaCat.createTag(iface.network_interface_id, tag['key'], tag['value'], region: @config['region'])
+ MU::MommaCat.createTag(iface.network_interface_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
}
end
- MU::Cloud::AWS.ec2(@config['region']).attach_network_interface(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_network_interface(
network_interface_id: iface.network_interface_id,
instance_id: instance.instance_id,
device_index: device_index
)
device_index = device_index + 1
@@ -803,33 +805,33 @@
# Generic deploy ID tag
# tagVolumes(instance.instance_id)
# Tag volumes with all our standard tags.
# Maybe replace tagVolumes with this? There is one more place tagVolumes is called from
- volumes = MU::Cloud::AWS.ec2(@config['region']).describe_volumes(filters: [name: "attachment.instance-id", values: [instance.instance_id]])
+ volumes = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(filters: [name: "attachment.instance-id", values: [instance.instance_id]])
volumes.each { |vol|
vol.volumes.each { |volume|
volume.attachments.each { |attachment|
MU::MommaCat.listStandardTags.each_pair { |key, value|
- MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'])
+ MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
if attachment.device == "/dev/sda" or attachment.device == "/dev/sda1"
- MU::MommaCat.createTag(attachment.volume_id, "Name", "ROOT-#{MU.deploy_id}-#{@config["name"].upcase}", region: @config['region'])
+ MU::MommaCat.createTag(attachment.volume_id, "Name", "ROOT-#{MU.deploy_id}-#{@config["name"].upcase}", region: @config['region'], credentials: @config['credentials'])
else
- MU::MommaCat.createTag(attachment.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{attachment.device.upcase}", region: @config['region'])
+ MU::MommaCat.createTag(attachment.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{attachment.device.upcase}", region: @config['region'], credentials: @config['credentials'])
end
}
if @config['optional_tags']
MU::MommaCat.listOptionalTags.each { |key, value|
- MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'])
+ MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
}
end
if @config['tags']
@config['tags'].each { |tag|
- MU::MommaCat.createTag(attachment.volume_id, tag['key'], tag['value'], region: @config['region'])
+ MU::MommaCat.createTag(attachment.volume_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
}
end
}
}
}
@@ -840,46 +842,48 @@
if !@config['add_private_ips'].nil?
instance.network_interfaces.each { |int|
if int.private_ip_address == instance.private_ip_address and int.private_ip_addresses.size < (@config['add_private_ips'] + 1)
MU.log "Adding #{@config['add_private_ips']} extra private IP addresses to #{instance.instance_id}"
- MU::Cloud::AWS.ec2(@config['region']).assign_private_ip_addresses(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).assign_private_ip_addresses(
network_interface_id: int.network_interface_id,
secondary_private_ip_address_count: @config['add_private_ips'],
allow_reassignment: false
)
end
}
notify
end
begin
- if windows?
- # kick off certificate generation early; WinRM will need it
- cert, key = @deploy.nodeSSLCerts(self)
- if @config.has_key?("basis")
- @deploy.nodeSSLCerts(self, true)
- end
- if !@groomer.haveBootstrapped?
- session = getWinRMSession(50, 60, reboot_on_problems: true)
- initialWinRMTasks(session)
- begin
- session.close
- rescue Exception
- # this is allowed to fail- we're probably rebooting anyway
+ if @config['groom'].nil? or @config['groom']
+ if windows?
+ # kick off certificate generation early; WinRM will need it
+ cert, key = @deploy.nodeSSLCerts(self)
+ if @config.has_key?("basis")
+ @deploy.nodeSSLCerts(self, true)
end
- else # for an existing Windows node: WinRM, then SSH if it fails
- begin
- session = getWinRMSession(1, 60)
- rescue Exception # yeah, yeah
- session = getSSHSession(1, 60)
- # XXX maybe loop at least once if this also fails?
+ if !@groomer.haveBootstrapped?
+ session = getWinRMSession(50, 60, reboot_on_problems: true)
+ initialWinRMTasks(session)
+ begin
+ session.close
+ rescue Exception
+ # this is allowed to fail- we're probably rebooting anyway
+ end
+ else # for an existing Windows node: WinRM, then SSH if it fails
+ begin
+ session = getWinRMSession(1, 60)
+ rescue Exception # yeah, yeah
+ session = getSSHSession(1, 60)
+ # XXX maybe loop at least once if this also fails?
+ end
end
+ else
+ session = getSSHSession(40, 30)
+ initialSSHTasks(session)
end
- else
- session = getSSHSession(40, 30)
- initialSSHTasks(session)
end
rescue BootstrapTempFail
sleep 45
retry
ensure
@@ -920,18 +924,20 @@
# See if this node already exists in our config management. If it does,
# we're done.
if @groomer.haveBootstrapped?
MU.log "Node #{node} has already been bootstrapped, skipping groomer setup.", MU::NOTICE
- @groomer.saveDeployData
+ if @config['groom'].nil? or @config['groom']
+ @groomer.saveDeployData
+ end
MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
MU::MommaCat.unlock(instance.instance_id+"-groom")
return true
end
begin
- @groomer.bootstrap
+ @groomer.bootstrap if @config['groom'].nil? or @config['groom']
rescue MU::Groomer::RunError
MU::MommaCat.unlock(instance.instance_id+"-groom")
MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
return false
end
@@ -952,15 +958,15 @@
# Locate an existing instance or instances and return an array containing matching AWS resource descriptors for those that match.
# @param cloud_id [String]: The cloud provider's identifier for this resource.
# @param region [String]: The cloud provider region
# @param tag_key [String]: A tag key to search.
# @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
- # @param ip [String]: An IP address associated with the instance
# @param flags [Hash]: Optional flags
# @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching instances
- def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, ip: nil, flags: {})
+ def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
# XXX put that 'ip' value into opts
+ ip ||= flags['ip']
instance = nil
if !region.nil?
regions = [region]
else
regions = MU::Cloud::AWS.listRegions
@@ -975,11 +981,11 @@
regions.each { |region|
search_threads << Thread.new {
MU.log "Hunting for instance with cloud id '#{cloud_id}' in #{region}", MU::DEBUG
retries = 0
begin
- MU::Cloud::AWS.ec2(region).describe_instances(
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_instances(
instance_ids: [cloud_id],
filters: [
{name: "instance-state-name", values: ["running", "pending"]}
]
).reservations.each { |resp|
@@ -1016,11 +1022,11 @@
# Ok, well, let's try looking it up by IP then
if instance.nil? and !ip.nil?
MU.log "Hunting for instance by IP '#{ip}'", MU::DEBUG
["ip-address", "private-ip-address"].each { |filter|
- response = MU::Cloud::AWS.ec2(region).describe_instances(
+ response = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_instances(
filters: [
{name: filter, values: [ip]},
{name: "instance-state-name", values: ["running", "pending"]}
]
).reservations.first
@@ -1033,11 +1039,11 @@
end
# Fine, let's try it by tag.
if !tag_value.nil?
MU.log "Searching for instance by tag '#{tag_key}=#{tag_value}'", MU::DEBUG
- MU::Cloud::AWS.ec2(region).describe_instances(
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_instances(
filters: [
{name: "tag:#{tag_key}", values: [tag_value]},
{name: "instance-state-name", values: ["running", "pending"]}
]
).reservations.each { |resp|
@@ -1156,11 +1162,11 @@
end
end
punchAdminNAT
- MU::Cloud::AWS::Server.tagVolumes(@cloud_id)
+ MU::Cloud::AWS::Server.tagVolumes(@cloud_id, credentials: @config['credentials'])
# If we have a loadbalancer configured, attach us to it
if !@config['loadbalancers'].nil?
if @loadbalancers.nil?
raise MuError, "#{@mu_name} is configured to use LoadBalancers, but none have been loaded by dependencies()"
@@ -1180,14 +1186,18 @@
# db.allowHost(@deploydata["public_ip_address"]+"/32")
# end
# }
# end
- @groomer.saveDeployData
+ if @config['groom'].nil? or @config['groom']
+ @groomer.saveDeployData
+ end
begin
- @groomer.run(purpose: "Full Initial Run", max_retries: 15, reboot_first_fail: windows?)
+ if @config['groom'].nil? or @config['groom']
+ @groomer.run(purpose: "Full Initial Run", max_retries: 15, reboot_first_fail: windows?, timeout: @config['groomer_timeout'])
+ end
rescue MU::Groomer::RunError => e
MU.log "Proceeding after failed initial Groomer run, but #{node} may not behave as expected!", MU::WARN, details: e.message
rescue Exception => e
MU.log "Caught #{e.inspect} on #{node} in an unexpected place (after @groomer.run on Full Initial Run)", MU::ERR
end
@@ -1219,36 +1229,38 @@
storage: @config['storage'],
exclude_storage: img_cfg['image_exclude_storage'],
copy_to_regions: img_cfg['copy_to_regions'],
make_public: img_cfg['public'],
region: @config['region'],
- tags: @config['tags'])
+ tags: @config['tags'],
+ credentials: @config['credentials']
+ )
@deploy.notify("images", @config['name'], {"image_id" => ami_id})
@config['image_created'] = true
if img_cfg['image_then_destroy']
- MU::Cloud::AWS::Server.waitForAMI(ami_id, region: @config['region'])
+ MU::Cloud::AWS::Server.waitForAMI(ami_id, region: @config['region'], credentials: @config['credentials'])
MU.log "AMI #{ami_id} ready, removing source node #{node}"
- MU::Cloud::AWS::Server.terminateInstance(id: @cloud_id, region: @config['region'], deploy_id: @deploy.deploy_id, mu_name: @mu_name)
+ MU::Cloud::AWS::Server.terminateInstance(id: @cloud_id, region: @config['region'], deploy_id: @deploy.deploy_id, mu_name: @mu_name, credentials: @config['credentials'])
destroy
end
end
MU::MommaCat.unlock(@cloud_id+"-groom")
end
# Canonical Amazon Resource Number for this resource
# @return [String]
def arn
- "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU.account_number+":instance/"+@cloud_id
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":instance/"+@cloud_id
end
def cloud_desc
max_retries = 5
retries = 0
if !@cloud_id.nil?
begin
- return MU::Cloud::AWS.ec2(@config['region']).describe_instances(instance_ids: [@cloud_id]).reservations.first.instances.first
+ return MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(instance_ids: [@cloud_id]).reservations.first.instances.first
rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
return nil
rescue NoMethodError => e
if retries >= max_retries
raise MuError, "Couldn't get a cloud descriptor for #{@mu_name} (#{@cloud_id})"
@@ -1289,11 +1301,11 @@
end
# Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
# which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
# The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
- if MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region']) or @deploydata["public_ip_address"].nil?
+ if MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials']) or @deploydata["public_ip_address"].nil?
@config['canonical_ip'] = instance.private_ip_address
@deploydata["private_ip_address"] = instance.private_ip_address
return instance.private_ip_address
else
@config['canonical_ip'] = instance.public_ip_address
@@ -1309,11 +1321,11 @@
# @param exclude_storage [Boolean]: Do not include the storage device profile of the running instance when creating this image.
# @param region [String]: The cloud provider region
# @param copy_to_regions [Array<String>]: Copy the resulting AMI into the listed regions.
# @param tags [Array<String>]: Extra/override tags to apply to the image.
# @return [String]: The cloud provider identifier of the new machine image.
- def self.createImage(name: nil, instance_id: nil, storage: {}, exclude_storage: false, make_public: false, region: MU.curRegion, copy_to_regions: [], tags: [])
+ def self.createImage(name: nil, instance_id: nil, storage: {}, exclude_storage: false, make_public: false, region: MU.curRegion, copy_to_regions: [], tags: [], credentials: nil)
ami_descriptor = {
:instance_id => instance_id,
:name => name,
:description => "Image automatically generated by Mu from #{name}"
}
@@ -1342,53 +1354,53 @@
ami_descriptor[:block_device_mappings].concat(@ephemeral_mappings)
end
MU.log "Creating AMI from #{name}", details: ami_descriptor
resp = nil
begin
- resp = MU::Cloud::AWS.ec2(region).create_image(ami_descriptor)
+ resp = MU::Cloud::AWS.ec2(region: region).create_image(ami_descriptor)
rescue Aws::EC2::Errors::InvalidAMINameDuplicate => e
MU.log "AMI #{name} already exists, skipping", MU::WARN
return nil
end
ami = resp.image_id
- MU::MommaCat.createStandardTags(ami, region: region)
- MU::MommaCat.createTag(ami, "Name", name, region: region)
+ MU::MommaCat.createStandardTags(ami, region: region, credentials: credentials)
+ MU::MommaCat.createTag(ami, "Name", name, region: region, credentials: credentials)
MU.log "AMI of #{name} in region #{region}: #{ami}"
if make_public
- MU::Cloud::AWS::Server.waitForAMI(ami, region: region)
- MU::Cloud::AWS.ec2(region).modify_image_attribute(
+ MU::Cloud::AWS::Server.waitForAMI(ami, region: region, credentials: credentials)
+ MU::Cloud::AWS.ec2(region: region).modify_image_attribute(
image_id: ami,
launch_permission: {add: [{group: "all"}]},
attribute: "launchPermission"
)
end
copythreads = []
if !copy_to_regions.nil? and copy_to_regions.size > 0
parent_thread_id = Thread.current.object_id
- MU::Cloud::AWS::Server.waitForAMI(ami, region: region) if !make_public
+ MU::Cloud::AWS::Server.waitForAMI(ami, region: region, credentials: credentials) if !make_public
copy_to_regions.each { |r|
next if r == region
copythreads << Thread.new {
MU.dupGlobals(parent_thread_id)
- copy = MU::Cloud::AWS.ec2(r).copy_image(
+ copy = MU::Cloud::AWS.ec2(region: r).copy_image(
source_region: region,
source_image_id: ami,
name: name,
description: "Image automatically generated by Mu from #{name}"
)
MU.log "Initiated copy of #{ami} from #{region} to #{r}: #{copy.image_id}"
MU::MommaCat.createStandardTags(copy.image_id, region: r)
- MU::MommaCat.createTag(copy.image_id, "Name", name, region: r)
+ MU::MommaCat.createTag(copy.image_id, "Name", name, region: r, credentials: credentials)
if !tags.nil?
tags.each { |tag|
- MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: r)
+ MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: r, credentials: credentials)
}
end
- MU::Cloud::AWS::Server.waitForAMI(copy.image_id, region: r)
+ MU::Cloud::AWS::Server.waitForAMI(copy.image_id, region: r, credentials: credentials)
if make_public
- MU::Cloud::AWS.ec2(r).modify_image_attribute(
+ MU::Cloud::AWS.ec2(region: r).modify_image_attribute(
image_id: copy.image_id,
launch_permission: {add: [{group: "all"}]},
attribute: "launchPermission"
)
end
@@ -1406,16 +1418,16 @@
# Given a cloud platform identifier for a machine image, wait until it's
# flagged as ready.
# @param image_id [String]: The machine image to wait for.
# @param region [String]: The cloud provider region
- def self.waitForAMI(image_id, region: MU.curRegion)
+ def self.waitForAMI(image_id, region: MU.curRegion, credentials: nil)
MU.log "Checking to see if AMI #{image_id} is available", MU::DEBUG
retries = 0
begin
- images = MU::Cloud::AWS.ec2(region).describe_images(image_ids: [image_id]).images
+ images = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_images(image_ids: [image_id]).images
if images.nil? or images.size == 0
raise MuError, "No such AMI #{image_id} found"
end
state = images.first.state
if state == "failed"
@@ -1495,11 +1507,11 @@
ssh_key_name = @deploy.ssh_key_name
retries = 0
MU.log "Waiting for Windows instance password to be set by Amazon and flagged as available from the API. Note- if you're using a source AMI that already has its password set, this may fail. You'll want to set use_cloud_provider_windows_password to false if this is the case.", MU::NOTICE
begin
- MU::Cloud::AWS.ec2(@config['region']).wait_until(:password_data_available, instance_id: @cloud_id) do |waiter|
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).wait_until(:password_data_available, instance_id: @cloud_id) do |waiter|
waiter.max_attempts = 60
waiter.before_attempt do |attempts|
MU.log "Waiting for Windows password data to be available for node #{@mu_name}", MU::NOTICE if attempts % 5 == 0
end
# waiter.before_wait do |attempts, resp|
@@ -1515,11 +1527,11 @@
MU.log "wait_until(:password_data_available, instance_id: #{@cloud_id}) in #{@config['region']} never returned- this image may not be configured to have its password set by AWS.", MU::ERR
return nil
end
end
- resp = MU::Cloud::AWS.ec2(@config['region']).get_password_data(instance_id: @cloud_id)
+ resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).get_password_data(instance_id: @cloud_id)
encrypted_password = resp.password_data
# Note: This is already implemented in the decrypt_windows_password API call
decoded = Base64.decode64(encrypted_password)
pem_bytes = File.open("#{ssh_keydir}/#{ssh_key_name}", 'rb') { |f| f.read }
@@ -1542,13 +1554,13 @@
filters << {name: "domain", values: ["standard"]}
end
filters << {name: "public-ip", values: [ip]} if ip != nil
if filters.size > 0
- resp = MU::Cloud::AWS.ec2(region).describe_addresses(filters: filters)
+ resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(filters: filters)
else
- resp = MU::Cloud::AWS.ec2(region).describe_addresses()
+ resp = MU::Cloud::AWS.ec2(region: region).describe_addresses()
end
resp.addresses.each { |address|
return address if (address.network_interface_id.nil? || address.network_interface_id.empty?) && !@eips_used.include?(address.public_ip)
}
if ip != nil
@@ -1557,14 +1569,14 @@
else
raise MuError, "Requested EIP #{ip}, but no such IP exists or is available in EC2 Classic"
end
end
if !classic
- resp = MU::Cloud::AWS.ec2(region).allocate_address(domain: "vpc")
+ resp = MU::Cloud::AWS.ec2(region: region).allocate_address(domain: "vpc")
new_ip = resp.public_ip
else
- new_ip = MU::Cloud::AWS.ec2(region).allocate_address().public_ip
+ new_ip = MU::Cloud::AWS.ec2(region: region).allocate_address().public_ip
end
filters = [{name: "public-ip", values: [new_ip]}]
if resp.domain
filters << {name: "domain", values: [resp.domain]}
end rescue NoMethodError
@@ -1576,11 +1588,11 @@
begin
begin
sleep 5
- resp = MU::Cloud::AWS.ec2(region).describe_addresses(
+ resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(
filters: filters
)
addr = resp.addresses.first
end while resp.addresses.size < 1 or addr.public_ip.nil?
rescue NoMethodError
@@ -1600,11 +1612,11 @@
if @cloud_id.nil? or @cloud_id.empty?
MU.log "#{self} didn't have a cloud id, couldn't determine 'active?' status", MU::ERR
return true
end
az = nil
- MU::Cloud::AWS.ec2(@config['region']).describe_instances(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(
instance_ids: [@cloud_id]
).reservations.each { |resp|
if !resp.nil? and !resp.instances.nil?
resp.instances.each { |instance|
az = instance.placement.availability_zone
@@ -1616,39 +1628,39 @@
}
}
end
}
MU.log "Creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
- creation = MU::Cloud::AWS.ec2(@config['region']).create_volume(
+ creation = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_volume(
availability_zone: az,
size: size,
volume_type: type
)
begin
sleep 3
- creation = MU::Cloud::AWS.ec2(@config['region']).describe_volumes(volume_ids: [creation.volume_id]).volumes.first
+ creation = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(volume_ids: [creation.volume_id]).volumes.first
if !["creating", "available"].include?(creation.state)
raise MuError, "Saw state '#{creation.state}' while creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
end
end while creation.state != "available"
if @deploy
MU::MommaCat.listStandardTags.each_pair { |key, value|
- MU::MommaCat.createTag(creation.volume_id, key, value, region: @config['region'])
+ MU::MommaCat.createTag(creation.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
}
- MU::MommaCat.createTag(creation.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{dev.upcase}", region: @config['region'])
+ MU::MommaCat.createTag(creation.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{dev.upcase}", region: @config['region'], credentials: @config['credentials'])
end
- attachment = MU::Cloud::AWS.ec2(@config['region']).attach_volume(
+ attachment = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_volume(
device: dev,
instance_id: @cloud_id,
volume_id: creation.volume_id
)
begin
sleep 3
- attachment = MU::Cloud::AWS.ec2(@config['region']).describe_volumes(volume_ids: [attachment.volume_id]).volumes.first.attachments.first
+ attachment = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_volumes(volume_ids: [attachment.volume_id]).volumes.first.attachments.first
if !["attaching", "attached"].include?(attachment.state)
raise MuError, "Saw state '#{creation.state}' while creating #{size}GB #{type} volume on #{dev} for #{@cloud_id}"
end
end while attachment.state != "attached"
end
@@ -1660,11 +1672,11 @@
if @cloud_id.nil? or @cloud_id.empty?
MU.log "#{self} didn't have a #{@cloud_id}, couldn't determine 'active?' status", MU::ERR
return true
end
begin
- MU::Cloud::AWS.ec2(@config['region']).describe_instances(
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(
instance_ids: [@cloud_id]
).reservations.each { |resp|
if !resp.nil? and !resp.instances.nil?
resp.instances.each { |instance|
if instance.state.name == "terminated" or
@@ -1692,11 +1704,11 @@
MU.log "associateElasticIp called: #{instance_id}, classic: #{classic}, ip: #{ip}, region: #{region}", MU::DEBUG
elastic_ip = nil
@eip_semaphore.synchronize {
if !ip.nil?
filters = [{name: "public-ip", values: [ip]}]
- resp = MU::Cloud::AWS.ec2(region).describe_addresses(filters: filters)
+ resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(filters: filters)
if @eips_used.include?(ip)
is_free = false
resp.addresses.each { |address|
if address.public_ip == ip and (address.instance_id.nil? and address.network_interface_id.nil?) or address.instance_id == instance_id
@eips_used.delete(ip)
@@ -1724,16 +1736,16 @@
MU.log "Associating Elastic IP #{elastic_ip.public_ip} with #{instance_id}", details: elastic_ip
}
attempts = 0
begin
if classic
- resp = MU::Cloud::AWS.ec2(region).associate_address(
+ resp = MU::Cloud::AWS.ec2(region: region).associate_address(
instance_id: instance_id,
public_ip: elastic_ip.public_ip
)
else
- resp = MU::Cloud::AWS.ec2(region).associate_address(
+ resp = MU::Cloud::AWS.ec2(region: region).associate_address(
instance_id: instance_id,
allocation_id: elastic_ip.allocation_id,
allow_reassociation: false
)
end
@@ -1745,11 +1757,11 @@
retry
end
raise MuError "#{e.message} associating #{elastic_ip.allocation_id} with #{instance_id}"
rescue Aws::EC2::Errors::ResourceAlreadyAssociated => e
# A previous association attempt may have succeeded, albeit slowly.
- resp = MU::Cloud::AWS.ec2(region).describe_addresses(
+ resp = MU::Cloud::AWS.ec2(region: region).describe_addresses(
allocation_ids: [elastic_ip.allocation_id]
)
first_addr = resp.addresses.first
if !first_addr.nil? and first_addr.instance_id == instance_id
MU.log "#{elastic_ip.public_ip} already associated with #{instance_id}", MU::WARN
@@ -1757,32 +1769,41 @@
MU.log "#{elastic_ip.public_ip} shows as already associated!", MU::ERR, details: resp
raise MuError, "#{elastic_ip.public_ip} shows as already associated with #{first_addr.instance_id}!"
end
end
- instance = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
+ instance = MU::Cloud::AWS.ec2(region: region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
waited = false
if instance.public_ip_address != elastic_ip.public_ip
waited = true
begin
sleep 10
MU.log "Waiting for Elastic IP association of #{elastic_ip.public_ip} to #{instance_id} to take effect", MU::NOTICE
- instance = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
+ instance = MU::Cloud::AWS.ec2(region: region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
end while instance.public_ip_address != elastic_ip.public_ip
end
MU.log "Elastic IP #{elastic_ip.public_ip} now associated with #{instance_id}" if waited
return elastic_ip.public_ip
end
+ # Does this resource type exist as a global (cloud-wide) artifact, or
+ # is it localized to a region/zone?
+ # @return [Boolean]
+ def self.isGlobal?
+ false
+ end
+
# Remove all instances associated with the currently loaded deployment. Also cleans up associated volumes, droppings in the MU master's /etc/hosts and ~/.ssh, and in whatever Groomer was used.
# @param noop [Boolean]: If true, will only print what would be done
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
# @param region [String]: The cloud provider region
# @return [void]
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, skipsnapshots: false, onlycloud: false, flags: {})
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+ onlycloud = flags["onlycloud"]
+ skipsnapshots = flags["skipsnapshots"]
tagfilters = [
{name: "tag:MU-ID", values: [MU.deploy_id]}
]
if !ignoremaster
tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
@@ -1792,11 +1813,11 @@
name_tags = Array.new
# Build a list of instances we need to clean up. We guard against
# accidental deletion here by requiring someone to have hand-terminated
# these, by default.
- resp = MU::Cloud::AWS.ec2(region).describe_instances(
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(
filters: tagfilters
)
return if resp.data.reservations.nil?
resp.data.reservations.each { |reservation|
@@ -1815,22 +1836,22 @@
threads = []
unterminated.each { |instance|
threads << Thread.new(instance) { |myinstance|
MU.dupGlobals(parent_thread_id)
Thread.abort_on_exception = true
- MU::Cloud::AWS::Server.terminateInstance(id: myinstance.instance_id, noop: noop, onlycloud: onlycloud, region: region, deploy_id: MU.deploy_id)
+ MU::Cloud::AWS::Server.terminateInstance(id: myinstance.instance_id, noop: noop, onlycloud: onlycloud, region: region, deploy_id: MU.deploy_id, credentials: credentials)
}
}
- resp = MU::Cloud::AWS.ec2(region).describe_volumes(
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_volumes(
filters: tagfilters
)
resp.data.volumes.each { |volume|
threads << Thread.new(volume) { |myvolume|
MU.dupGlobals(parent_thread_id)
Thread.abort_on_exception = true
- MU::Cloud::AWS::Server.delete_volume(myvolume, noop, skipsnapshots)
+ MU::Cloud::AWS::Server.delete_volume(myvolume, noop, skipsnapshots, credentials: credentials)
}
}
# Wait for all of the instances to finish cleanup before proceeding
threads.each { |t|
@@ -1841,16 +1862,16 @@
# Terminate an instance.
# @param instance [OpenStruct]: The cloud provider's description of the instance.
# @param id [String]: The cloud provider's identifier for the instance, to use if the full description is not available.
# @param region [String]: The cloud provider region
# @return [void]
- def self.terminateInstance(instance: nil, noop: false, id: nil, onlycloud: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil)
+ def self.terminateInstance(instance: nil, noop: false, id: nil, onlycloud: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, credentials: nil)
ips = Array.new
if !instance
if id
begin
- resp = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [id])
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
MU.log "Instance #{id} no longer exists", MU::WARN
end
if !resp.nil? and !resp.reservations.nil? and !resp.reservations.first.nil?
instance = resp.reservations.first.instances.first
@@ -1878,30 +1899,30 @@
cloud_id: id,
mu_name: mu_name
).first
begin
- MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [id])
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
MU.log "Instance #{id} no longer exists", MU::DEBUG
end
- if !server_obj.nil? and MU::Cloud::AWS.hosted and !MU::Cloud::AWS.isGovCloud?
+ if !server_obj.nil? and MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
# DNS cleanup is now done in MU::Cloud::DNSZone. Keeping this for now
cleaned_dns = false
mu_name = server_obj.mu_name
- mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu").values.first
+ mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: credentials).values.first
if !mu_zone.nil?
zone_rrsets = []
- rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: mu_zone.id)
+ rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: mu_zone.id)
rrsets.resource_record_sets.each{ |record|
zone_rrsets << record
}
# AWS API returns a maximum of 100 results. DNS zones are likely to have more than 100 records, lets page and make sure we grab all records in a given zone
while rrsets.next_record_name && rrsets.next_record_type
- rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: mu_zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
+ rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: mu_zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
rrsets.resource_record_sets.each{ |record|
zone_rrsets << record
}
end
end
@@ -2000,18 +2021,18 @@
MU.log "#{instance.instance_id} (#{name}) is in state #{instance.state.name}, waiting"
else
MU.log "Terminating #{instance.instance_id} (#{name}) #{noop}"
if !noop
begin
- MU::Cloud::AWS.ec2(region).modify_instance_attribute(
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_instance_attribute(
instance_id: instance.instance_id,
disable_api_termination: {value: false}
)
- MU::Cloud::AWS.ec2(region).terminate_instances(instance_ids: [instance.instance_id])
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).terminate_instances(instance_ids: [instance.instance_id])
# Small race window here with the state changing from under us
rescue Aws::EC2::Errors::IncorrectInstanceState => e
- resp = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [id])
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
if !resp.nil? and !resp.reservations.nil? and !resp.reservations.first.nil?
instance = resp.reservations.first.instances.first
if !instance.nil? and instance.state.name != "terminated" and instance.state.name != "terminating"
sleep 5
retry
@@ -2024,11 +2045,11 @@
end
end
end
while instance.state.name != "terminated" and !noop
sleep 30
- instance_response = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [instance.instance_id])
+ instance_response = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [instance.instance_id])
instance = instance_response.reservations.first.instances.first
end
MU.log "#{instance.instance_id} (#{name}) terminated" if !noop
end
end
@@ -2154,10 +2175,11 @@
ok = false
end
else
role = {
"name" => server["name"],
+ "credentials" => server["credentials"],
"can_assume" => [
{
"entity_id" => "ec2.amazonaws.com",
"entity_type" => "service"
}
@@ -2239,13 +2261,13 @@
# Destroy a volume.
# @param volume [OpenStruct]: The cloud provider's description of the volume.
# @param id [String]: The cloud provider's identifier for the volume, to use if the full description is not available.
# @param region [String]: The cloud provider region
# @return [void]
- def self.delete_volume(volume, noop, skipsnapshots, id: nil, region: MU.curRegion)
+ def self.delete_volume(volume, noop, skipsnapshots, id: nil, region: MU.curRegion, credentials: nil)
if !volume.nil?
- resp = MU::Cloud::AWS.ec2(region).describe_volumes(volume_ids: [volume.volume_id])
+ resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_volumes(volume_ids: [volume.volume_id])
volume = resp.data.volumes.first
end
name = ""
volume.tags.each { |tag|
name = tag.value if tag.key == "Name"
@@ -2258,18 +2280,28 @@
desc = "#{MU.deploy_id}-MUfinal (#{name})"
else
desc = "#{MU.deploy_id}-MUfinal"
end
- MU::Cloud::AWS.ec2(region).create_snapshot(
+ begin
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_snapshot(
volume_id: volume.volume_id,
description: desc
- )
+ )
+ rescue Aws::EC2::Errors::IncorrectState => e
+ if e.message.match(/'deleting'/)
+ MU.log "Cannot snapshot volume '#{name}', is already being deleted", MU::WARN
+ end
+ end
end
retries = 0
begin
- MU::Cloud::AWS.ec2(region).delete_volume(volume_id: volume.volume_id)
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).delete_volume(volume_id: volume.volume_id)
+ rescue Aws::EC2::Errors::IncorrectState => e
+ MU.log "Volume #{volume.volume_id} (#{name}) in incorrect state (#{e.message}), will retry", MU::WARN
+ sleep 30
+ retry
rescue Aws::EC2::Errors::InvalidVolumeNotFound
MU.log "Volume #{volume.volume_id} (#{name}) disappeared before I could remove it!", MU::WARN
rescue Aws::EC2::Errors::VolumeInUse
if retries < 10
volume.attachments.each { |attachment|