modules/mu/providers/aws.rb in cloud-mu-3.4.0 vs modules/mu/providers/aws.rb in cloud-mu-3.5.0

- old
+ new

@@ -14,11 +14,10 @@ require "net/http" require 'open-uri' require 'timeout' require 'inifile' -gem 'aws-sdk-core' autoload :Aws, "aws-sdk-core" module MU class Cloud @@ -52,18 +51,23 @@ # @param cloudobj [MU::Cloud] # @param _deploy [MU::MommaCat] def self.resourceInitHook(cloudobj, _deploy) class << self attr_reader :cloudformation_data + attr_reader :region end + return if !cloudobj cloudobj.instance_variable_set(:@cloudformation_data, {}) + + cloudobj.instance_variable_set(:@region, cloudobj.config['region']) end # Load some credentials for using the AWS API # @param name [String]: The name of the mu.yaml AWS credential set to use. If not specified, will use the default credentials, and set the global Aws.config credentials to those. # @return [Aws::Credentials] def self.loadCredentials(name = nil) + gem 'aws-sdk-core' @@creds_loaded ||= {} if name.nil? return @@creds_loaded["#default"] if @@creds_loaded["#default"] else @@ -122,27 +126,31 @@ elsif cred_cfg['credentials'] and !cred_cfg['credentials'].empty? # pull access key and secret from a vault begin vault, item = cred_cfg["credentials"].split(/:/) - data = MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h - if data["access_key"] and data["access_secret"] + data = if !vault or !item + raise MuError.new "AWS #{name} credentials field value '#{cred_cfg["credentials"]}' malformed, should be vaultname:itemname", details: cred_cfg + else + MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h + end + if data and data["access_key"] and data["access_secret"] cred_obj = Aws::Credentials.new( - cred_cfg['access_key'], cred_cfg['access_secret'] + data['access_key'], data['access_secret'] ) if name.nil? # Aws.config = { # access_key_id: data['access_key'], # secret_access_key: data['access_secret'], # region: cred_cfg['region'] # } end else - MU.log "AWS credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", MU::WARN + raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", details: cred_cfg end rescue MU::Groomer::MuNoSuchSecret - MU.log "AWS credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", MU::WARN + raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", details: cred_cfg end end if !cred_obj and hosted? # assume we've got an IAM profile and hope for the best @@ -184,10 +192,11 @@ # Given an AWS region, check the API to make sure it's a valid one # @param r [String] # @return [String] def self.validate_region(r, credentials: nil) + require "aws-sdk-ec2" begin MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name rescue ::Aws::EC2::Errors::UnauthorizedOperation => e MU.log "Got '#{e.message}' trying to validate region #{r} (hosted: #{hosted?.to_s})", MU::ERR, details: loadCredentials(credentials) raise MuError, "Got '#{e.message}' trying to validate region #{r} with credentials #{credentials ? credentials : "<default>"} (hosted: #{hosted?.to_s})" @@ -202,10 +211,11 @@ # @param optional [Boolean]: Whether to apply our optional generic tags # @param nametag [String]: A +Name+ tag to apply # @param othertags [Array<Hash>]: Miscellaneous custom tags, in Basket of Kittens style # @return [void] def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil) + require "aws-sdk-ec2" tags = [] MU::MommaCat.listStandardTags.each_pair { |name, value| tags << {key: name, value: value} if !value.nil? } if optional @@ -259,24 +269,37 @@ return nil if vpc.nil? or vpc.size == 0 @@myVPCObj = vpc.first @@myVPCObj end - # If we've configured AWS as a provider, or are simply hosted in AWS, + # If we've configured AWS as a provider, or are simply hosted in AWS, # decide what our default region is. - def self.myRegion(credentials = nil) - return @@myRegion_var if @@myRegion_var + def self.myRegion(credentials = nil, debug: false) + loglevel = debug ? MU::NOTICE : MU::DEBUG + if @@myRegion_var + MU.log "AWS.myRegion: returning #{@@myRegion_var} from cache", loglevel + return @@myRegion_var + end + MU.log "AWS.myRegion: credConfig", loglevel, details: credConfig + MU.log "AWS.myRegion: hosted?", loglevel, details: hosted?.to_s + MU.log "AWS.myRegion: ENV['EC2_REGION']", loglevel, details: ENV['EC2_REGION'] + MU.log "AWS.myRegion: $MU_CFG['aws']", loglevel, details: $MU_CFG['aws'] if credConfig.nil? and !hosted? and !ENV['EC2_REGION'] + MU.log "AWS.myRegion: nothing of use set, returning", loglevel return nil end if $MU_CFG and $MU_CFG['aws'] $MU_CFG['aws'].each_pair { |credset, cfg| + MU.log "AWS.myRegion: #{credset} != #{credentials} ?", loglevel, details: cfg next if credentials and credset != credentials + MU.log "AWS.myRegion: validating credset #{credset}", loglevel, details: cfg next if !cfg['region'] - if (cfg['default'] or !@@myRegion_var) and validate_region(cfg['region'], credentials: credset) + MU.log "AWS.myRegion: validation response", loglevel, details: validate_region(cfg['region'], credentials: credset) + if (cfg['default'] or !@@myRegion_var or $MU_CFG['aws'].size == 1) and validate_region(cfg['region'], credentials: credset) + MU.log "AWS.myRegion: liking this set", loglevel, details: cfg @@myRegion_var = cfg['region'] break if cfg['default'] or credentials end } elsif ENV.has_key?("EC2_REGION") and !ENV['EC2_REGION'].empty? and @@ -284,19 +307,25 @@ ( (ENV.has_key?("AWS_SECRET_ACCESS_KEY") and ENV.has_key?("AWS_SECRET_ACCESS_KEY") ) or (Aws.config['access_key'] and Aws.config['access_secret']) ) # Make sure this string is valid by way of the API + MU.log "AWS.myRegion: using ENV", loglevel, details: ENV @@myRegion_var = ENV['EC2_REGION'] end if hosted? and !@@myRegion_var # hacky, but useful in a pinch (and if we're hosted in AWS) az_str = MU::Cloud::AWS.getAWSMetaData("placement/availability-zone") + MU.log "AWS.myRegion: using hosted", loglevel, details: az_str @@myRegion_var = az_str.sub(/[a-z]$/i, "") if az_str end + if credConfig and credConfig["region"] + @@myRegion_var ||= credConfig["region"] + end + @@myRegion_var end # Is the region we're dealing with a GovCloud region? @@ -380,12 +409,13 @@ end # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it # @param deploy_id [String]: The deploy for which we're writing the secret # @param value [String]: The contents of the secret - def self.writeDeploySecret(deploy_id, value, name = nil, credentials: nil) - name ||= deploy_id+"-secret" + def self.writeDeploySecret(deploy, value, name = nil, credentials: nil) + require "aws-sdk-s3" + name ||= deploy.deploy_id+"-secret" begin MU.log "Writing #{name} to S3 bucket #{adminBucketName(credentials)}" MU::Cloud::AWS.s3(region: myRegion, credentials: credentials).put_object( acl: "private", bucket: adminBucketName(credentials), @@ -399,49 +429,96 @@ # Log bucket policy for enabling CloudTrail logging to our log bucket in S3. def self.cloudtrailBucketPolicy(credentials = nil) cfg = credConfig(credentials) policy_json = '{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AWSCloudTrailAclCheck20131101", - "Effect": "Allow", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AWSCloudTrailAclCheck20131101", + "Effect": "Allow", "Principal": { "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root", "Service": "cloudtrail.amazonaws.com" }, - "Action": "s3:GetBucketAcl", - "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'" - }, - { - "Sid": "AWSCloudTrailWrite20131101", - "Effect": "Allow", + "Action": "s3:GetBucketAcl", + "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'" + }, + { + "Sid": "AWSCloudTrailWrite20131101", + "Effect": "Allow", "Principal": { "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::'+credToAcct(credentials)+':root", "Service": "cloudtrail.amazonaws.com" }, - "Action": "s3:PutObject", - "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*", - "Condition": { - "StringEquals": { - "s3:x-amz-acl": "bucket-owner-full-control" - } - } - } - ] - }' + "Action": "s3:PutObject", + "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*", + "Condition": { + "StringEquals": { + "s3:x-amz-acl": "bucket-owner-full-control" + } + } + } + ] + }' ERB.new(policy_json).result end @@is_in_aws = nil # Alias for #{MU::Cloud::AWS.hosted?} def self.hosted MU::Cloud::AWS.hosted? end + # If we're in AWS and NVME-aware, return a mapping of AWS-side device + # names to actual NVME devices. + # @return [Hash] + def self.attachedNVMeDisks + if !hosted? or !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme") + return {} + end + map = {} + devices = MU::Master.listBlockDevices + return {} if !devices + devices.each { |d| + if d =~ /^\/dev\/nvme/ + %x{/sbin/nvme id-ctrl -v #{d}}.each_line { |desc| + if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./) + virt_dev = Regexp.last_match[1] + map[virt_dev] = d + break + end + } + end + } + map + end + + # Map our own idea of what a block device is called back to whatever AWS + # and the operating system decided on amongst themselves. This currently + # exists to map generic "xvd[a-z]" style names back to real NVMe devices. + # @param dev [String] + def self.realDevicePath(dev) + return dev if !hosted? + value = nil + should_retry = Proc.new { + !value and MU::Master.nvme? + } + MU.retrier(loop_if: should_retry, wait: 5, max: 6) { + map = attachedNVMeDisks + value = if map[dev] + map[dev] + elsif map[dev.gsub(/.*?\//, '')] + map[dev.gsub(/.*?\//, '')] + else + dev # be nice to actually handle this too + end + } + value + end + # Determine whether we (the Mu master, presumably) are hosted in this # cloud. # @return [Boolean] def self.hosted? if $MU_CFG.has_key?("aws_is_hosted") @@ -455,11 +532,11 @@ return @@is_in_aws end begin Timeout.timeout(4) do - instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read + instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read if !instance_id.nil? and instance_id.size > 0 @@is_in_aws = true region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "") begin validate_region(region) @@ -564,21 +641,22 @@ if !$MU_CFG['aws'] return hosted? ? ["#default"] : nil end $MU_CFG['aws'].keys - end + end # Resolve the administrative S3 bucket for a given credential set, or # return a default. # @param credentials [String] # @return [String] def self.adminBucketName(credentials = nil) + require "aws-sdk-s3" cfg = credConfig(credentials) return nil if !cfg if !cfg['log_bucket_name'] - cfg['log_bucket_name'] = $MU_CFG['hostname'] + cfg['log_bucket_name'] = $MU_CFG['hostname'] MU.log "No AWS log bucket defined for credentials #{credentials}, attempting to use default of #{cfg['log_bucket_name']}", MU::WARN end resp = MU::Cloud::AWS.s3(credentials: credentials).list_buckets found = false resp.buckets.each { |b| @@ -608,11 +686,11 @@ "s3://"+adminBucketName(credentials)+"/" end # Return the $MU_CFG data associated with a particular profile/name/set of # credentials. If no account name is specified, will return one flagged as - # default. Returns nil if AWS is not configured. Throws an exception if + # default. Returns nil if AWS is not configured. Throws an exception if # an account name is specified which does not exist. # @param name [String]: The name of the key under 'aws' in mu.yaml to return # @return [Hash,nil] def self.credConfig(name = nil, name_only: false) # If there's nothing in mu.yaml (which is wrong), but we're running @@ -623,14 +701,17 @@ return name_only ? "#default" : @@my_hosted_cfg end if hosted? begin - iam_data = JSON.parse(getAWSMetaData("iam/info")) - if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty? - @@my_hosted_cfg = hosted_config - return name_only ? "#default" : @@my_hosted_cfg + iam_blob = getAWSMetaData("iam/info") + if iam_blob + iam_data = JSON.parse(iam_blob) + if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty? + @@my_hosted_cfg = hosted_config + return name_only ? "#default" : @@my_hosted_cfg + end end rescue JSON::ParserError => e end elsif ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY'] env_config = { @@ -690,10 +771,11 @@ # in AWS at all, or otherwise cannot be determined, return nil. here. # XXX account for Google and non-cloud situations # XXX this needs to be "myAccountNumber" or somesuch # XXX and maybe do the IAM thing for arbitrary, non-resident accounts def self.account_number + require "aws-sdk-ec2" return nil if credConfig.nil? return @@my_acct_num if @@my_acct_num loadCredentials # XXX take optional credential set argument @@ -747,11 +829,11 @@ @@regions.keys.delete_if { |r| !r.match(/^us\-/) }.uniq else @@regions.keys.uniq end -# XXX GovCloud doesn't show up if you query a commercial endpoint... that's +# XXX GovCloud doesn't show up if you query a commercial endpoint... that's # *probably* ok for most purposes? We can't call listAZs on it from out here # apparently, so getting around it is nontrivial # if !@@regions.has_key?("us-gov-west-1") # @@regions["us-gov-west-1"] = Proc.new { listAZs("us-gov-west-1") } # end @@ -772,10 +854,11 @@ # OpenSSH-style public key and a name. # @param keyname [String]: The name of the key to create. # @param public_key [String]: The public key # @return [Array<String>]: keypairname, ssh_private_key, ssh_public_key def self.createEc2SSHKey(keyname, public_key, credentials: nil) + require "aws-sdk-ec2" # We replicate this key in all regions if !MU::Cloud::CloudFormation.emitCloudFormation MU::Cloud::AWS.listRegions.each { |region| MU.log "Replicating #{keyname} to EC2 in #{region}", MU::DEBUG, details: @ssh_public_key begin @@ -800,12 +883,20 @@ # @param region [String]: Supported machine types can vary from region to region, so we look for the set we're interested in specifically # @return [Hash] def self.listInstanceTypes(region = myRegion) return @@instance_types if @@instance_types and @@instance_types[region] return {} if credConfig.nil? + if region.nil? + region = myRegion(debug: true) + end + return {} if region.nil? human_region = @@regionLookup[region] + if human_region.nil? + MU.log "Failed to map a Pricing API region name from #{region}", MU::ERR + return {} + end @@instance_types ||= {} @@instance_types[region] ||= {} # Pricing API isn't widely available, so ask a region we know supports @@ -855,10 +946,11 @@ # disambiguate. # @param name [String]: The name of the cert. For IAM certs this can be any IAM name; for ACM, it's usually the domain name. If multiple matches are found, or no matches, an exception is raised. # @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied. # @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired. def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true) + require "aws-sdk-iam" if (name.nil? or name.empty?) and (id.nil? or id.empty?) raise MuError, "Can't call findSSLCertificate without specifying either a name or an id" end if id and @@certificates[id] return [id, @@certificates[id]] @@ -889,11 +981,11 @@ raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}" else return nil end elsif matches.size > 1 - raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use." + raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use." end end domains = [] @@ -951,11 +1043,11 @@ end # Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate, # look up and validate the specified certificate. This is intended to be # invoked from resource implementations' +validateConfig+ methods. - # @param certblock [Hash,MU::Config::Ref]: + # @param certblock [Hash,MU::Config::Ref]: # @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any # @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any # @return [Boolean] def self.resolveSSLCertificate(certblock, region: nil, credentials: nil) return false if !certblock @@ -1119,19 +1211,19 @@ region ||= myRegion @@elasticache_api[credentials] ||= {} @@elasticache_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElastiCache", region: region, credentials: credentials) @@elasticache_api[credentials][region] end - + # Amazon's SNS API def self.sns(region: MU.curRegion, credentials: nil) region ||= myRegion @@sns_api[credentials] ||= {} @@sns_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SNS", region: region, credentials: credentials) @@sns_api[credentials][region] end - + # Amazon's SQS API def self.sqs(region: MU.curRegion, credentials: nil) region ||= myRegion @@sqs_api[credentials] ||= {} @@sqs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SQS", region: region, credentials: credentials) @@ -1159,11 +1251,11 @@ region ||= myRegion @@apig_api[credentials] ||= {} @@apig_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "APIGateway", region: region, credentials: credentials) @@apig_api[credentials][region] end - + # Amazon's Cloudwatch Events API def self.cloudwatch_events(region = MU.cureRegion) region ||= myRegion @@cloudwatch_events_api[credentials] ||= {} @@cloudwatch_events_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials) @@ -1272,11 +1364,11 @@ def self.getAWSMetaData(param) base_url = "http://169.254.169.254/latest/meta-data/" begin response = nil Timeout.timeout(1) do - response = open("#{base_url}/#{param}").read + response = URI.open("#{base_url}/#{param}").read end response rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH, Net::HTTPServerException, Errno::EHOSTUNREACH => e # This is normal on machines checking to see if they're AWS-hosted @@ -1297,10 +1389,11 @@ def self.createTag(resource = nil, tag_name="MU-ID", tag_value=MU.deploy_id, region: MU.curRegion, credentials: nil) + require "aws-sdk-ec2" attempts = 0 return nil if resource.nil? resource = [resource] if resource.is_a?(String) @@ -1337,9 +1430,10 @@ @syslog_port_semaphore = Mutex.new # Punch AWS security group holes for client nodes to talk back to us, the # Mu Master, if we're in AWS. # @return [void] def self.openFirewallForClients + require "aws-sdk-ec2" MU::Cloud.resourceClass("AWS", :FirewallRule) begin if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb") ::Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb") end