modules/mu/clouds/google.rb in cloud-mu-2.0.0.pre.beta2 vs modules/mu/clouds/google.rb in cloud-mu-2.0.0.pre.beta3
- old
+ new
@@ -26,10 +26,11 @@
@@default_project = nil
@@myRegion_var = nil
@@my_hosted_cfg = nil
@@authorizers = {}
@@acct_to_profile_map = {}
+ @@enable_semaphores = {}
# Any cloud-specific instance methods we require our resource
# implementations to have, above and beyond the ones specified by
# {MU::Cloud}
# @return [Array<Symbol>]
@@ -69,10 +70,40 @@
end
$MU_CFG['google'].keys
end
+ # A shortcut for {MU::MommaCat.findStray} to resolve a shorthand project
+ # name into a cloud object, whether it refers to a sibling by internal
+ # name or by cloud identifier.
+ # @param name [String]
+ # @param deploy [String]
+ # @param raise_on_fail [Boolean]
+ # @param sibling_only [Boolean]
+ # @return [MU::Cloud::Habitat,nil]
+ def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false)
+ project_obj = deploy.findLitterMate(type: "habitats", name: name)
+
+ if !project_obj and !sibling_only
+ resp = MU::MommaCat.findStray(
+ "Google",
+ "habitats",
+ deploy_id: deploy.deploy_id,
+ cloud_id: name,
+ name: name,
+ dummy_ok: true
+ )
+ project_obj = resp.first if resp
+ end
+
+ if (!project_obj or !project_obj.cloud_id) and raise_on_fail
+ raise MuError, "Failed to find project '#{name}' in deploy #{deploy.deploy_id}"
+ end
+
+ project_obj
+ end
+
# Resolve the administrative Cloud Storage bucket for a given credential
# set, or return a default.
# @param credentials [String]
# @return [String]
def self.adminBucketName(credentials = nil)
@@ -209,11 +240,11 @@
entity: "user-"+acct
)
[name, "log_vol_ebs_key"].each { |obj|
MU.log "Granting #{acct} access to #{obj} in Cloud Storage bucket #{adminBucketName(credentials)}"
- pp aclobj
+
MU::Cloud::Google.storage(credentials: credentials).insert_object_access_control(
adminBucketName(credentials),
obj,
aclobj
)
@@ -321,10 +352,14 @@
return @@authorizers[credentials][scopes.to_s]
end
cfg = credConfig(credentials)
+ if cfg['project']
+ @@enable_semaphores[cfg['project']] ||= Mutex.new
+ end
+
if cfg
data = nil
@@authorizers[credentials] ||= {}
def self.get_machine_credentials(scopes)
@@ -595,27 +630,28 @@
# @param subclass [<Google::Apis::CloudresourcemanagerV1>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV1::subclass instead of an API client instance
def self.resource_manager(subclass = nil, credentials: nil)
require 'google/apis/cloudresourcemanager_v1'
if subclass.nil?
- @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
+# @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/cloudplatformprojects'], masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
+ @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/cloudplatformprojects'], credentials: credentials)
return @@resource_api[credentials]
elsif subclass.is_a?(Symbol)
return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass)
end
end
# Google's Cloud Resource Manager API V2, which apparently has all the folder bits
- # @param subclass [<Google::Apis::CloudresourcemanagerV2beta1>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV2beta1::subclass instead of an API client instance
+ # @param subclass [<Google::Apis::CloudresourcemanagerV2>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV2::subclass instead of an API client instance
def self.folder(subclass = nil, credentials: nil)
- require 'google/apis/cloudresourcemanager_v2beta1'
+ require 'google/apis/cloudresourcemanager_v2'
if subclass.nil?
- @@resource2_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV2beta1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
+ @@resource2_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV2::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/cloudplatformfolders'], credentials: credentials)
return @@resource2_api[credentials]
elsif subclass.is_a?(Symbol)
- return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV2beta1").const_get(subclass)
+ return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV2").const_get(subclass)
end
end
# Google's Container API
# @param subclass [<Google::Apis::ContainerV1>]: If specified, will return the class ::Google::Apis::ContainerV1::subclass instead of an API client instance
@@ -680,11 +716,36 @@
elsif subclass.is_a?(Symbol)
return Object.const_get("::Google").const_get("Apis").const_get("LoggingV2").const_get(subclass)
end
end
+ # Google's Cloud Billing Service API
+ # @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
+ def self.billing(subclass = nil, credentials: nil)
+ require 'google/apis/cloudbilling_v1'
+ if subclass.nil?
+ @@billing_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudbillingV1::CloudbillingService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
+ return @@billing_api[credentials]
+ elsif subclass.is_a?(Symbol)
+ return Object.const_get("::Google").const_get("Apis").const_get("CloudbillingV1").const_get(subclass)
+ end
+ end
+
+
+ # Retrieve the organization, if any, to which these credentials belong.
+ # @param credentials [String]
+ # @return [Array<OpenStruct>],nil]
+ def self.getOrg(credentials = nil)
+ resp = MU::Cloud::Google.resource_manager(credentials: credentials).search_organizations
+ if resp and resp.organizations
+ # XXX no idea if it's possible to be a member of multiple orgs
+ return resp.organizations.first
+ end
+ nil
+ end
+
private
# Wrapper class for Google APIs, so that we can catch some common
# transient endpoint errors without having to spray rescues all over the
# codebase.
@@ -760,11 +821,11 @@
else
failed = false
end
# TODO validate that the resource actually went away, because it seems not to do so very reliably
rescue ::Google::Apis::ClientError => e
- raise e if !e.message.match(/^notFound: /)
+ raise e if !e.message.match(/(^notFound: |operation in progress)/)
end while failed and retries < 6
end
}
}
threads.each do |t|
@@ -796,44 +857,52 @@
if arguments.size > 0
raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym} in project #{arguments.first}"
else
raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym}"
end
- rescue ::Google::Apis::ClientError => e
+ rescue ::Google::Apis::ClientError, OpenSSL::SSL::SSLError => e
if e.message.match(/^invalidParameter:/)
MU.log "#{method_sym.to_s}: "+e.message, MU::ERR, details: arguments
# uncomment for debugging stuff; this can occur in benign situations so we don't normally want it logging
elsif e.message.match(/^forbidden:/)
MU.log "Using credentials #{@credentials}: #{method_sym.to_s}: "+e.message, MU::ERR, details: caller
end
- if retries <= 1 and e.message.match(/^accessNotConfigured/)
+ @@enable_semaphores ||= {}
+ max_retries = 3
+ wait_time = 90
+ if retries <= max_retries and e.message.match(/^accessNotConfigured/)
enable_obj = nil
project = arguments.size > 0 ? arguments.first.to_s : MU::Cloud::Google.defaultProject(@credentials)
+ @@enable_semaphores[project] ||= Mutex.new
enable_obj = MU::Cloud::Google.service_manager(:EnableServiceRequest).new(
consumer_id: "project:"+project
)
# XXX dumbass way to get this string
- e.message.match(/Enable it by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)
+ e.message.match(/by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)
+
svc_name = Regexp.last_match[1]
save_verbosity = MU.verbosity
if svc_name != "servicemanagement.googleapis.com"
- MU.setLogging(MU::Logger::NORMAL)
- MU.log "Attempting to enable #{svc_name} in project #{project}, then waiting for 30s", MU::WARN
- MU.setLogging(save_verbosity)
- MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
- sleep 30
retries += 1
+ @@enable_semaphores[project].synchronize {
+ MU.setLogging(MU::Logger::NORMAL)
+ MU.log "Attempting to enable #{svc_name} in project #{project}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
+ MU.setLogging(save_verbosity)
+ MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
+ }
+ sleep wait_time/retries
retry
else
MU.setLogging(MU::Logger::NORMAL)
MU.log "Google Cloud's Service Management API must be enabled manually by visiting #{e.message.gsub(/.*?(https?:\/\/[^\s]+)(?:$|\s).*/, '\1')}", MU::ERR
MU.setLogging(save_verbosity)
raise MU::MuError, "Service Management API not yet enabled for this account/project"
end
elsif retries <= 10 and
e.message.match(/^resourceNotReady:/) or
- (e.message.match(/^resourceInUseByAnotherResource:/) and method_sym.to_s.match(/^delete_/))
+ (e.message.match(/^resourceInUseByAnotherResource:/) and method_sym.to_s.match(/^delete_/)) or
+ e.message.match(/SSL_connect/)
if retries > 0 and retries % 3 == 0
MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::NOTICE, details: arguments
else
MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::DEBUG, details: arguments
end
@@ -966,8 +1035,9 @@
@@resource_api = {}
@@resource2_api = {}
@@service_api = {}
@@firestore_api = {}
@@admin_directory_api = {}
+ @@billing_api = {}
end
end
end