modules/mu/clouds/google.rb in cloud-mu-3.1.3 vs modules/mu/clouds/google.rb in cloud-mu-3.1.4
- old
+ new
@@ -50,10 +50,21 @@
# @return [Array<Symbol>]
def self.required_instance_methods
[:url]
end
+ # Most of our resource implementation +find+ methods have to mangle their
+ # args to make sure they've extracted a project or location argument from
+ # other available information. This does it for them.
+ # @return [Hash]
+ def self.findLocationArgs(**args)
+ args[:project] ||= args[:habitat]
+ args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+ args[:location] ||= args[:region] || args[:availability_zone] || "-"
+ args
+ end
+
# A hook that is always called just before any of the instance method of
# our resource implementations gets invoked, so that we can ensure that
# repetitive setup tasks (like resolving +:resource_group+ for Azure
# resources) have always been done.
# @param cloudobj [MU::Cloud]
@@ -139,24 +150,22 @@
# @param cloudobj [MU::Cloud::Google]: The resource from which to extract the habitat id
# @return [String,nil]
def self.habitat(cloudobj, nolookup: false, deploy: nil)
@@habmap ||= {}
# XXX whaddabout config['habitat'] HNNNGH
-
return nil if !cloudobj.cloudclass.canLiveIn.include?(:Habitat)
-# XXX users are assholes because they're valid two different ways ugh ugh
+# XXX these are assholes because they're valid two different ways ugh ugh
return nil if [MU::Cloud::Google::Group, MU::Cloud::Google::Folder].include?(cloudobj.cloudclass)
if cloudobj.config and cloudobj.config['project']
if nolookup
return cloudobj.config['project']
end
if @@habmap[cloudobj.config['project']]
return @@habmap[cloudobj.config['project']]
end
deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy)
-
projectobj = projectLookup(cloudobj.config['project'], deploy, raise_on_fail: false)
if projectobj
@@habmap[cloudobj.config['project']] = projectobj.cloud_id
return projectobj.cloud_id
@@ -361,11 +370,11 @@
# @param noop [Boolean]: If true, will only print what would be done
def self.removeDeploySecretsAndRoles(deploy_id = MU.deploy_id, flags: {}, noop: false, credentials: nil)
cfg = credConfig(credentials)
return if !cfg or !cfg['project']
flags["project"] ||= cfg['project']
- name = deploy_id+"-secret"
+
resp = MU::Cloud::Google.storage(credentials: credentials).list_objects(
adminBucketName(credentials),
prefix: deploy_id
)
if resp and resp.items
@@ -418,20 +427,20 @@
}
rescue ::Google::Apis::ClientError => e
MU.log e.message, MU::WARN, details: e.inspect
if e.inspect.match(/body: "Not Found"/)
raise MuError, "Google admin bucket #{adminBucketName(credentials)} or key #{name} does not appear to exist or is not visible with #{credentials ? credentials : "default"} credentials"
- elsif e.message.match(/notFound: |Unknown user:/)
+ elsif e.message.match(/notFound: |Unknown user:|conflict: /)
if retries < 5
sleep 5
retries += 1
retry
else
raise e
end
elsif e.inspect.match(/The metadata for object "null" was edited during the operation/)
- MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::WARN, details: aclobj
+ MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::DEBUG, details: aclobj
sleep 10
retry
else
raise MuError, "Got #{e.message} trying to set ACLs for #{deploy_id} in #{adminBucketName(credentials)}"
end
@@ -538,11 +547,11 @@
@@default_project ||= MU::Cloud::Google.getGoogleMetaData("project/project-id")
begin
listRegions(credentials: credentials)
listInstanceTypes(credentials: credentials)
listProjects(credentials)
- rescue ::Google::Apis::ClientError => e
+ rescue ::Google::Apis::ClientError
MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes
@@authorizers.delete(credentials)
return nil
end
@@authorizers[credentials][scopes.to_s]
@@ -715,11 +724,10 @@
raise MuError, "Insufficient permissions to list Google Cloud region. The service account #{myServiceAccount} should probably have the project owner role."
end
raise e
end
- regions = []
result.items.each { |region|
@@regions[region.name] = []
region.zones.each { |az|
@@regions[region.name] << az.sub(/^.*?\/([^\/]+)$/, '\1')
}
@@ -844,11 +852,11 @@
end
if subclass.nil?
begin
@@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
- rescue Signet::AuthorizationError => e
+ rescue Signet::AuthorizationError
MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
@@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
@@readonly[credentials] ||= {}
@@readonly[credentials]["AdminDirectoryV1"] = true
end
@@ -1050,12 +1058,10 @@
end
@@customer_ids_cache[credentials]
end
- private
-
# Wrapper class for Google APIs, so that we can catch some common
# transient endpoint errors without having to spray rescues all over the
# codebase.
class GoogleEndpoint
@api = nil
@@ -1107,16 +1113,17 @@
# @param region [String]: The region in which to loop for the resources
# @param noop [Boolean]: If true, will only log messages about resources to be deleted, without actually deleting them
# @param filter [String]: The Compute API filter string to use to isolate appropriate resources
def delete(type, project, region = nil, noop = false, filter = "description eq #{MU.deploy_id}", credentials: nil)
list_sym = "list_#{type.sub(/y$/, "ie")}s".to_sym
+ credentials ||= @credentials
resp = nil
begin
if region
- resp = MU::Cloud::Google.compute(credentials: @credentials).send(list_sym, project, region, filter: filter, mu_gcp_enable_apis: false)
+ resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, region, filter: filter, mu_gcp_enable_apis: false)
else
- resp = MU::Cloud::Google.compute(credentials: @credentials).send(list_sym, project, filter: filter, mu_gcp_enable_apis: false)
+ resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, filter: filter, mu_gcp_enable_apis: false)
end
rescue ::Google::Apis::ClientError => e
return if e.message.match(/^notFound: /)
end
@@ -1135,13 +1142,13 @@
failed = false
begin
resp = nil
failed = false
if region
- resp = MU::Cloud::Google.compute(credentials: @credentials).send(delete_sym, project, region, obj.name)
+ resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, region, obj.name)
else
- resp = MU::Cloud::Google.compute(credentials: @credentials).send(delete_sym, project, obj.name)
+ resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, obj.name)
end
if resp.error and resp.error.errors and resp.error.errors.size > 0
failed = true
retries += 1
@@ -1193,15 +1200,23 @@
begin
MU.log "Calling #{method_sym}", MU::DEBUG, details: arguments
retval = nil
retries = 0
wait_backoff = 5
- if next_page_token
- if arguments.size == 1 and arguments.first.is_a?(Hash)
- arguments[0][:page_token] = next_page_token
- else
- arguments << { :page_token => next_page_token }
+ if next_page_token
+ if method_sym != :list_entry_log_entries
+ if arguments.size == 1 and arguments.first.is_a?(Hash)
+ arguments[0][:page_token] = next_page_token
+ else
+ arguments << { :page_token => next_page_token }
+ end
+ elsif arguments.first.class == ::Google::Apis::LoggingV2::ListLogEntriesRequest
+ arguments[0] = ::Google::Apis::LoggingV2::ListLogEntriesRequest.new(
+ resource_names: arguments.first.resource_names,
+ filter: arguments.first.filter,
+ page_token: next_page_token
+ )
end
end
begin
if !arguments.nil? and arguments.size == 1
retval = @api.method(method_sym).call(arguments[0])
@@ -1316,11 +1331,10 @@
end
if retval.class.name.match(/.*?::Operation$/)
retries = 0
- orig_target = retval.name
# Check whether the various types of +Operation+ responses say
# they're done, without knowing which specific API they're from
def is_done?(retval)
(retval.respond_to?(:status) and retval.status == "DONE") or (retval.respond_to?(:done) and retval.done)
@@ -1388,11 +1402,11 @@
# Most insert methods have a predictable get_* counterpart. Let's
# take advantage.
# XXX might want to do something similar for delete ops? just the
# but where we wait for the operation to definitely be done
- had_been_found = false
+# had_been_found = false
if method_sym.to_s.match(/^(insert|create|patch)_/)
get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym
cloud_id = if retval.respond_to?(:target_link)
retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1')
elsif retval.respond_to?(:metadata) and retval.metadata["target"]
@@ -1415,11 +1429,11 @@
end
actual_resource = @api.method(get_method).call(*faked_args)
#if method_sym == :insert_instance
#MU.log "actual_resource", MU::WARN, details: actual_resource
#end
- had_been_found = true
+# had_been_found = true
if actual_resource.respond_to?(:status) and
["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status)
retries = 0
begin
if retries > 0 and retries % 3 == 0
@@ -1438,10 +1452,11 @@
# This atrocity appends the pages of list_* results
if overall_retval
if method_sym.to_s.match(/^list_(.*)/)
require 'google/apis/iam_v1'
+ require 'google/apis/logging_v2'
what = Regexp.last_match[1].to_sym
whatassign = (Regexp.last_match[1]+"=").to_sym
if overall_retval.class == ::Google::Apis::IamV1::ListServiceAccountsResponse
what = :accounts
whatassign = :accounts=
@@ -1449,10 +1464,10 @@
if retval.respond_to?(what) and retval.respond_to?(whatassign)
if !retval.public_send(what).nil?
newarray = retval.public_send(what) + overall_retval.public_send(what)
overall_retval.public_send(whatassign, newarray)
end
- else
+ elsif !retval.respond_to?(:next_page_token) or retval.next_page_token.nil? or retval.next_page_token.empty?
MU.log "Not sure how to append #{method_sym.to_s} results to #{overall_retval.class.name} (apparently #{what.to_s} and #{whatassign.to_s} aren't it), returning first page only", MU::WARN, details: retval
return retval
end
else
MU.log "Not sure how to append #{method_sym.to_s} results, returning first page only", MU::WARN, details: retval