# Copyright:: Copyright (c) 2017 eGlobalTech, Inc., all rights reserved
#
# Licensed under the BSD-3 license (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the root of the project or at
#
#     http://egt-labs.com/mu/LICENSE.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

require 'googleauth'
require "net/http"
require 'net/https'
require 'multi_json'
require 'stringio'

module MU
  class Cloud
    # Support for Google Cloud Platform as a provisioning layer.
    class Google
      @@authtoken = nil
      @@default_project = nil
      @@myRegion_var = nil
      @@my_hosted_cfg = nil
      @@authorizers = {}
      @@acct_to_profile_map = {}
      @@enable_semaphores = {}
      @@readonly_semaphore = Mutex.new
      @@readonly = {}

      # Module used by {MU::Cloud} to insert additional instance methods into
      # instantiated resources in this cloud layer.
      module AdditionalResourceMethods
        # Google Cloud url attribute, found in some form on most GCP cloud
        # resources.
        # @return [String]
        def url
          desc = cloud_desc
          (desc and desc.self_link) ? desc.self_link : nil
        end
      end

      # Any cloud-specific instance methods we require our resource
      # implementations to have, above and beyond the ones specified by
      # {MU::Cloud}
      # @return [Array<Symbol>]
      def self.required_instance_methods
        [:url]
      end

      # A hook that is always called just before any of the instance method of
      # our resource implementations gets invoked, so that we can ensure that
      # repetitive setup tasks (like resolving +:resource_group+ for Azure
      # resources) have always been done.
      # @param cloudobj [MU::Cloud]
      # @param deploy [MU::MommaCat]
      def self.resourceInitHook(cloudobj, deploy)
        class << self
          attr_reader :project_id
          attr_reader :customer
          # url is too complex for an attribute (we get it from the cloud API),
          # so it's up in AdditionalResourceMethods instead
        end
        return if !cloudobj

        cloudobj.instance_variable_set(:@customer, MU::Cloud::Google.customerID(cloudobj.config['credentials']))

# XXX ensure @cloud_id and @project_id if this is a habitat
# XXX skip project_id if this is a folder or group
        if deploy
# XXX this may be wrong for new deploys (but def right for regrooms)
          project = MU::Cloud::Google.projectLookup(cloudobj.config['project'], deploy, sibling_only: true, raise_on_fail: false)
          project_id = project.nil? ? cloudobj.config['project'] : project.cloudobj.cloud_id
          cloudobj.instance_variable_set(:@project_id, project_id)
        else
          cloudobj.instance_variable_set(:@project_id, cloudobj.config['project'])
        end

# XXX @url? Well we're not likely to have @cloud_desc at this point, so maybe
# that needs to be a generic-to-google wrapper like def url; cloud_desc.self_link;end

# XXX something like: vpc["habitat"] = MU::Cloud::Google.projectToRef(vpc["project"], config: configurator, credentials: vpc["credentials"])
      end

      # If we're running this cloud, return the $MU_CFG blob we'd use to
      # describe this environment as our target one.
      def self.hosted_config
        return nil if !hosted?
        getGoogleMetaData("instance/zone").match(/^projects\/[^\/]+\/zones\/([^\/]+)$/)
        zone = Regexp.last_match[1]
        {
          "project" => MU::Cloud::Google.getGoogleMetaData("project/project-id"),
          "region" => zone.sub(/-[a-z]$/, "")
        }
      end

      # A non-working example configuration
      def self.config_example
        sample = hosted_config
        sample ||= {
          "project" => "my-project",
          "region" => "us-east4"
        }
        sample["credentials_file"] = "#{Etc.getpwuid(Process.uid).dir}/gcp_serviceacct.json"
        sample["log_bucket_name"] = "my-mu-cloud-storage-bucket"
        sample
      end

      # If we reside in this cloud, return the VPC in which we, the Mu Master, reside.
      # @return [MU::Cloud::VPC]
      def self.myVPCObj
        return nil if !hosted?
        instance = MU.myCloudDescriptor
        return nil if !instance or !instance.network_interfaces or instance.network_interfaces.size == 0
        vpc = MU::MommaCat.findStray("Google", "vpc", cloud_id: instance.network_interfaces.first.network.gsub(/.*?\/([^\/]+)$/, '\1'), dummy_ok: true, habitats: [myProject])
        return nil if vpc.nil? or vpc.size == 0
        vpc.first
      end

      # Return the name strings of all known sets of credentials for this cloud
      # @return [Array<String>]
      def self.listCredentials
        if !$MU_CFG['google']
          return hosted? ? ["#default"] : nil
        end

        $MU_CFG['google'].keys
      end

      @@habmap = {}

      # Return what we think of as a cloud object's habitat. In GCP, this means
      # the +project_id+ in which is resident. If this is not applicable, such
      # as for a {Habitat} or {Folder}, returns nil.
      # @param cloudobj [MU::Cloud::Google]: The resource from which to extract the habitat id
      # @return [String,nil]
      def self.habitat(cloudobj, nolookup: false, deploy: nil)
        @@habmap ||= {}
# XXX whaddabout config['habitat'] HNNNGH

        return nil if !cloudobj.cloudclass.canLiveIn.include?(:Habitat)

# XXX users are assholes because they're valid two different ways ugh ugh
        return nil if [MU::Cloud::Google::Group, MU::Cloud::Google::Folder].include?(cloudobj.cloudclass)
        if cloudobj.config and cloudobj.config['project']
          if nolookup
            return cloudobj.config['project']
          end
          if @@habmap[cloudobj.config['project']]
            return @@habmap[cloudobj.config['project']]
          end
          deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy)

          projectobj = projectLookup(cloudobj.config['project'], deploy, raise_on_fail: false)

          if projectobj
            @@habmap[cloudobj.config['project']] = projectobj.cloud_id
            return projectobj.cloud_id
          end
        end

        # blow up if this resource *has* to live in a project
        if cloudobj.cloudclass.canLiveIn == [:Habitat]
          MU.log "Failed to find project for cloudobj #{cloudobj.to_s}", MU::ERR, details: cloudobj
          raise MuError, "Failed to find project for cloudobj #{cloudobj.to_s}"
        end

        nil
      end

      # Take a plain string that might be a reference to sibling project
      # declared elsewhere in the active stack, or the project id of a live
      # cloud resource, and return a {MU::Config::Ref} object
      # @param project [String]: The name of a sibling project, or project id of an active project in GCP
      # @param config [MU::Config]: A {MU::Config} object containing sibling resources, typically what we'd pass if we're calling during configuration parsing
      # @param credentials [String]: 
      # @return [MU::Config::Ref]
      def self.projectToRef(project, config: nil, credentials: nil)
        return nil if !project

        if config and config.haveLitterMate?(project, "habitat")
          ref = MU::Config::Ref.new(
            name: project,
            cloud: "Google",
            credentials: credentials,
            type: "habitats"
          )
        end
        
        if !ref
          resp = MU::MommaCat.findStray(
            "Google",
            "habitats",
            cloud_id: project,
            credentials: credentials,
            dummy_ok: true
          )
          if resp and resp.size > 0
            project_obj = resp.first
            ref = MU::Config::Ref.new(
              id: project_obj.cloud_id,
              cloud: "Google",
              credentials: credentials,
              type: "habitats"
            )
          end
        end

        ref
      end

      # A shortcut for {MU::MommaCat.findStray} to resolve a shorthand project
      # name into a cloud object, whether it refers to a sibling by internal
      # name or by cloud identifier.
      # @param name [String]
      # @param deploy [String]
      # @param raise_on_fail [Boolean]
      # @param sibling_only [Boolean]
      # @return [MU::Config::Habitat,nil]
      def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false)
        project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy

        if !project_obj and !sibling_only
          resp = MU::MommaCat.findStray(
            "Google",
            "habitats",
            deploy_id: deploy ? deploy.deploy_id : nil,
            cloud_id: name,
            name: name,
            dummy_ok: true
          )

          project_obj = resp.first if resp and resp.size > 0
        end

        if (!project_obj or !project_obj.cloud_id) and raise_on_fail
          raise MuError, "Failed to find project '#{name}' in deploy #{deploy.deploy_id}"
        end

        project_obj
      end

      # Resolve the administrative Cloud Storage bucket for a given credential
      # set, or return a default.
      # @param credentials [String]
      # @return [String]
      def self.adminBucketName(credentials = nil)
         #XXX find a default if this particular account doesn't have a log_bucket_name configured
        cfg = credConfig(credentials)
        if cfg.nil?
          raise MuError, "Failed to load Google credential set #{credentials}"
        end
        cfg['log_bucket_name']
      end

      # Resolve the administrative Cloud Storage bucket for a given credential
      # set, or return a default.
      # @param credentials [String]
      # @return [String]
      def self.adminBucketUrl(credentials = nil)
        "gs://"+adminBucketName(credentials)+"/"
      end

      # Return the $MU_CFG data associated with a particular profile/name/set of
      # credentials. If no account name is specified, will return one flagged as
      # default. Returns nil if GCP is not configured. Throws an exception if 
      # an account name is specified which does not exist.
      # @param name [String]: The name of the key under 'google' in mu.yaml to return
      # @return [Hash,nil]
      def self.credConfig(name = nil, name_only: false)
        # If there's nothing in mu.yaml (which is wrong), but we're running
        # on a machine hosted in GCP, fake it with that machine's service
        # account and hope for the best.
        if !$MU_CFG['google'] or !$MU_CFG['google'].is_a?(Hash) or $MU_CFG['google'].size == 0
          return @@my_hosted_cfg if @@my_hosted_cfg

          if hosted?
            @@my_hosted_cfg = hosted_config
            return name_only ? "#default" : @@my_hosted_cfg
          end

          return nil
        end

        if name.nil?
          $MU_CFG['google'].each_pair { |set, cfg|
            if cfg['default']
              return name_only ? set : cfg
            end
          }
        else
          if $MU_CFG['google'][name]
            return name_only ? name : $MU_CFG['google'][name]
          elsif @@acct_to_profile_map[name.to_s]
            return name_only ? name : @@acct_to_profile_map[name.to_s]
          end
# XXX whatever process might lead us to populate @@acct_to_profile_map with some mappings, like projectname -> account profile, goes here
          return nil
        end
      end

      # If we've configured Google as a provider, or are simply hosted in GCP, 
      # decide what our default region is.
      def self.myRegion(credentials = nil)
        cfg = credConfig(credentials)
        if cfg and cfg['region']
          @@myRegion_var = cfg['region']
        elsif MU::Cloud::Google.hosted?
          zone = MU::Cloud::Google.getGoogleMetaData("instance/zone")
          @@myRegion_var = zone.gsub(/^.*?\/|\-\d+$/, "")
        else
          @@myRegion_var = "us-east4"
        end
        @@myRegion_var
      end

      # Do cloud-specific deploy instantiation tasks, such as copying SSH keys
      # around, sticking secrets in buckets, creating resource groups, etc
      # @param deploy [MU::MommaCat]
      def self.initDeploy(deploy)
      end

      # Purge cloud-specific deploy meta-artifacts (SSH keys, resource groups,
      # etc)
      # @param deploy_id [MU::MommaCat]
      def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
      end

      # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
      # @param deploy_id [String]: The deploy for which we're writing the secret
      # @param value [String]: The contents of the secret
      def self.writeDeploySecret(deploy_id, value, name = nil, credentials: nil)
        name ||= deploy_id+"-secret"
        begin
          MU.log "Writing #{name} to Cloud Storage bucket #{adminBucketName(credentials)}"

          f = Tempfile.new(name) # XXX this is insecure and stupid
          f.write value
          f.close
          objectobj = MU::Cloud::Google.storage(:Object).new(
            bucket: adminBucketName(credentials),
            name: name
          )
          MU::Cloud::Google.storage(credentials: credentials).insert_object(
            adminBucketName(credentials),
            objectobj,
            upload_source: f.path
          )
          f.unlink
        rescue ::Google::Apis::ClientError => e
          raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{adminBucketName(credentials)}"
        end
      end

      # Remove the service account and various deploy secrets associated with a deployment. Intended for invocation from MU::Cleanup.
      # @param deploy_id [String]: The deploy for which we're granting the secret
      # @param noop [Boolean]: If true, will only print what would be done
      def self.removeDeploySecretsAndRoles(deploy_id = MU.deploy_id, flags: {}, noop: false, credentials: nil)
        cfg = credConfig(credentials)
        return if !cfg or !cfg['project']
        flags["project"] ||= cfg['project']
        name = deploy_id+"-secret"
        resp = MU::Cloud::Google.storage(credentials: credentials).list_objects(
          adminBucketName(credentials),
          prefix: deploy_id
        )
        if resp and resp.items
          resp.items.each { |obj|
            MU.log "Deleting gs://#{adminBucketName(credentials)}/#{obj.name}"
            if !noop
              MU::Cloud::Google.storage(credentials: credentials).delete_object(
                adminBucketName(credentials),
                obj.name
              )
            end
          }
        end
      end

      # Grant access to appropriate Cloud Storage objects in our log/secret bucket for a deploy member.
      # @param acct [String]: The service account (by email addr) to which we'll grant access
      # @param deploy_id [String]: The deploy for which we're granting the secret
      # XXX add equivalent for AWS and call agnostically
      def self.grantDeploySecretAccess(acct, deploy_id = MU.deploy_id, name = nil, credentials: nil)
        name ||= deploy_id+"-secret"
        aclobj = nil

        retries = 0
        begin
          MU.log "Granting #{acct} access to list Cloud Storage bucket #{adminBucketName(credentials)}"
          MU::Cloud::Google.storage(credentials: credentials).insert_bucket_access_control(
            adminBucketName(credentials),
            MU::Cloud::Google.storage(:BucketAccessControl).new(
              bucket: adminBucketName(credentials),
              role: "READER",
              entity: "user-"+acct
            )
          )

          aclobj = MU::Cloud::Google.storage(:ObjectAccessControl).new(
            bucket: adminBucketName(credentials),
            role: "READER",
            entity: "user-"+acct
          )

          [name].each { |obj|
            MU.log "Granting #{acct} access to #{obj} in Cloud Storage bucket #{adminBucketName(credentials)}"

            MU::Cloud::Google.storage(credentials: credentials).insert_object_access_control(
              adminBucketName(credentials),
              obj,
              aclobj
            )
          }
        rescue ::Google::Apis::ClientError => e
MU.log e.message, MU::WARN, details: e.inspect
          if e.inspect.match(/body: "Not Found"/)
            raise MuError, "Google admin bucket #{adminBucketName(credentials)} or key #{name} does not appear to exist or is not visible with #{credentials ? credentials : "default"} credentials"
          elsif e.message.match(/notFound: |Unknown user:/)
            if retries < 5
              sleep 5
              retries += 1
              retry
            else
              raise e
            end
          elsif e.inspect.match(/The metadata for object "null" was edited during the operation/)
            MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::WARN, details: aclobj
            sleep 10
            retry
          else
            raise MuError, "Got #{e.message} trying to set ACLs for #{deploy_id} in #{adminBucketName(credentials)}"
          end
        end
      end

      @@is_in_gcp = nil

      # Alias for #{MU::Cloud::Google.hosted?}
      def self.hosted
        MU::Cloud::Google.hosted?
      end

      # Determine whether we (the Mu master, presumably) are hosted in this
      # cloud.
      # @return [Boolean]
      def self.hosted?
        if $MU_CFG.has_key?("google_is_hosted")
          @@is_in_aws = $MU_CFG["google_is_hosted"]
          return $MU_CFG["google_is_hosted"]
        end
        if !@@is_in_gcp.nil?
          return @@is_in_gcp
        end

        if getGoogleMetaData("project/project-id")
          @@is_in_gcp = true
          return true
        end
        @@is_in_gcp = false
        false
      end

      # Fetch a Google instance metadata parameter (example: instance/id).
      # @param param [String]: The parameter name to fetch
      # @return [String, nil]
      def self.getGoogleMetaData(param)
        base_url = "http://metadata.google.internal/computeMetadata/v1"
        begin
          Timeout.timeout(2) do
            response = open(
              "#{base_url}/#{param}",
              "Metadata-Flavor" => "Google"
            ).read
            return response
          end
        rescue Net::HTTPServerException, OpenURI::HTTPError, Timeout::Error, SocketError, Errno::EHOSTUNREACH, Errno::ENETUNREACH => e
          # This is fairly normal, just handle it gracefully
          logger = MU::Logger.new
          logger.log "Failed metadata request #{base_url}/#{param}: #{e.inspect}", MU::DEBUG
        end

        nil
      end

      # Create an SSL Certificate resource from some local x509 cert files.
      # @param name [String]: A resource name for the certificate
      # @param cert [String,OpenSSL::X509::Certificate]: An x509 certificate
      # @param key [String,OpenSSL::PKey]: An x509 private key
      # @return [Google::Apis::ComputeV1::SslCertificate]
      def self.createSSLCertificate(name, cert, key, flags = {}, credentials: nil)
        flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
        flags["description"] ||= MU.deploy_id
        certobj = ::Google::Apis::ComputeV1::SslCertificate.new(
          name: name,
          certificate: cert.to_s,
          private_key: key.to_s,
          description: flags["description"]
        )
        MU::Cloud::Google.compute(credentials: credentials).insert_ssl_certificate(flags["project"], certobj)
      end

      @@svc_account_name = nil
      # Fetch the name of the service account we were using last time we loaded
      # GCP credentials.
      # @return [String]
      def self.svc_account_name
        @@svc_account_name
      end
      # Pull our global Google Cloud Platform credentials out of their secure
      # vault, feed them to the googleauth gem, and stash the results on hand
      # for consumption by the various GCP APIs.
      # @param scopes [Array<String>]: One or more scopes for which to authorizer the caller. Will vary depending on the API you're calling.
      def self.loadCredentials(scopes = nil, credentials: nil)
        if @@authorizers[credentials] and @@authorizers[credentials][scopes.to_s]
          return @@authorizers[credentials][scopes.to_s]
        end

        cfg = credConfig(credentials)

        if cfg
          if cfg['project']
            @@enable_semaphores[cfg['project']] ||= Mutex.new
          end
          data = nil
          @@authorizers[credentials] ||= {}
  
          def self.get_machine_credentials(scopes, credentials = nil)
            @@svc_account_name = MU::Cloud::Google.getGoogleMetaData("instance/service-accounts/default/email")
            MU.log "We are hosted in GCP, so I will attempt to use the service account #{@@svc_account_name} to make API requests.", MU::DEBUG

            @@authorizers[credentials][scopes.to_s] = ::Google::Auth.get_application_default(scopes)
            @@authorizers[credentials][scopes.to_s].fetch_access_token!
            @@default_project ||= MU::Cloud::Google.getGoogleMetaData("project/project-id")
            begin
              listRegions(credentials: credentials)
              listInstanceTypes(credentials: credentials)
              listProjects(credentials)
            rescue ::Google::Apis::ClientError => e
              MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes
              @@authorizers.delete(credentials)
              return nil
            end
            @@authorizers[credentials][scopes.to_s]
          end

          if cfg["credentials_file"] or cfg["credentials_encoded"]

            begin
              data = if cfg["credentials_encoded"]
                JSON.parse(Base64.decode64(cfg["credentials_encoded"]))
              else
                JSON.parse(File.read(cfg["credentials_file"]))
              end
              @@default_project ||= data["project_id"]
              creds = {
                :json_key_io => StringIO.new(MultiJson.dump(data)),
                :scope => scopes
              }
              @@svc_account_name = data["client_email"]
              @@authorizers[credentials][scopes.to_s] = ::Google::Auth::ServiceAccountCredentials.make_creds(creds)
              return @@authorizers[credentials][scopes.to_s]
            rescue JSON::ParserError, Errno::ENOENT, Errno::EACCES => e
              if !MU::Cloud::Google.hosted?
                raise MuError, "Google Cloud credentials file #{cfg["credentials_file"]} is missing or invalid (#{e.message})"
              end
              MU.log "Google Cloud credentials file #{cfg["credentials_file"]} is missing or invalid", MU::WARN, details: e.message
              return get_machine_credentials(scopes, credentials)
            end
          elsif cfg["credentials"]
            begin
              vault, item = cfg["credentials"].split(/:/)
              data = MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
            rescue MU::Groomer::MuNoSuchSecret
              if !MU::Cloud::Google.hosted?
                raise MuError, "Google Cloud credentials not found in Vault #{vault}:#{item}"
              end
              MU.log "Google Cloud credentials not found in Vault #{vault}:#{item}", MU::WARN
              found = get_machine_credentials(scopes, credentials)
              raise MuError, "No valid credentials available! Either grant admin privileges to machine service account, or manually add a different one with mu-configure" if found.nil?
              return found
            end

            @@default_project ||= data["project_id"]
            creds = {
              :json_key_io => StringIO.new(MultiJson.dump(data)),
              :scope => scopes
            }
            @@svc_account_name = data["client_email"]
            @@authorizers[credentials][scopes.to_s] = ::Google::Auth::ServiceAccountCredentials.make_creds(creds)
            return @@authorizers[credentials][scopes.to_s]
          elsif MU::Cloud::Google.hosted?
            found = get_machine_credentials(scopes, credentials)
            raise MuError, "No valid credentials available! Either grant admin privileges to machine service account, or manually add a different one with mu-configure" if found.nil?
            return found
          else
            raise MuError, "Google Cloud credentials not configured"
          end

        end
        nil
      end

      # Fetch a URL
      def self.get(url)
        uri = URI url
        resp = nil

        Net::HTTP.start(uri.host, uri.port) do |http|
          resp = http.get(uri)
        end

        unless resp.code == "200"
          puts resp.code, resp.body
          exit
        end
        resp.body
      end

      # If this Mu master resides in the Google Cloud Platform, return the
      # project id in which we reside. Nil if we're not in GCP.
      def self.myProject
        if MU::Cloud::Google.hosted?
          return MU::Cloud::Google.getGoogleMetaData("project/project-id")
        end
        nil
      end

      # If this Mu master resides in the Google Cloud Platform, return the
      # default service account associated with its metadata.
      def self.myServiceAccount
        if MU::Cloud::Google.hosted?
          MU::Cloud::Google.getGoogleMetaData("instance/service-accounts/default/email")
        else
          nil
        end
      end

      @@default_project_cache = {}

      # Our credentials map to a project, an organizational structure in Google
      # Cloud. This fetches the identifier of the project associated with our
      # default credentials.
      # @param credentials [String]
      # @return [String]
      def self.defaultProject(credentials = nil)
        if @@default_project_cache.has_key?(credentials)
          return @@default_project_cache[credentials]
        end
        cfg = credConfig(credentials)
        if !cfg or !cfg['project']
          if hosted?
            @@default_project_cache[credentials] = myProject
            return myProject 
          end
          if cfg
            begin
              result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
              result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" }
              available = result.projects.map { |p| p.project_id }
              if available.size == 1
                @@default_project_cache[credentials] = available[0]
                return available[0]
              end
            rescue # fine
            end
          end
        end
        return nil if !cfg
        loadCredentials(credentials) if !@@authorizers[credentials]
        @@default_project_cache[credentials] = cfg['project']
        cfg['project']
      end

      # We want a default place to put new projects for the Habitat resource,
      # so if we have a root folder, we can go ahead and use that.
      # @param credentials [String]
      # @return [String]
      def self.defaultFolder(credentials = nil)
        project = defaultProject(credentials)
        resp = MU::Cloud::Google.resource_manager(credentials: credentials).get_project_ancestry(project)
        resp.ancestor.each { |a|
          if a.resource_id.type == "folder"
            return a.resource_id.id
          end
        }
        nil
      end

      # List all Google Cloud Platform projects available to our credentials
      def self.listProjects(credentials = nil)
        cfg = credConfig(credentials)
        return [] if !cfg or !cfg['project']
        result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
        result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" }
        result.projects.map { |p| p.project_id }
      end

      @@regions = {}
      # List all known Google Cloud Platform regions
      # @param us_only [Boolean]: Restrict results to United States only
      def self.listRegions(us_only = false, credentials: nil)
        if !MU::Cloud::Google.defaultProject(credentials)
          return []
        end
        if @@regions.size == 0
          begin
            result = MU::Cloud::Google.compute(credentials: credentials).list_regions(MU::Cloud::Google.defaultProject(credentials))
          rescue ::Google::Apis::ClientError => e
            if e.message.match(/forbidden/)
              raise MuError, "Insufficient permissions to list Google Cloud region. The service account #{myServiceAccount} should probably have the project owner role."
            end
            raise e
          end

          regions = []
          result.items.each { |region|
            @@regions[region.name] = []
            region.zones.each { |az|
              @@regions[region.name] << az.sub(/^.*?\/([^\/]+)$/, '\1')
            }
          }
        end
        if us_only
          @@regions.keys.delete_if { |r| !r.match(/^us/) }
        else
          @@regions.keys
        end
      end


      @@instance_types = nil
      # Query the GCP API for the list of valid Compute instance types and some of
      # their attributes. We can use this in config validation and to help
      # "translate" machine types across cloud providers.
      # @param region [String]: Supported machine types can vary from region to region, so we look for the set we're interested in specifically
      # @return [Hash]
      def self.listInstanceTypes(region = self.myRegion, credentials: nil, project: MU::Cloud::Google.defaultProject)
        return {} if !credConfig(credentials)
        if @@instance_types and
           @@instance_types[project] and
           @@instance_types[project][region]
          return @@instance_types
        end

        return {} if !project

        @@instance_types ||= {}
        @@instance_types[project] ||= {}
        @@instance_types[project][region] ||= {}
        result = MU::Cloud::Google.compute(credentials: credentials).list_machine_types(project, listAZs(region).first)
        result.items.each { |type|
          @@instance_types[project][region][type.name] ||= {}
          @@instance_types[project][region][type.name]["memory"] = sprintf("%.1f", type.memory_mb/1024.0).to_f
          @@instance_types[project][region][type.name]["vcpu"] = type.guest_cpus.to_f
          if type.is_shared_cpu
            @@instance_types[project][region][type.name]["ecu"] = "Variable"
          else
            @@instance_types[project][region][type.name]["ecu"] = type.guest_cpus
          end
        }
        @@instance_types
      end

      # Google has fairly strict naming conventions (all lowercase, no
      # underscores, etc). Provide a wrapper to our standard names to handle it.
      def self.nameStr(name)
        name.downcase.gsub(/[^a-z0-9\-]/, "-")
      end
  
      # List the Availability Zones associated with a given Google Cloud
      # region. If no region is given, search the one in which this MU master
      # server resides (if it resides in this cloud provider's ecosystem).
      # @param region [String]: The region to search.
      # @return [Array<String>]: The Availability Zones in this region.
      def self.listAZs(region = self.myRegion)
        return [] if !credConfig
        MU::Cloud::Google.listRegions if !@@regions.has_key?(region)
        if !@@regions.has_key?(region)
          MU.log "Failed to get GCP region #{region}", MU::ERR, details: @@regions
          raise MuError, "No such Google Cloud region '#{region}'" if !@@regions.has_key?(region)
        end
        @@regions[region]
      end

      # Google's Compute Service API
      # @param subclass [<Google::Apis::ComputeV1>]: If specified, will return the class ::Google::Apis::ComputeV1::subclass instead of an API client instance
      def self.compute(subclass = nil, credentials: nil)
        require 'google/apis/compute_v1'

        if subclass.nil?
          @@compute_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ComputeV1::ComputeService", scopes: ['cloud-platform', 'compute.readonly'], credentials: credentials)
          return @@compute_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("ComputeV1").const_get(subclass)
        end
      end

      # Google's Storage Service API
      # @param subclass [<Google::Apis::StorageV1>]: If specified, will return the class ::Google::Apis::StorageV1::subclass instead of an API client instance
      def self.storage(subclass = nil, credentials: nil)
        require 'google/apis/storage_v1'

        if subclass.nil?
          @@storage_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "StorageV1::StorageService", scopes: ['cloud-platform'], credentials: credentials)
          return @@storage_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("StorageV1").const_get(subclass)
        end
      end

      # Google's IAM Service API
      # @param subclass [<Google::Apis::IamV1>]: If specified, will return the class ::Google::Apis::IamV1::subclass instead of an API client instance
      def self.iam(subclass = nil, credentials: nil)
        require 'google/apis/iam_v1'

        if subclass.nil?
          @@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials)
          return @@iam_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("IamV1").const_get(subclass)
        end
      end

      # GCP's AdminDirectory Service API
      # @param subclass [<Google::Apis::AdminDirectoryV1>]: If specified, will return the class ::Google::Apis::AdminDirectoryV1::subclass instead of an API client instance
      def self.admin_directory(subclass = nil, credentials: nil)
        require 'google/apis/admin_directory_v1'

        # fill in the default credential set name so we don't generate
        # dopey extra warnings about falling back on scopes
        credentials ||= MU::Cloud::Google.credConfig(credentials, name_only: true)

        writescopes = ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema']
        readscopes = ['admin.directory.group.member.readonly', 'admin.directory.group.readonly', 'admin.directory.user.readonly', 'admin.directory.domain.readonly', 'admin.directory.orgunit.readonly', 'admin.directory.rolemanagement.readonly', 'admin.directory.customer.readonly', 'admin.directory.user.alias.readonly', 'admin.directory.userschema.readonly']
        @@readonly_semaphore.synchronize {
          use_scopes = readscopes+writescopes
          if @@readonly[credentials] and @@readonly[credentials]["AdminDirectoryV1"]
            use_scopes = readscopes.dup
          end

          if subclass.nil?
            begin
              @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
            rescue Signet::AuthorizationError => e
              MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
              @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
              @@readonly[credentials] ||= {}
              @@readonly[credentials]["AdminDirectoryV1"] = true
            end
            return @@admin_directory_api[credentials]
          elsif subclass.is_a?(Symbol)
            return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass)
          end
        }
      end

      # Google's Cloud Resource Manager API
      # @param subclass [<Google::Apis::CloudresourcemanagerV1>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV1::subclass instead of an API client instance
      def self.resource_manager(subclass = nil, credentials: nil)
        require 'google/apis/cloudresourcemanager_v1'

        if subclass.nil?
          if !MU::Cloud::Google.credConfig(credentials)
            raise MuError, "No such credential set #{credentials} defined in mu.yaml!"
          end
          @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
          return @@resource_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass)
        end
      end

      # Google's Cloud Resource Manager API V2, which apparently has all the folder bits
      # @param subclass [<Google::Apis::CloudresourcemanagerV2>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV2::subclass instead of an API client instance
      def self.folder(subclass = nil, credentials: nil)
        require 'google/apis/cloudresourcemanager_v2'

        if subclass.nil?
          @@resource2_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV2::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformfolders'], credentials: credentials,  masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
          return @@resource2_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV2").const_get(subclass)
        end
      end

      # Google's Container API
      # @param subclass [<Google::Apis::ContainerV1>]: If specified, will return the class ::Google::Apis::ContainerV1::subclass instead of an API client instance
      def self.container(subclass = nil, credentials: nil)
        require 'google/apis/container_v1'

        if subclass.nil?
          @@container_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ContainerV1::ContainerService", scopes: ['cloud-platform'], credentials: credentials)
          return @@container_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("ContainerV1").const_get(subclass)
        end
      end

      # Google's Service Manager API (the one you use to enable pre-project APIs)
      # @param subclass [<Google::Apis::ServicemanagementV1>]: If specified, will return the class ::Google::Apis::ServicemanagementV1::subclass instead of an API client instance
      def self.service_manager(subclass = nil, credentials: nil)
        require 'google/apis/servicemanagement_v1'

        if subclass.nil?
          @@service_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ServicemanagementV1::ServiceManagementService", scopes: ['cloud-platform'], credentials: credentials)
          return @@service_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("ServicemanagementV1").const_get(subclass)
        end
      end

      # Google's SQL Service API
      # @param subclass [<Google::Apis::SqladminV1beta4>]: If specified, will return the class ::Google::Apis::SqladminV1beta4::subclass instead of an API client instance
      def self.sql(subclass = nil, credentials: nil)
        require 'google/apis/sqladmin_v1beta4'

        if subclass.nil?
          @@sql_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "SqladminV1beta4::SQLAdminService", scopes: ['cloud-platform'], credentials: credentials)
          return @@sql_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("SqladminV1beta4").const_get(subclass)
        end
      end

      # Google's Firestore (NoSQL) Service API
      # @param subclass [<Google::Apis::FirestoreV1>]: If specified, will return the class ::Google::Apis::FirestoreV1::subclass instead of an API client instance
      def self.firestore(subclass = nil, credentials: nil)
        require 'google/apis/firestore_v1'

        if subclass.nil?
          @@firestore_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "FirestoreV1::FirestoreService", scopes: ['cloud-platform'], credentials: credentials)
          return @@firestore_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("FirestoreV1").const_get(subclass)
        end
      end

      # Google's StackDriver Logging Service API
      # @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
      def self.logging(subclass = nil, credentials: nil)
        require 'google/apis/logging_v2'

        if subclass.nil?
          @@logging_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "LoggingV2::LoggingService", scopes: ['cloud-platform'], credentials: credentials)
          return @@logging_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("LoggingV2").const_get(subclass)
        end
      end

      # Google's Cloud Billing Service API
      # @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
      def self.billing(subclass = nil, credentials: nil)
        require 'google/apis/cloudbilling_v1'

        if subclass.nil?
          @@billing_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudbillingV1::CloudbillingService", scopes: ['cloud-platform', 'cloud-billing'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
          return @@billing_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("CloudbillingV1").const_get(subclass)
        end
      end

      # Google's Cloud Function Service API
      # @param subclass [<Google::Apis::CloudfunctionsV1>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
      def self.function(subclass = nil, credentials: nil)
        require 'google/apis/cloudfunctions_v1'

        if subclass.nil?
          @@function_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudfunctionsV1::CloudFunctionsService", scopes: ['cloud-platform'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
          return @@function_api[credentials]
        elsif subclass.is_a?(Symbol)
          return Object.const_get("::Google").const_get("Apis").const_get("CloudfunctionsV1").const_get(subclass)
        end
      end

      # Retrieve the domains, if any, which these credentials can manage via
      # GSuite or Cloud Identity.
      # @param credentials [String]
      # @return [Array<String>],nil]
      def self.getDomains(credentials = nil)
        my_org = getOrg(credentials)
        return nil if !my_org

        resp = MU::Cloud::Google.admin_directory(credentials: credentials).list_domains(MU::Cloud::Google.customerID(credentials))
        resp.domains.map { |d| d.domain_name.downcase }
      end

      @@orgmap = {}
      # Retrieve the organization, if any, to which these credentials belong.
      # @param credentials [String]
      # @return [Array<OpenStruct>],nil]
      def self.getOrg(credentials = nil, with_id: nil)
        creds = MU::Cloud::Google.credConfig(credentials)
        credname = if creds and creds['name']
          creds['name']
        else
          "default"
        end 

        return @@orgmap[credname] if @@orgmap.has_key?(credname)
        resp = MU::Cloud::Google.resource_manager(credentials: credname).search_organizations
        if resp and resp.organizations
          # XXX no idea if it's possible to be a member of multiple orgs
          if !with_id
            @@orgmap[credname] = resp.organizations.first
            return resp.organizations.first
          else
            resp.organizations.each { |org|
              if org.name == with_id
                @@orgmap[credname] = org
                return org
              end
            }
            return nil
          end
        end

        @@orgmap[credname] = nil

        
        MU.log "Unable to list_organizations with credentials #{credname}. If this account is part of a GSuite or Cloud Identity domain, verify that Oauth delegation is properly configured and that 'masquerade_as' is properly set for the #{credname} Google credential set in mu.yaml.", MU::ERR, details: ["https://cloud.google.com/resource-manager/docs/creating-managing-organization", "https://admin.google.com/AdminHome?chromeless=1#OGX:ManageOauthClients"]

        nil
      end

      @@customer_ids_cache = {}

      # Fetch the GSuite/Cloud Identity customer id for the domain associated
      # with the given credentials, if a domain is set via the +masquerade_as+
      # configuration option.
      def self.customerID(credentials = nil)
        cfg = credConfig(credentials)
        if !cfg or !cfg['masquerade_as']
          return nil
        end

        if @@customer_ids_cache[credentials]
          return @@customer_ids_cache[credentials]
        end

        user = MU::Cloud::Google.admin_directory(credentials: credentials).get_user(cfg['masquerade_as'])
        if user and user.customer_id
          @@customer_ids_cache[credentials] = user.customer_id
        end

        @@customer_ids_cache[credentials]
      end

      private

      # Wrapper class for Google APIs, so that we can catch some common
      # transient endpoint errors without having to spray rescues all over the
      # codebase.
      class GoogleEndpoint
        @api = nil
        @credentials = nil
        @scopes = nil
        @masquerade = nil
        attr_reader :issuer

        # Create a Google Cloud Platform API client
        # @param api [String]: Which API are we wrapping?
        # @param scopes [Array<String>]: Google auth scopes applicable to this API
        def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false)
          @credentials = credentials
          @scopes = scopes.map { |s|
            if !s.match(/\//) # allow callers to use shorthand
              s = "https://www.googleapis.com/auth/"+s
            end
            s
          }
          @masquerade = masquerade
          @api = Object.const_get("Google::Apis::#{api}").new
          @api.authorization = MU::Cloud::Google.loadCredentials(@scopes, credentials: credentials)
          raise MuError, "No useable Google credentials found#{credentials ? " with set '#{credentials}'" : ""}" if @api.authorization.nil?
          if @masquerade
            begin
              @api.authorization.sub = @masquerade
              @api.authorization.fetch_access_token!
            rescue Signet::AuthorizationError => e
              if auth_error_quiet
                MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::DEBUG, details: @scopes
              else
                MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::ERROR, details: @scopes
                if e.message.match(/client not authorized for any of the scopes requested/)
# XXX it'd be helpful to list *all* scopes we like, as well as the API client's numeric id
                  MU.log "To grant access to API scopes for this service account, see:", MU::ERR, details: "https://admin.google.com/AdminHome?chromeless=1#OGX:ManageOauthClients"
                end
              end

              raise e
            end
          end
          @issuer = @api.authorization.issuer
        end

        # Generic wrapper for deleting Compute resources, which are consistent
        # enough that we can get away with this.
        # @param type [String]: The type of resource, typically the string you'll find in all of the API calls referring to it
        # @param project [String]: The project in which we should look for the resources
        # @param region [String]: The region in which to loop for the resources
        # @param noop [Boolean]: If true, will only log messages about resources to be deleted, without actually deleting them
        # @param filter [String]: The Compute API filter string to use to isolate appropriate resources
        def delete(type, project, region = nil, noop = false, filter = "description eq #{MU.deploy_id}", credentials: nil)
          list_sym = "list_#{type.sub(/y$/, "ie")}s".to_sym
          resp = nil
          begin
            if region
              resp = MU::Cloud::Google.compute(credentials: @credentials).send(list_sym, project, region, filter: filter, mu_gcp_enable_apis: false)
            else
              resp = MU::Cloud::Google.compute(credentials: @credentials).send(list_sym, project, filter: filter, mu_gcp_enable_apis: false)
            end

          rescue ::Google::Apis::ClientError => e
            return if e.message.match(/^notFound: /)
          end

          if !resp.nil? and !resp.items.nil?
            threads = []
            parent_thread_id = Thread.current.object_id
            resp.items.each { |obj|
              threads << Thread.new {
                MU.dupGlobals(parent_thread_id)
                Thread.abort_on_exception = false
                MU.log "Removing #{type.gsub(/_/, " ")} #{obj.name}"
                delete_sym = "delete_#{type}".to_sym
                if !noop
                  retries = 0
                  failed = false
                  begin
                    resp = nil
                    failed = false
                    if region
                      resp = MU::Cloud::Google.compute(credentials: @credentials).send(delete_sym, project, region, obj.name)
                    else
                      resp = MU::Cloud::Google.compute(credentials: @credentials).send(delete_sym, project, obj.name)
                    end

                    if resp.error and resp.error.errors and resp.error.errors.size > 0
                      failed = true
                      retries += 1
                      if resp.error.errors.first.code == "RESOURCE_IN_USE_BY_ANOTHER_RESOURCE" and retries < 6
                        sleep 10
                      else
                        MU.log "Error deleting #{type.gsub(/_/, " ")} #{obj.name}", MU::ERR, details: resp.error.errors
                        Thread.abort_on_exception = false
                        raise MuError, "Failed to delete #{type.gsub(/_/, " ")} #{obj.name}"
                      end
                    else
                      failed = false
                    end
# TODO validate that the resource actually went away, because it seems not to do so very reliably
                  rescue ::Google::Apis::ClientError => e
                    raise e if !e.message.match(/(^notFound: |operation in progress)/)
                  rescue MU::Cloud::MuDefunctHabitat => e
                    # this is ok- it's already deleted
                  end while failed and retries < 6
                end
              }
            }
            threads.each do |t|
              t.join
            end

          end
        end

        @instance_cache = {}
        # Catch-all for AWS client methods. Essentially a pass-through with some
        # rescues for known silly endpoint behavior.
        def method_missing(method_sym, *arguments)
          retries = 0
          actual_resource = nil

          enable_on_fail = true
          arguments.each { |arg|
            if arg.is_a?(Hash) and arg.has_key?(:mu_gcp_enable_apis)
              enable_on_fail = arg[:mu_gcp_enable_apis]
              arg.delete(:mu_gcp_enable_apis)
              
            end
          }
          arguments.delete({})
          next_page_token = nil
          overall_retval = nil

          begin
            MU.log "Calling #{method_sym}", MU::DEBUG, details: arguments
            retval = nil
            retries = 0
            wait_backoff = 5
            if next_page_token
              if arguments.size == 1 and arguments.first.is_a?(Hash)
                arguments[0][:page_token] = next_page_token
              else
                arguments << { :page_token => next_page_token }
              end
            end
            begin
              if !arguments.nil? and arguments.size == 1
                retval = @api.method(method_sym).call(arguments[0])
              elsif !arguments.nil? and arguments.size > 0
                retval = @api.method(method_sym).call(*arguments)
              else
                retval = @api.method(method_sym).call
              end
            rescue ArgumentError => e
              MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s}: #{e.message}", MU::ERR, details: arguments
              raise e
            rescue ::Google::Apis::AuthorizationError => e
              if arguments.size > 0
                raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym} in project #{arguments.first}"
              else
                raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym}"
              end
            rescue ::Google::Apis::RateLimitError, ::Google::Apis::TransmissionError, ::ThreadError, ::Google::Apis::ServerError => e
              if retries <= 10
                sleep wait_backoff
                retries += 1
                wait_backoff = wait_backoff * 2
                retry
              else
                raise e
              end
            rescue ::Google::Apis::ClientError, OpenSSL::SSL::SSLError => e
              if e.message.match(/^quotaExceeded: Request rate/)
                if retries <= 10
                  sleep wait_backoff
                  retries += 1
                  wait_backoff = wait_backoff * 2
                  retry
                else
                  raise e
                end
              elsif e.message.match(/^invalidParameter:|^badRequest:/)
                MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s}: "+e.message, MU::ERR, details: arguments
# uncomment for debugging stuff; this can occur in benign situations so we don't normally want it logging
              elsif e.message.match(/^forbidden:/)
                MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s} got \"#{e.message}\" using credentials #{@credentials}#{@masquerade ? " (OAuth'd as #{@masquerade})": ""}.#{@scopes ? "\nScopes:\n#{@scopes.join("\n")}" : "" }", MU::DEBUG, details: arguments
                raise e
              end
              @@enable_semaphores ||= {}
              max_retries = 3
              wait_time = 90
              if enable_on_fail and retries <= max_retries and e.message.match(/^accessNotConfigured/)
                enable_obj = nil

                project = if arguments.size > 0 and arguments.first.is_a?(String)
                  arguments.first
                else
                  MU::Cloud::Google.defaultProject(@credentials)
                end
# XXX validate that this actually looks like a project id, maybe
                if method_sym == :delete and !MU::Cloud::Google::Habitat.isLive?(project, @credentials)
                  MU.log "Got accessNotConfigured while attempting to delete a resource in #{project}", MU::WARN
                   
                  return
                end

                @@enable_semaphores[project] ||= Mutex.new
                enable_obj = MU::Cloud::Google.service_manager(:EnableServiceRequest).new(
                  consumer_id: "project:"+project.gsub(/^projects\/([^\/]+)\/.*/, '\1')
                )
                # XXX dumbass way to get this string
                if e.message.match(/by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)

                  svc_name = Regexp.last_match[1]
                  save_verbosity = MU.verbosity
                  if svc_name != "servicemanagement.googleapis.com" and method_sym != :delete
                    retries += 1
                    @@enable_semaphores[project].synchronize {
                      MU.setLogging(MU::Logger::NORMAL)
                      MU.log "Attempting to enable #{svc_name} in project #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
                      MU.setLogging(save_verbosity)
                      begin
                        MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
                      rescue ::Google::Apis::ClientError => e
                        MU.log "Error enabling #{svc_name} in #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj
                        raise e
                      end
                    }
                    sleep wait_time/retries
                    retry
                  else
                    MU.setLogging(MU::Logger::NORMAL)
                    MU.log "Google Cloud's Service Management API must be enabled manually by visiting #{e.message.gsub(/.*?(https?:\/\/[^\s]+)(?:$|\s).*/, '\1')}", MU::ERR
                    MU.setLogging(save_verbosity)
                    raise MU::MuError, "Service Management API not yet enabled for this account/project"
                  end
                elsif e.message.match(/scheduled for deletion and cannot be used for API calls/)
                  raise MuDefunctHabitat, e.message
                else
                  MU.log "Unfamiliar error calling #{method_sym.to_s} "+e.message, MU::ERR, details: arguments
                end
              elsif retries <= 10 and
                 e.message.match(/^resourceNotReady:/) or
                 (e.message.match(/^resourceInUseByAnotherResource:/) and method_sym.to_s.match(/^delete_/)) or
                 e.message.match(/SSL_connect/)
                if retries > 0 and retries % 3 == 0
                  MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::NOTICE, details: arguments
                else
                  MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::DEBUG, details: arguments
                end
                retries = retries + 1
                sleep retries*10
                retry
              else
                raise e
              end
            end

            if retval.class.name.match(/.*?::Operation$/)

              retries = 0
              orig_target = retval.name

              # Check whether the various types of +Operation+ responses say
              # they're done, without knowing which specific API they're from
              def is_done?(retval)
                (retval.respond_to?(:status) and retval.status == "DONE") or (retval.respond_to?(:done) and retval.done)
              end

              begin
                if retries > 0 and retries % 3 == 0
                  MU.log "Waiting for #{method_sym} to be done (retry #{retries})", MU::NOTICE
                else
                  MU.log "Waiting for #{method_sym} to be done (retry #{retries})", MU::DEBUG, details: retval
                end

                if !is_done?(retval)
                  sleep 7
                  begin
                    if retval.class.name.match(/::Compute[^:]*::/)
                      resp = MU::Cloud::Google.compute(credentials: @credentials).get_global_operation(
                        arguments.first, # there's always a project id
                        retval.name
                      )
                      retval = resp
                    elsif retval.class.name.match(/::Servicemanagement[^:]*::/)
                      resp = MU::Cloud::Google.service_manager(credentials: @credentials).get_operation(
                        retval.name
                      )
                      retval = resp
                    elsif retval.class.name.match(/::Cloudresourcemanager[^:]*::/)
                      resp = MU::Cloud::Google.resource_manager(credentials: @credentials).get_operation(
                        retval.name
                      )
                      retval = resp
                      if retval.error
                        raise MuError, retval.error.message
                      end
                    elsif retval.class.name.match(/::Container[^:]*::/)
                      resp = MU::Cloud::Google.container(credentials: @credentials).get_project_location_operation(
                        retval.self_link.sub(/.*?\/projects\//, 'projects/')
                      )
                      retval = resp
                    elsif retval.class.name.match(/::Cloudfunctions[^:]*::/)
                      resp = MU::Cloud::Google.function(credentials: @credentials).get_operation(
                        retval.name
                      )
                      retval = resp
#MU.log method_sym.to_s, MU::WARN, details: retval
                      if retval.error
                        raise MuError, retval.error.message
                      end
                    else
                      pp retval
                      raise MuError, "I NEED TO IMPLEMENT AN OPERATION HANDLER FOR #{retval.class.name}"
                    end
                  rescue ::Google::Apis::ClientError => e
                    # this is ok; just means the operation is done and went away
                    if e.message.match(/^notFound:/)
                      break
                    else
                      raise e
                    end
                  end
                  retries = retries + 1
                end

              end while !is_done?(retval)

              # Most insert methods have a predictable get_* counterpart. Let's
              # take advantage.
              # XXX might want to do something similar for delete ops? just the
              # but where we wait for the operation to definitely be done
              had_been_found = false
              if method_sym.to_s.match(/^(insert|create|patch)_/)
                get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym
                cloud_id = if retval.respond_to?(:target_link)
                  retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1')
                elsif retval.respond_to?(:metadata) and retval.metadata["target"]
                  retval.metadata["target"]
                else
                  arguments[0] # if we're lucky
                end
                faked_args = arguments.dup
                faked_args.pop
                if get_method == :get_snapshot
                  faked_args.pop
                  faked_args.pop
                end
                faked_args.push(cloud_id)
                if get_method == :get_project_location_cluster
                  faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1]
                  faked_args.pop
                elsif get_method == :get_project_location_function
                  faked_args = [cloud_id]
                end
                actual_resource = @api.method(get_method).call(*faked_args)
#if method_sym == :insert_instance
#MU.log "actual_resource", MU::WARN, details: actual_resource
#end
                had_been_found = true
                if actual_resource.respond_to?(:status) and
                  ["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status)
                  retries = 0
                  begin 
                    if retries > 0 and retries % 3 == 0
                      MU.log "Waiting for #{cloud_id} to get past #{actual_resource.status} (retry #{retries})", MU::NOTICE
                    else
                      MU.log "Waiting for #{cloud_id} to get past #{actual_resource.status} (retry #{retries})", MU::DEBUG, details: actual_resource
                    end
                    sleep 10
                    actual_resource = @api.method(get_method).call(*faked_args)
                    retries = retries + 1
                  end while ["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status)
                end
                return actual_resource
              end
            end

            # This atrocity appends the pages of list_* results
            if overall_retval
              if method_sym.to_s.match(/^list_(.*)/)
                require 'google/apis/iam_v1'
                what = Regexp.last_match[1].to_sym
                whatassign = (Regexp.last_match[1]+"=").to_sym
                if overall_retval.class == ::Google::Apis::IamV1::ListServiceAccountsResponse
                  what = :accounts
                  whatassign = :accounts=
                end
                if retval.respond_to?(what) and retval.respond_to?(whatassign)
                  if !retval.public_send(what).nil?
                    newarray = retval.public_send(what) + overall_retval.public_send(what)
                    overall_retval.public_send(whatassign, newarray)
                  end
                else
                  MU.log "Not sure how to append #{method_sym.to_s} results to #{overall_retval.class.name} (apparently #{what.to_s} and #{whatassign.to_s} aren't it), returning first page only", MU::WARN, details: retval
                  return retval
                end
              else
                MU.log "Not sure how to append #{method_sym.to_s} results, returning first page only", MU::WARN, details: retval
                return retval
              end
            else
              overall_retval = retval
            end

            arguments.delete({ :page_token => next_page_token })
            next_page_token = nil

            if retval.respond_to?(:next_page_token) and !retval.next_page_token.nil?
              next_page_token = retval.next_page_token
              MU.log "Getting another page of #{method_sym.to_s}", MU::DEBUG, details: next_page_token
            else
              return overall_retval
            end
          rescue ::Google::Apis::ServerError, ::Google::Apis::ClientError, ::Google::Apis::TransmissionError => e
            if e.class.name == "Google::Apis::ClientError" and
               (!method_sym.to_s.match(/^insert_/) or !e.message.match(/^notFound: /) or
                (e.message.match(/^notFound: /) and method_sym.to_s.match(/^insert_/))
               )
              if e.message.match(/^notFound: /) and method_sym.to_s.match(/^insert_/) and retval
                logreq = MU::Cloud::Google.logging(:ListLogEntriesRequest).new(
                  resource_names: ["projects/"+arguments.first],
                  filter: %Q{labels."compute.googleapis.com/resource_id"="#{retval.target_id}" OR labels."ssl_certificate_id"="#{retval.target_id}"} # XXX I guess we need to cover all of the possible keys, ugh
                )
                logs = MU::Cloud::Google.logging(credentials: @credentials).list_entry_log_entries(logreq)
                details = nil
                if logs.entries
                  details = logs.entries.map { |err| err.json_payload }
                  details.reject! { |err| err["error"].nil? or err["error"].size == 0 }
                end

                raise MuError, "#{method_sym.to_s} of #{retval.target_id} appeared to succeed, but then the resource disappeared! #{details.to_s}"
              end
              raise e
            end
            retries = retries + 1
            debuglevel = MU::DEBUG
            interval = 5 + Random.rand(4) - 2
            if retries < 10 and retries > 2
              debuglevel = MU::NOTICE
              interval = 20 + Random.rand(10) - 3
            # elsif retries >= 10 and retries <= 100
            elsif retries >= 10
              debuglevel = MU::WARN
              interval = 40 + Random.rand(15) - 5
            # elsif retries > 100
              # raise MuError, "Exhausted retries after #{retries} attempts while calling Compute's #{method_sym} in #{@region}.  Args were: #{arguments}"
            end

            MU.log "Got #{e.inspect} calling Google's #{method_sym}, waiting #{interval.to_s}s and retrying. Called from: #{caller[1]}", debuglevel, details: arguments
            sleep interval
            MU.log method_sym.to_s.bold+" "+e.inspect, MU::WARN, details: arguments
            retry
          end while !next_page_token.nil?
        end
      end

      @@compute_api = {}
      @@container_api = {}
      @@storage_api = {}
      @@sql_api = {}
      @@iam_api = {}
      @@logging_api = {}
      @@resource_api = {}
      @@resource2_api = {}
      @@service_api = {}
      @@firestore_api = {}
      @@admin_directory_api = {}
      @@billing_api = {}
      @@function_api = {}
    end
  end
end