# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
#
# Licensed under the BSD-3 license (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the root of the project or at
#
#	http://egt-labs.com/mu/LICENSE.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module MU
  class Cloud
    class Google
      # A Kubernetes cluster as configured in {MU::Config::BasketofKittens::container_clusters}
      class ContainerCluster < MU::Cloud::ContainerCluster

        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like <tt>@vpc</tt>, for us.
        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
        def initialize(**args)
          super

          if !@mu_name
            @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 40)
          end
        end


        # Called automatically by {MU::Deploy#createResources}
        # @return [String]: The cloud provider's identifier for this GKE instance.
        def create
          labels = Hash[@tags.keys.map { |k|
            [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
          ]
          labels["name"] = MU::Cloud::Google.nameStr(@mu_name)

          if @vpc.nil? and @config['vpc'] and @config['vpc']['vpc_name']
            @vpc = @deploy.findLitterMate(name: @config['vpc']['vpc_name'], type: "vpcs")
          end

          if !@vpc
            raise MuError, "ContainerCluster #{@config['name']} unable to locate its resident VPC from #{@config['vpc']}"
          end


          sa = MU::Config::Ref.get(@config['service_account'])
          if sa.name and @deploy.findLitterMate(name: sa.name, type: "users")
            @service_acct = @deploy.findLitterMate(name: sa.name, type: "users").cloud_desc
          else
            if !sa or !sa.kitten or !sa.kitten.cloud_desc
              raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
            end
            @service_acct = sa.kitten.cloud_desc
          end
          if !@config['scrub_mu_isms']
            MU::Cloud::Google.grantDeploySecretAccess(@service_acct.email, credentials: @config['credentials'])
          end

          @config['ssh_user'] ||= "muadmin"


          nodeobj = if @config['min_size'] and @config['max_size']
            MU::Cloud::Google.container(:NodePool).new(
              name: @mu_name.downcase,
              initial_node_count: @config['instance_count'] || @config['min_size'],
              autoscaling: MU::Cloud::Google.container(:NodePoolAutoscaling).new(
                enabled: true,
                min_node_count: @config['min_size'],
                max_node_count: @config['max_size'],
              ),
              management: MU::Cloud::Google.container(:NodeManagement).new(
                auto_upgrade: @config['auto_upgrade'],
                auto_repair: @config['auto_repair']
              ),
              config: MU::Cloud::Google.container(:NodeConfig).new(node_desc)
            )
          else
            MU::Cloud::Google.container(:NodeConfig).new(node_desc)
          end
          locations = if @config['availability_zone']
            [@config['availability_zone']]
          else
            MU::Cloud::Google.listAZs(@config['region'])
          end

          master_user = @config['master_user']
          # We'll create a temporary basic auth config so that we can grant
          # useful permissions to the Client Certificate user
          master_user ||= "master_user"
          master_pw = Password.pronounceable(18)

          desc = {
            :name => @mu_name.downcase,
            :description => @deploy.deploy_id,
            :network => @vpc.cloud_id,
            :enable_tpu => @config['tpu'],
            :resource_labels => labels,
            :locations => locations,
            :master_auth => MU::Cloud::Google.container(:MasterAuth).new(
              :client_certificate_config => MU::Cloud::Google.container(:ClientCertificateConfig).new(
                :issue_client_certificate => true
              ),
              :username => master_user,
              :password => master_pw
            ),
          }

          if @config['kubernetes']
            desc[:addons_config] = MU::Cloud::Google.container(:AddonsConfig).new(
              horizontal_pod_autoscaling: MU::Cloud::Google.container(:HorizontalPodAutoscaling).new(
                disabled: !@config['kubernetes']['horizontal_pod_autoscaling']
              ),
              http_load_balancing: MU::Cloud::Google.container(:HttpLoadBalancing).new(
                disabled: !@config['kubernetes']['http_load_balancing']
              ),
              kubernetes_dashboard: MU::Cloud::Google.container(:KubernetesDashboard).new(
                disabled: !@config['kubernetes']['dashboard']
              ),
              network_policy_config: MU::Cloud::Google.container(:NetworkPolicyConfig).new(
                disabled: !@config['kubernetes']['network_policy_addon']
              )
            )
          end

          # Pick an existing subnet from our VPC, if we're not going to create
          # one.
          if !@config['custom_subnet']
            @vpc.subnets.each { |s|
              if s.az == @config['region']
                desc[:subnetwork] = s.cloud_id
                break
              end
            }
          end
          if @config['log_facility'] == "kubernetes"
            desc[:logging_service] = "logging.googleapis.com/kubernetes"
            desc[:monitoring_service] = "monitoring.googleapis.com/kubernetes"
          elsif @config['log_facility'] == "basic"
            desc[:logging_service] = "logging.googleapis.com"
            desc[:monitoring_service] = "monitoring.googleapis.com"
          else
            desc[:logging_service] = "none"
            desc[:monitoring_service] = "none"
          end

          if nodeobj.is_a?(::Google::Apis::ContainerV1::NodeConfig)
            desc[:node_config] = nodeobj
            desc[:initial_node_count] = @config['instance_count']
          else
            desc[:node_pools] = [nodeobj]
          end

          if @config['kubernetes']
            if @config['kubernetes']['version']
              desc[:initial_cluster_version] = @config['kubernetes']['version']
            end
            if @config['kubernetes']['alpha']
              desc[:enable_kubernetes_alpha] = @config['kubernetes']['alpha']
            end
          end

          if @config['preferred_maintenance_window']
            desc[:maintenance_policy] = MU::Cloud::Google.container(:MaintenancePolicy).new(
              window: MU::Cloud::Google.container(:MaintenanceWindow).new(
                daily_maintenance_window: MU::Cloud::Google.container(:DailyMaintenanceWindow).new(
                  start_time: @config['preferred_maintenance_window']
                )
              )
            )
          end

          if @config['private_cluster']
            desc[:private_cluster_config] = MU::Cloud::Google.container(:PrivateClusterConfig).new(
              enable_private_endpoint: @config['private_cluster']['private_master'],
              enable_private_nodes: @config['private_cluster']['private_nodes'],
              master_ipv4_cidr_block: @config['private_cluster']['master_ip_block']
            )
            desc[:ip_allocation_policy] = MU::Cloud::Google.container(:IpAllocationPolicy).new(
              use_ip_aliases: true
            )
          end

          if @config['ip_aliases'] or @config['custom_subnet'] or
             @config['services_ip_block'] or @config['services_ip_block_name'] or
             @config['pod_ip_block'] or @config['pod_ip_block_name'] or
             @config['tpu_ip_block']
            alloc_desc = { :use_ip_aliases => @config['ip_aliases'] }

            if @config['custom_subnet']
              alloc_desc[:create_subnetwork] = true
              alloc_desc[:subnetwork_name] = if @config['custom_subnet']['name']
                @config['custom_subnet']['name']
              else
                @mu_name.downcase
              end

              if @config['custom_subnet']['node_ip_block']
                alloc_desc[:node_ipv4_cidr_block] = @config['custom_subnet']['node_ip_block']
              end
            else
              if @config['pod_ip_block_name']
                alloc_desc[:cluster_secondary_range_name] = @config['pod_ip_block_name']
              end
              if @config['services_ip_block_name']
                alloc_desc[:services_secondary_range_name] = @config['services_ip_block_name']
              end
            end

            if @config['services_ip_block']
              alloc_desc[:services_ipv4_cidr_block] = @config['services_ip_block']
            end
            if @config['tpu_ip_block']
              alloc_desc[:tpu_ipv4_cidr_block] = @config['tpu_ip_block']
            end
            if @config['pod_ip_block']
              alloc_desc[:cluster_ipv4_cidr_block] = @config['pod_ip_block']
            end

            desc[:ip_allocation_policy] = MU::Cloud::Google.container(:IpAllocationPolicy).new(alloc_desc)
            pp alloc_desc
          end

          if @config['authorized_networks'] and @config['authorized_networks'].size > 0
            desc[:master_authorized_networks_config] = MU::Cloud::Google.container(:MasterAuthorizedNetworksConfig).new(
              enabled: true,
              cidr_blocks: @config['authorized_networks'].map { |n|
                MU::Cloud::Google.container(:CidrBlock).new(
                  cidr_block: n['ip_block'],
                  display_name: n['label']
                )
              }
            )
          end

          if @config['kubernetes'] and @config['kubernetes']['max_pods'] and
             @config['ip_aliases']
            desc[:default_max_pods_constraint] = MU::Cloud::Google.container(:MaxPodsConstraint).new(
              max_pods_per_node: @config['kubernetes']['max_pods']
            )
          end

          requestobj = MU::Cloud::Google.container(:CreateClusterRequest).new(
            :cluster => MU::Cloud::Google.container(:Cluster).new(desc),
          )

          MU.log "Creating GKE cluster #{@mu_name.downcase}", details: requestobj
          @config['master_az'] = @config['region']
          parent_arg = "projects/"+@config['project']+"/locations/"+@config['master_az']

          MU::Cloud::Google.container(credentials: @config['credentials']).create_project_location_cluster(
            parent_arg,
            requestobj
          )
          @cloud_id = parent_arg+"/clusters/"+@mu_name.downcase

          resp = nil
          begin
            resp = MU::Cloud::Google.container(credentials: @config['credentials']).get_project_location_cluster(@cloud_id)
            if resp.status == "ERROR"
              MU.log "GKE cluster #{@cloud_id} failed", MU::ERR, details: resp.status_message
              raise MuError, "GKE cluster #{@cloud_id} failed: #{resp.status_message}"
            end
            sleep 30 if resp.status != "RUNNING"
          end while resp.nil? or resp.status != "RUNNING"

          writeKubeConfig

        end


        # Called automatically by {MU::Deploy#createResources}
        def groom
          labelCluster 

          me = cloud_desc

          # Enable/disable basic auth
          authcfg = {}

          if @config['master_user'] and (me.master_auth.username != @config['master_user'] or !me.master_auth.password)
            authcfg[:username] = @config['master_user']
            authcfg[:password] = Password.pronounceable(16..18)
            MU.log "Enabling basic auth for GKE cluster #{@mu_name.downcase}", MU::NOTICE, details: authcfg
          elsif !@config['master_user'] and me.master_auth.username
            authcfg[:username] = ""
            MU.log "Disabling basic auth for GKE cluster #{@mu_name.downcase}", MU::NOTICE
          end
          if authcfg.size > 0
            MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_master_auth(
              @cloud_id,
              MU::Cloud::Google.container(:SetMasterAuthRequest).new(
                name: @cloud_id,
                action: "SET_USERNAME",
                update: MU::Cloud::Google.container(:MasterAuth).new(
                  authcfg
                )
              )
            )
            me = cloud_desc(use_cache: false)
          end

          # Now go through all the things that use update_project_location_cluster
          updates = []

          locations = if @config['availability_zone']
            [@config['availability_zone']]
          else
            MU::Cloud::Google.listAZs(@config['region'])
          end
          if me.locations != locations
            updates << { :desired_locations => locations }
          end

          if @config['min_size'] and @config['max_size'] and
             (me.node_pools.first.autoscaling.min_node_count != @config['min_size'] or
             me.node_pools.first.autoscaling.max_node_count != @config['max_size'])
            updates << {
              :desired_node_pool_autoscaling => MU::Cloud::Google.container(:NodePoolAutoscaling).new(
                enabled: true,
                max_node_count: @config['max_size'],
                min_node_count: @config['min_size']
              )
            }
          end

          if @config['authorized_networks'] and @config['authorized_networks'].size > 0
            desired = @config['authorized_networks'].map { |n|
              MU::Cloud::Google.container(:CidrBlock).new(
                cidr_block: n['ip_block'],
                display_name: n['label']
              )
            }
            if !me.master_authorized_networks_config or
               !me.master_authorized_networks_config.enabled or
               !me.master_authorized_networks_config.cidr_blocks or
               me.master_authorized_networks_config.cidr_blocks.map {|n| n.cidr_block+n.display_name }.sort != desired.map {|n| n.cidr_block+n.display_name }.sort
              updates << { :desired_master_authorized_networks_config => MU::Cloud::Google.container(:MasterAuthorizedNetworksConfig).new(
                enabled: true,
                cidr_blocks: desired
              )}
            end
          elsif me.master_authorized_networks_config and
                me.master_authorized_networks_config.enabled
            updates << { :desired_master_authorized_networks_config => MU::Cloud::Google.container(:MasterAuthorizedNetworksConfig).new(
              enabled: false
            )}
          end

          if @config['log_facility'] == "kubernetes" and me.logging_service != "logging.googleapis.com/kubernetes"
            updates << {
              :desired_logging_service => "logging.googleapis.com/kubernetes",
              :desired_monitoring_service => "monitoring.googleapis.com/kubernetes"
            }
          elsif @config['log_facility'] == "basic" and me.logging_service != "logging.googleapis.com"
            updates << {
              :desired_logging_service => "logging.googleapis.com",
              :desired_monitoring_service => "monitoring.googleapis.com"
            }
          elsif @config['log_facility'] == "none" and me.logging_service != "none"
            updates << {
              :desired_logging_service => "none",
              :desired_monitoring_service => "none"
            }
          end

          # map from GKE Kuberentes addon parameter names to our BoK equivalent
          # fields so we can check all these programmatically
          addon_map = {
            :horizontal_pod_autoscaling => 'horizontal_pod_autoscaling',
            :http_load_balancing => 'http_load_balancing',
            :kubernetes_dashboard => 'dashboard',
            :network_policy_config => 'network_policy_addon'
          }

          if @config['kubernetes']
            have_changes = false
            addon_map.each_pair { |param, bok_param|
              if (me.addons_config.send(param).disabled and @config['kubernetes'][bok_param]) or
                 (!me.addons_config.send(param) and !@config['kubernetes'][bok_param])
                have_changes = true
              end
            }
            if have_changes
              updates << { :desired_addons_config => MU::Cloud::Google.container(:AddonsConfig).new(
                horizontal_pod_autoscaling: MU::Cloud::Google.container(:HorizontalPodAutoscaling).new(
                  disabled: !@config['kubernetes']['horizontal_pod_autoscaling']
                ),
                http_load_balancing: MU::Cloud::Google.container(:HttpLoadBalancing).new(
                  disabled: !@config['kubernetes']['http_load_balancing']
                ),
                kubernetes_dashboard: MU::Cloud::Google.container(:KubernetesDashboard).new(
                  disabled: !@config['kubernetes']['dashboard']
                ),
                network_policy_config: MU::Cloud::Google.container(:NetworkPolicyConfig).new(
                  disabled: !@config['kubernetes']['network_policy_addon']
                )
              )}
            end 
          end

          if @config['kubernetes'] and @config['kubernetes']['version']
            if MU.version_sort(@config['kubernetes']['version'], me.current_master_version) > 0
              updates << {  :desired_master_version => @config['kubernetes']['version'] }
            end
          end

          if @config['kubernetes'] and @config['kubernetes']['nodeversion']
            if MU.version_sort(@config['kubernetes']['nodeversion'], me.current_node_version) > 0
              updates << { :desired_node_version => @config['kubernetes']['nodeversion'] }
            end
          end

          if updates.size > 0
            updates.each { |mapping|
              requestobj = MU::Cloud::Google.container(:UpdateClusterRequest).new(
                :name => @cloud_id,
                :update => MU::Cloud::Google.container(:ClusterUpdate).new(
                  mapping
                )
              )
              MU.log "Updating GKE Cluster #{@mu_name.downcase}", MU::NOTICE, details: mapping
              begin
                MU::Cloud::Google.container(credentials: @config['credentials']).update_project_location_cluster(
                  @cloud_id,
                  requestobj
                )
              rescue ::Google::Apis::ClientError => e
                MU.log e.message, MU::WARN
              end
            }
            me = cloud_desc(use_cache: false)
          end

          if @config['preferred_maintenance_window'] and
             (!me.maintenance_policy.window or
              !me.maintenance_policy.window.daily_maintenance_window or
              me.maintenance_policy.window.daily_maintenance_window.start_time != @config['preferred_maintenance_window'])
            MU.log "Setting GKE Cluster #{@mu_name.downcase} maintenance time to #{@config['preferred_maintenance_window']}", MU::NOTICE
            MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_maintenance_policy(
              @cloud_id,
              MU::Cloud::Google.container(:SetMaintenancePolicyRequest).new(
                maintenance_policy: MU::Cloud::Google.container(:MaintenancePolicy).new(
                  window: MU::Cloud::Google.container(:MaintenanceWindow).new(
                    daily_maintenance_window: MU::Cloud::Google.container(:DailyMaintenanceWindow).new(
                      start_time: @config['preferred_maintenance_window']
                    )
                  )
                )
              )
            )
          elsif !@config['preferred_maintenance_window'] and me.maintenance_policy.window
            MU.log "Unsetting GKE Cluster #{@mu_name.downcase} maintenance time to #{@config['preferred_maintenance_window']}", MU::NOTICE
            MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_maintenance_policy(
              @cloud_id,
              nil
            )
          end


          kube_conf = writeKubeConfig

          if @config['kubernetes_resources']
            MU::Master.applyKubernetesResources(
              @config['name'], 
              @config['kubernetes_resources'],
              kubeconfig: kube_conf,
              outputdir: @deploy.deploy_dir
            )
          end

          MU.log %Q{How to interact with your GKE cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
        end


        # Locate an existing ContainerCluster or ContainerClusters and return an array containing matching GCP resource descriptors for those that match.
        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ContainerClusters
        def self.find(**args)
          args = MU::Cloud::Google.findLocationArgs(args)

          found = {}

          if args[:cloud_id]
            resp = begin
              MU::Cloud::Google.container(credentials: args[:credentials]).get_project_location_cluster(args[:cloud_id])
            rescue ::Google::Apis::ClientError => e
              raise e if !e.message.match(/forbidden:/)
            end
            found[args[:cloud_id]] = resp if resp
          else
            resp = begin
              MU::Cloud::Google.container(credentials: args[:credentials]).list_project_location_clusters("projects/#{args[:project]}/locations/#{args[:location]}")
            rescue ::Google::Apis::ClientError => e
              raise e if !e.message.match(/forbidden:/)
            end
            if resp and resp.clusters and !resp.clusters.empty?
              resp.clusters.each { |c|
                found[c.self_link.sub(/.*?\/projects\//, 'projects/')] = c
              }
            end
          end

          found
        end

        # Reverse-map our cloud description into a runnable config hash.
        # We assume that any values we have in +@config+ are placeholders, and
        # calculate our own accordingly based on what's live in the cloud.
        def toKitten(**_args)

          bok = {
            "cloud" => "Google",
            "project" => @config['project'],
            "credentials" => @config['credentials'],
            "cloud_id" => @cloud_id,
            "name" => cloud_desc.name.dup
          }

          bok['region'] = cloud_desc.location.sub(/\-[a-z]$/, "")
          if cloud_desc.locations.size == 1
            bok['availability_zone'] = cloud_desc.locations.first
          end
          bok["instance_count"] = cloud_desc.current_node_count
          cloud_desc.network_config.network.match(/^projects\/(.*?)\/.*?\/networks\/([^\/]+)(?:$|\/)/)
          vpc_proj = Regexp.last_match[1]
          vpc_id = Regexp.last_match[2]

          bok['vpc'] = MU::Config::Ref.get(
            id: vpc_id,
            cloud: "Google",
            habitat: MU::Config::Ref.get(
              id: vpc_proj,
              cloud: "Google",
              credentials: @credentials,
              type: "habitats"
            ),
            credentials: @config['credentials'],
            type: "vpcs"
          )


          bok['kubernetes'] = {
            "version" => cloud_desc.current_master_version,
            "nodeversion" => cloud_desc.current_node_version
          }
          if cloud_desc.default_max_pods_constraint and
             cloud_desc.default_max_pods_constraint.max_pods_per_node
            bok['kubernetes']['max_pods'] = cloud_desc.default_max_pods_constraint.max_pods_per_node
          end

          if cloud_desc.addons_config.horizontal_pod_autoscaling and
             cloud_desc.addons_config.horizontal_pod_autoscaling.disabled
            bok['kubernetes']['horizontal_pod_autoscaling'] = false
          end
          if cloud_desc.addons_config.http_load_balancing and
             cloud_desc.addons_config.http_load_balancing.disabled
            bok['kubernetes']['http_load_balancing'] = false
          end
          if !cloud_desc.addons_config.kubernetes_dashboard or
             !cloud_desc.addons_config.kubernetes_dashboard.disabled
            bok['kubernetes']['dashboard'] = true
          end
          if !cloud_desc.addons_config.network_policy_config or
             !cloud_desc.addons_config.network_policy_config.disabled
            bok['kubernetes']['network_policy_addon'] = true
          end

          if cloud_desc.ip_allocation_policy
            if cloud_desc.ip_allocation_policy.use_ip_aliases
              bok['ip_aliases'] = true
            end
            if cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
              bok['pod_ip_block'] = cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
            end
            if cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
              bok['services_ip_block'] = cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
            end

            if cloud_desc.ip_allocation_policy.create_subnetwork
              bok['custom_subnet'] = {
                "name" => (cloud_desc.ip_allocation_policy.subnetwork_name || cloud_desc.subnetwork)
              }
              if cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
                bok['custom_subnet']['node_ip_block'] = cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
              end
            end
          end

          bok['log_facility'] = if cloud_desc.logging_service == "logging.googleapis.com"
            "basic"
          elsif cloud_desc.logging_service == "logging.googleapis.com/kubernetes"
            "kubernetes"
          else
            "none"
          end

          if cloud_desc.master_auth and cloud_desc.master_auth.username
            bok['master_user'] = cloud_desc.master_auth.username
          end

          if cloud_desc.maintenance_policy and
             cloud_desc.maintenance_policy.window and
             cloud_desc.maintenance_policy.window.daily_maintenance_window and
             cloud_desc.maintenance_policy.window.daily_maintenance_window.start_time
            bok['preferred_maintenance_window'] = cloud_desc.maintenance_policy.window.daily_maintenance_window.start_time
          end

          if cloud_desc.enable_tpu
            bok['tpu'] = true
          end
          if cloud_desc.enable_kubernetes_alpha
            bok['kubernetes'] ||= {}
            bok['kubernetes']['alpha'] = true
          end

          if cloud_desc.node_pools and cloud_desc.node_pools.size > 0
            pool = cloud_desc.node_pools.first # we don't really support multiples atm
            bok["instance_type"] = pool.config.machine_type
            bok["instance_count"] = pool.initial_node_count
            bok['scopes'] = pool.config.oauth_scopes
            if pool.config.metadata
              bok["metadata"] = pool.config.metadata.keys.map { |k|
                { "key" => k, "value" => pool.config.metadata[k] }
              }
            end
            if pool.autoscaling and pool.autoscaling.enabled
              bok['max_size'] = pool.autoscaling.max_node_count
              bok['min_size'] = pool.autoscaling.min_node_count
            end
            bok['auto_repair'] = false
            bok['auto_upgrade'] = false
            if pool.management
              bok['auto_repair'] = true if pool.management.auto_repair
              bok['auto_upgrade'] = true if pool.management.auto_upgrade
            end
            [:local_ssd_count, :min_cpu_platform, :image_type, :disk_size_gb, :preemptible, :service_account].each { |field|
              if pool.config.respond_to?(field)
                bok[field.to_s] = pool.config.method(field).call
                bok.delete(field.to_s) if bok[field.to_s].nil?
              end
            }
          else
            bok["instance_type"] = cloud_desc.node_config.machine_type
            bok['scopes'] = cloud_desc.node_config.oauth_scopes
            if cloud_desc.node_config.metadata
              bok["metadata"] = cloud_desc.node_config.metadata.keys.map { |k|
                { "key" => k, "value" => pool.config.metadata[k] }
              }
            end
            [:local_ssd_count, :min_cpu_platform, :image_type, :disk_size_gb, :preemptible, :service_account].each { |field|
              if cloud_desc.node_config.respond_to?(field)
                bok[field.to_s] = cloud_desc.node_config.method(field).call
                bok.delete(field.to_s) if bok[field.to_s].nil?
              end
            }
          end

          if bok['service_account']
            found = MU::Cloud::Google::User.find(
              credentials: bok['credentials'],
              project: bok['project'],
              cloud_id: bok['service_account']
            )
            if found and found.size == 1
              sa = found.values.first
              # Ignore generic Mu service accounts
              if cloud_desc.resource_labels and
                 cloud_desc.resource_labels["mu-id"] and 
                 sa.description and
                 cloud_desc.resource_labels["mu-id"].downcase == sa.description.downcase
                bok.delete("service_account")
              else
                bok['service_account'] = MU::Config::Ref.get(
                  id: found.values.first.name,
                  cloud: "Google",
                  credentials: @config['credentials'],
                  type: "users"
                )
              end
            else
              bok.delete("service_account")
            end
          end

          if cloud_desc.private_cluster_config
            if cloud_desc.private_cluster_config.enable_private_nodes?
              bok["private_cluster"] ||= {}
              bok["private_cluster"]["private_nodes"] = true
            end
            if cloud_desc.private_cluster_config.enable_private_endpoint?
              bok["private_cluster"] ||= {}
              bok["private_cluster"]["private_master"] = true
            end
            if cloud_desc.private_cluster_config.master_ipv4_cidr_block
              bok["private_cluster"] ||= {}
              bok["private_cluster"]["master_ip_block"] = cloud_desc.private_cluster_config.master_ipv4_cidr_block
            end
          end

          if cloud_desc.master_authorized_networks_config and
             cloud_desc.master_authorized_networks_config.cidr_blocks and
             cloud_desc.master_authorized_networks_config.cidr_blocks.size > 0
            bok['authorized_networks'] = []
            cloud_desc.master_authorized_networks_config.cidr_blocks.each { |c|
              bok['authorized_networks'] << {
                "ip_block" => c.cidr_block,
                "label" => c.display_name
              }
            }
          end

          bok
        end


        # Register a description of this cluster instance with this deployment's metadata.
        def notify
          resp = MU::Cloud::Google.container(credentials: @config['credentials']).get_project_location_cluster(@cloud_id)
          desc = MU.structToHash(resp)
          desc["project"] = @config['project']
          desc["cloud_id"] = @cloud_id
          desc["project_id"] = @project_id
          desc["mu_name"] = @mu_name.downcase
          desc
        end

        # Does this resource type exist as a global (cloud-wide) artifact, or
        # is it localized to a region/zone?
        # @return [Boolean]
        def self.isGlobal?
          false
        end

        # Denote whether this resource implementation is experiment, ready for
        # testing, or ready for production use.
        def self.quality
          MU::Cloud::RELEASE
        end

        # Called by {MU::Cleanup}. Locates resources that were created by the
        # currently-loaded deployment, and purges them.
        # @param noop [Boolean]: If true, will only print what would be done
        # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
        # @param region [String]: The cloud provider region in which to operate
        # @return [void]
        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})

          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
          clusters = []

          # Make sure we catch regional *and* zone clusters
          found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{region}")
          clusters.concat(found.clusters) if found and found.clusters
          MU::Cloud::Google.listAZs(region).each { |az|
            found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{az}")
            clusters.concat(found.clusters) if found and found.clusters
          }

          clusters.uniq.each { |cluster|
            if !cluster.resource_labels or (
                 !cluster.name.match(/^#{Regexp.quote(MU.deploy_id)}\-/i) and
                 (cluster.resource_labels['mu-id'] != MU.deploy_id.downcase or
                  (!ignoremaster and cluster.resource_labels['mu-master-ip'] != MU.mu_public_ip.gsub(/\./, "_"))
                 )
               )
              next
            end
            MU.log "Deleting GKE cluster #{cluster.name}"
            if !noop
              cloud_id = cluster.self_link.sub(/.*?\/projects\//, 'projects/')
              retries = 0
              begin
                MU::Cloud::Google.container(credentials: credentials).delete_project_location_cluster(cloud_id)
                MU::Cloud::Google.container(credentials: credentials).get_project_location_cluster(cloud_id)
                sleep 60
              rescue ::Google::Apis::ClientError => e
                if e.message.match(/notFound: /)
                  MU.log cloud_id, MU::WARN, details: e.inspect
                  break
                elsif e.message.match(/failedPrecondition: /)
                  if (retries % 5) == 0
                    MU.log "Waiting to delete GKE cluster #{cluster.name}: #{e.message}", MU::NOTICE
                  end
                  sleep 60
                  retries += 1
                  retry
                else
                  MU.log cloud_id, MU::WARN, details: e.inspect
                  raise e
                end
              end while true
            end
          }

        end

        # Cloud-specific configuration properties.
        # @param config [MU::Config]: The calling MU::Config object
        # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
        def self.schema(config)
          toplevel_required = []
          gke_defaults = defaults
          schema = {
            "auto_upgrade" => {
              "type" => "boolean",
              "description" => "Automatically upgrade worker nodes during maintenance windows",
              "default" => true
            },
            "auto_repair" => {
              "type" => "boolean",
              "description" => "Automatically replace worker nodes which fail health checks",
              "default" => true
            },
            "local_ssd_count" => {
              "type" => "integer",
              "description" => "The number of local SSD disks to be attached to workers. See https://cloud.google.com/compute/docs/disks/local-ssd#local_ssd_limits"
            },
            "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
            "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
            "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
            "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
            "private_cluster" => {
              "description" => "Set a GKE cluster to be private, that is segregated into its own hidden VPC.",
              "type" => "object",
              "properties" => {
                "private_nodes" => {
                  "type" => "boolean",
                  "default" => true,
                  "description" => "Whether GKE worker nodes have internal IP addresses only."
                },
                "private_master" => {
                  "type" => "boolean",
                  "default" => false,
                  "description" => "Whether the GKE Kubernetes master's internal IP address is used as the cluster endpoint."
                },
                "master_ip_block" => {
                  "type" => "string",
                  "pattern" => MU::Config::CIDR_PATTERN,
                  "default" => "172.20.0.0/28",
                  "description" => "The private IP address range to use for the GKE master's network"
                }
              }
            },
            "custom_subnet" => {
              "type" => "object",
              "description" => "If set, GKE will create a new subnetwork specifically for this cluster",
              "properties" => {
                "name" => {
                  "type" => "string",
                  "description" => "Set a custom name for the generated subnet"
                },
                "node_ip_block" => {
                  "type" => "string",
                  "pattern" => MU::Config::CIDR_PATTERN,
                  "description" => "The IP address range of the worker nodes in this cluster, in CIDR notation"
                }
              }
            },
            "pod_ip_block" => {
              "type" => "string",
              "pattern" => MU::Config::CIDR_PATTERN,
              "description" => "The IP address range of the container pods in this cluster, in CIDR notation"
            },
            "pod_ip_block_name" => {
              "type" => "string",
              "description" => "The name of the secondary range to be used for the pod CIDR block"
            },
            "services_ip_block" => {
              "type" => "string",
              "pattern" => MU::Config::CIDR_PATTERN,
              "description" => "The IP address range of the services in this cluster, in CIDR notation"
            },
            "services_ip_block_name" => {
              "type" => "string",
              "description" => "The name of the secondary range to be used for the services CIDR block"
            },
            "ip_aliases" => {
              "type" => "boolean",
              "description" => "Whether alias IPs will be used for pod IPs in the cluster. Will be automatically enabled for functionality, such as +private_cluster+, which requires it."
            },
            "tpu_ip_block" => {
              "type" => "string",
              "pattern" => MU::Config::CIDR_PATTERN,
              "description" => "The IP address range of any Cloud TPUs in this cluster, in CIDR notation"
            },
            "disk_size_gb" => {
              "type" => "integer",
              "description" => "Size of the disk attached to each worker, specified in GB. The smallest allowed disk size is 10GB",
              "default" => 100
            },
            "min_size" => {
              "description" => "In GKE, this is the minimum number of nodes *per availability zone*, when scaling is enabled. Setting +min_size+ and +max_size+ enables scaling."
            },
            "max_size" => {
              "description" => "In GKE, this is the maximum number of nodes *per availability zone*, when scaling is enabled. Setting +min_size+ and +max_size+ enables scaling."
            },
            "instance_count" => {
              "description" => "In GKE, this value is ignored if +min_size+ and +max_size+ are set."
            },
            "min_cpu_platform" => {
              "type" => "string",
              "description" => "Minimum CPU platform to be used by workers. The instances may be scheduled on the specified or newer CPU platform. Applicable values are the friendly names of CPU platforms, such as minCpuPlatform: 'Intel Haswell' or minCpuPlatform: 'Intel Sandy Bridge'."
            },
            "preemptible" => {
              "type" => "boolean",
              "default" => false,
              "description" => "Whether the workers are created as preemptible VM instances. See: https://cloud.google.com/compute/docs/instances/preemptible for more information about preemptible VM instances."
            },
            "image_type" => {
              "type" => "string",
              "enum" => gke_defaults ? gke_defaults.valid_image_types : ["COS"],
              "description" => "The image type to use for workers. Note that for a given image type, the latest version of it will be used.",
              "default" => gke_defaults ? gke_defaults.default_image_type : "COS"
            },
            "availability_zone" => {
              "type" => "string",
              "description" => "Target a specific availability zone for this cluster"
            },
            "preferred_maintenance_window" => {
              "type" => "string",
              "description" => "The preferred daily time to perform node maintenance. Time format should be in [RFC3339](http://www.ietf.org/rfc/rfc3339.txt) format +HH:MM+ GMT.",
              "pattern" => '^\d\d:\d\d$'
            },
            "kubernetes" => {
              "description" => "Kubernetes-specific options",
              "properties" => {
                "version" => {
                  "type" => "string"
                },
                "nodeversion" => {
                  "type" => "string",
                  "description" => "The version of Kubernetes to install on GKE worker nodes."
                },
                "alpha" => {
                  "type" => "boolean",
                  "default" => false,
                  "description" => "Enable alpha-quality Kubernetes features on this cluster"
                },
                "dashboard" => {
                  "type" => "boolean",
                  "default" => false,
                  "description" => "Enable the Kubernetes Dashboard"
                },
                "horizontal_pod_autoscaling" => {
                  "type" => "boolean",
                  "default" => true,
                  "description" => "Increases or decreases the number of replica pods a replication controller has based on the resource usage of the existing pods."
                },
                "http_load_balancing" => {
                  "type" => "boolean",
                  "default" => true,
                  "description" => "HTTP (L7) load balancing controller addon, which makes it easy to set up HTTP load balancers for services in a cluster."
                },
                "network_policy_addon" => {
                  "type" => "boolean",
                  "default" => false,
                  "description" => "Enable the Network Policy addon"
                }
              }
            },
            "pod_ip_range" => {
              "type" => "string",
              "pattern" => MU::Config::CIDR_PATTERN,
              "description" => "The IP address range of the container pods in this cluster, in CIDR notation"
            },
            "tpu" => {
              "type" => "boolean",
              "default" => false,
              "description" => "Enable the ability to use Cloud TPUs in this cluster."
            },
            "log_facility" => {
              "type" => "string",
              "default" => "kubernetes",
              "description" => "The +logging.googleapis.com+ and +monitoring.googleapis.com+ facilities that this cluster should use to write logs and metrics.",
              "enum" => ["basic", "kubernetes", "none"]
            },
            "master_user" => {
              "type" => "string",
              "description" => "Enables Basic Auth for a GKE cluster with string as the master username"
            },
            "authorized_networks" => {
              "type" => "array",
              "items" => {
                "description" => "GKE's Master authorized networks functionality",
                "type" => "object",
                "ip_block" => {
                  "type" => "string",
                  "description" => "CIDR block to allow",
                  "pattern" => MU::Config::CIDR_PATTERN,
                },
                "label" =>{
                  "description" => "Label for this CIDR block",
                  "type" => "string",
                }
              }
            },
            "master_az" => {
              "type" => "string",
              "description" => "Target a specific Availability Zone for the GKE master. If not set, we will choose one which has the most current versions of Kubernetes available."
            }
          }
          [toplevel_required, schema]
        end

        # Cloud-specific pre-processing of {MU::Config::BasketofKittens::container_clusters}, bare and unvalidated.
        # @param cluster [Hash]: The resource to process and validate
        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
        # @return [Boolean]: True if validation succeeded, False otherwise
        def self.validateConfig(cluster, configurator)
          ok = true
          cluster['project'] ||= MU::Cloud::Google.defaultProject(cluster['credentials'])

          cluster['master_az'] ||= cluster['availability_zone'] if cluster['availability_zone']

          if cluster['private_cluster'] or cluster['custom_subnet'] or
             cluster['services_ip_block'] or cluster['services_ip_block_name'] or
             cluster['pod_ip_block'] or cluster['pod_ip_block_name'] or
             cluster['tpu_ip_block']
            cluster['ip_aliases'] = true
          end

          if cluster['service_account']
            cluster['service_account']['cloud'] = "Google"
            cluster['service_account']['habitat'] ||= MU::Config::Ref.get(
              id: cluster['project'],
              cloud: "Google",
              credentials: cluster['credentials'],
              type: "habitats"
            )
            if cluster['service_account']['name'] and
               !cluster['service_account']['id'] and
               !cluster['service_account']['deploy_id']
              cluster['dependencies'] ||= []
              cluster['dependencies'] << {
                "type" => "user",
                "name" => cluster['service_account']['name']
              }
            end
            found = MU::Config::Ref.get(cluster['service_account'])
            # XXX verify that found.kitten fails when it's supposed to
            if cluster['service_account']['id'] and !found.kitten
              MU.log "GKE cluster #{cluster['name']} failed to locate service account #{cluster['service_account']} in project #{cluster['project']}", MU::ERR
              ok = false
            end
          else
            cluster = MU::Cloud::Google::User.genericServiceAccount(cluster, configurator)
          end

          if cluster['dependencies']
            cluster['dependencies'].each { |dep|
              if dep['type'] == "vpc"
                dep['phase'] = "groom"
              end
            }
          end

          if (cluster['pod_ip_block_name'] or cluster['services_ip_block_name']) and
             cluster['custom_subnet']
            MU.log "GKE cluster #{cluster['name']} cannot specify pod_ip_block_name or services_ip_block_name when using a custom subnet", MU::ERR
            ok = false
          end

          # If we've enabled master authorized networks, make sure our Mu
          # Master is one of the things allowed in.
          if cluster['authorized_networks']
            found_me = false
            my_cidr = NetAddr::IPv4.parse(MU.mu_public_ip)
            cluster['authorized_networks'].each { |block|
              cidr_obj = NetAddr::IPv4Net.parse(block['ip_block'])
              if cidr_obj.contains(my_cidr)
                found_me = true
                break
              end
            }
            if !found_me
              cluster['authorized_networks'] << {
                "ip_block" => MU.mu_public_ip+"/32",
                "label" => "Mu Master #{$MU_CFG['hostname']}"
              }
            end
          end

          master_versions = defaults(az: cluster['master_az']).valid_master_versions.sort { |a, b| MU.version_sort(a, b) }
          if cluster['kubernetes'] and cluster['kubernetes']['version']
            if cluster['kubernetes']['version'] == "latest"
              cluster['kubernetes']['version'] = master_versions.last
            elsif !master_versions.include?(cluster['kubernetes']['version'])
              match = false
              master_versions.each { |v|
                if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
                  match = true
                  break
                end
              }
              if !match
                MU.log "No version matching #{cluster['kubernetes']['version']} available, will try floating minor revision", MU::WARN
                cluster['kubernetes']['version'].sub!(/^(\d+\.\d+\.).*/i, '\1')
                master_versions.each { |v|
                  if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
                    match = true
                    break
                  end
                }
                if !match
                  MU.log "Failed to find a GKE master version matching #{cluster['kubernetes']['version']} among available versions in #{cluster['master_az'] || cluster['region']}.", MU::ERR, details: master_versions
                  ok = false
                end
              end
            end
          end

          node_versions = defaults(az: cluster['master_az']).valid_node_versions.sort { |a, b| MU.version_sort(a, b) }

          if cluster['kubernetes'] and cluster['kubernetes']['nodeversion']
            if cluster['kubernetes']['nodeversion'] == "latest"
              cluster['kubernetes']['nodeversion'] = node_versions.last
            elsif !node_versions.include?(cluster['kubernetes']['nodeversion'])
              match = false
              node_versions.each { |v|
                if v.match(/^#{Regexp.quote(cluster['kubernetes']['nodeversion'])}/)
                  match = true
                  break
                end
              }
              if !match
                MU.log "No version matching #{cluster['kubernetes']['nodeversion']} available, will try floating minor revision", MU::WARN
                cluster['kubernetes']['nodeversion'].sub!(/^(\d+\.\d+\.).*/i, '\1')
                node_versions.each { |v|
                  if v.match(/^#{Regexp.quote(cluster['kubernetes']['nodeversion'])}/)
                    match = true
                    break
                  end
                }
                if !match
                  MU.log "Failed to find a GKE node version matching #{cluster['kubernetes']['nodeversion']} among available versions in #{cluster['master_az'] || cluster['region']}.", MU::ERR, details: node_versions
                  ok = false
                end
              end
            end
          end

          cluster['instance_type'] = MU::Cloud::Google::Server.validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
          ok = false if cluster['instance_type'].nil?

          ok
        end

        private

        def node_desc
          labels = Hash[@tags.keys.map { |k|
            [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
          ]
          labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
          desc = {
            :machine_type => @config['instance_type'],
            :preemptible => @config['preemptible'],
            :disk_size_gb => @config['disk_size_gb'],
            :labels => labels,
            :tags => [@mu_name.downcase],
            :service_account => @service_acct.email,
            :oauth_scopes => @config['scopes']
          }
          desc[:metadata] = {}
          deploykey = @config['ssh_user']+":"+@deploy.ssh_public_key
          if @config['metadata']
            desc[:metadata] = Hash[@config['metadata'].map { |m|
              [m["key"], m["value"]]
            }]
          end
          if desc[:metadata]["ssh-keys"]
            desc[:metadata]["ssh-keys"] += "\n"+deploykey
          else
            desc[:metadata]["ssh-keys"] = deploykey
          end
          [:local_ssd_count, :min_cpu_platform, :image_type].each { |field|
            if @config[field.to_s]
              desc[field] = @config[field.to_s]
            end
          }
          desc
        end

        def labelCluster
          labels = Hash[@tags.keys.map { |k|
            [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
          ]
          labels["name"] = MU::Cloud::Google.nameStr(@mu_name)

          labelset = MU::Cloud::Google.container(:SetLabelsRequest).new(
            resource_labels: labels,
            label_fingerprint: cloud_desc.label_fingerprint
          )
          MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_resource_labels(@cloud_id, labelset)
        end

        @@server_config = {}
        def self.defaults(credentials = nil, az: nil)
          az ||= MU::Cloud::Google.listAZs.sample
          return nil if az.nil?
          @@server_config[credentials] ||= {}
          if @@server_config[credentials][az]
            return @@server_config[credentials][az]
          end

          parent_arg = "projects/"+MU::Cloud::Google.defaultProject(credentials)+"/locations/"+az

          @@server_config[credentials][az] = MU::Cloud::Google.container(credentials: credentials).get_project_location_server_config(parent_arg)
          @@server_config[credentials][az]
        end
        private_class_method :defaults

        def writeKubeConfig
          kube_conf = @deploy.deploy_dir+"/kubeconfig-#{@config['name']}"
          client_binding = @deploy.deploy_dir+"/k8s-client-user-admin-binding.yaml"
          @endpoint = "https://"+cloud_desc.endpoint
          @cacert = cloud_desc.master_auth.cluster_ca_certificate
          @cluster = cloud_desc.name
          @clientcert = cloud_desc.master_auth.client_certificate
          @clientkey = cloud_desc.master_auth.client_key
          if cloud_desc.master_auth.username
            @username = cloud_desc.master_auth.username
          end
          if cloud_desc.master_auth.password
            @password = cloud_desc.master_auth.password
          end

          kube = ERB.new(File.read(MU.myRoot+"/cookbooks/mu-tools/templates/default/kubeconfig-gke.erb"))
          File.open(kube_conf, "w"){ |k|
            k.puts kube.result(binding)
          }

          # Take this opportunity to ensure that the 'client' service account
          # used by certificate authentication exists and has appropriate
          # privilege
          if @username and @password
            File.open(client_binding, "w"){ |k|
              k.puts <<-EOF
kind: ClusterRoleBinding 
apiVersion: rbac.authorization.k8s.io/v1
metadata: 
  name: client-binding
  namespace: kube-system
roleRef: 
  kind: ClusterRole 
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
subjects: 
- kind: User
  name: client
  namespace: kube-system
              EOF
            }
            bind_cmd = %Q{#{MU::Master.kubectl} create serviceaccount client --namespace=kube-system --kubeconfig "#{kube_conf}" ; #{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f #{client_binding}}
            MU.log bind_cmd
            system(bind_cmd)
          end
          # unset the variables we set just for ERB
          [:@endpoint, :@cacert, :@cluster, :@clientcert, :@clientkey, :@username, :@password].each { |var|
            begin
              remove_instance_variable(var)
            rescue NameError
            end
          }

          kube_conf
        end

      end #class
    end #class
  end
end #module