google_server_create.rb in knife-google-1.0.0

- old
+ new
@@ -1,261 +1,350 @@
-# Author:: Chirag Jog (<chiragj@websym.com>)
-# Copyright:: Copyright (c) 2012 Opscode, Inc.
-# License:: Apache License, Version 2.0
+# Copyright 2013 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-require 'highline'
-require 'net/ssh/multi'
-require 'net/scp'
-require 'tempfile'
-
-require 'chef/knife'
+#
 require 'chef/knife/google_base'
 
 class Chef
   class Knife
     class GoogleServerCreate < Knife
 
+      include Knife::GoogleBase
+
       deps do
-        require 'readline'
+        require 'google/compute'
         require 'chef/json_compat'
         require 'chef/knife/bootstrap'
         Chef::Knife::Bootstrap.load_deps
       end
 
-      include Knife::GoogleBase
+      banner "knife google server create NAME -m MACHINE_TYPE -I IMAGE -Z ZONE (options)"
 
-      banner "knife google server create NAME [RUN LIST...] (options)"
+      attr_accessor :initial_sleep_delay
+      attr_reader :instance
 
-      option :run_list,
-        :short => "-r RUN_LIST",
-        :long => "--run-list RUN_LIST",
-        :description => "Comma separated list of roles/recipes to apply",
-        :proc => lambda { |o| o.split(/[\s,]+/) },
-        :default => []
+      option :machine_type,
+        :short => "-m MACHINE_TYPE",
+        :long => "--google-compute-machine MACHINE_TYPE",
+        :description => "The machine type of server (n1-highcpu-2, n1-highcpu-2-d, etc)",
+        :required => true
 
-      option :availability_zone,
+      option :image,
+        :short => "-I IMAGE",
+        :long => "--google-compute-image IMAGE",
+        :description => "The Image for the server",
+        :required => true
+
+      option :zone,
         :short => "-Z ZONE",
-        :long => "--availability-zone ZONE",
-        :description => "The Availability Zone",
-        :default => "us-east-a",
-        :proc => Proc.new { |key| Chef::Config[:knife][:availability_zone] = key }
+        :long => "--google-compute-zone ZONE",
+        :description => "The Zone for this server",
+        :required => true
 
-      option :distro,
-        :short => "-d DISTRO",
-        :long => "--distro DISTRO",
-        :description => "Bootstrap a distro using a template; default is 'ubuntu10.04-gems'",
-        :proc => Proc.new { |d| Chef::Config[:knife][:distro] = d },
-        :default => "ubuntu10.04-gems"
+      option :network,
+        :short => "-n NETWORK",
+        :long => "--google-compute-network NETWORK",
+        :description => "The network for this server; default is 'default'",
+        :default => "default"
 
-      option :template_file,
-        :long => "--template-file TEMPLATE",
-        :description => "Full path to location of template to use",
-        :proc => Proc.new { |t| Chef::Config[:knife][:template_file] = t },
-        :default => false
+      option :tags,
+        :short => "-T TAG1,TAG2,TAG3",
+        :long => "--google-compute-tags TAG1,TAG2,TAG3",
+        :description => "Tags for this server",
+        :proc => Proc.new { |tags| tags.split(',') },
+        :default => []
 
+      option :metadata,
+        :short => "-M K=V[,K=V,...]",
+        :long => "--google-compute-metadata Key=Value[,Key=Value...]",
+        :description => "The metadata for this server",
+        :proc => Proc.new { |metadata| metadata.split(',') },
+        :default => []
+
       option :chef_node_name,
         :short => "-N NAME",
         :long => "--node-name NAME",
-        :description => "The Chef node name for your new node",
-        :proc => Proc.new { |t| Chef::Config[:knife][:chef_node_name] = t }
+        :description => "The Chef node name for your new node"
 
       option :ssh_user,
         :short => "-x USERNAME",
         :long => "--ssh-user USERNAME",
-        :description => "The ssh username; default is 'ubuntu'",
-        :default => "ubuntu"
+        :description => "The ssh username; default is 'root'",
+        :default => "root"
 
-      option :server_name,
-        :short => "-N NAME",
-        :long => "--server-name NAME",
-        :description => "The server name",
-        :proc => Proc.new { |server_name| Chef::Config[:knife][:server_name] = server_name } 
+      option :ssh_password,
+        :short => "-P PASSWORD",
+        :long => "--ssh-password PASSWORD",
+        :description => "The ssh password"
 
-      option :flavor,
-        :short => "-f FLAVOR",
-        :long => "--flavor FLAVOR",
-        :description => "The flavor of server (standard-1-cpu,standard-2-cpu-ephemeral-disk, etc)",
-        :proc => Proc.new { |f| Chef::Config[:knife][:flavor] = f },
-        :default => "standard-1-cpu"
+      option :ssh_port,
+        :short => "-p PORT",
+        :long => "--ssh-port PORT",
+        :description => "The ssh port; default is '22'",
+        :default => "22"
 
-      option :image,
-        :short => "-I IMAGE",
-        :long => "--google-image IMAGE",
-        :description => "Your google virtual app template/image name",
-        :proc => Proc.new { |template| Chef::Config[:knife][:image] = template },
-        :default => "gcompute8-standard"
-        
-      option :private_key_file,
-        :short => "-i PRIVATE_KEY_FILE",
-        :long => "--private-key-file PRIVATE_KEY_FILE",
-        :description => "The SSH private key file used for authentication",
-        :proc => Proc.new { |identity| Chef::Config[:knife][:private_key_file] = identity } 
- 
-      option :public_key_file,
-        :short => "-k PUBLIC_KEY_FILE",
-        :long => "--public-key-file PUBLIC_KEY_FILE",
-        :description => "The SSH public key file used for authentication",
-        :proc => Proc.new { |identity| Chef::Config[:knife][:public_key_file] = identity } 
- 
-      option :network,
-        :short => "-n NETWORKNAME",
-        :long => "--network NETWORKNAME",
-        :description => "The Network in which to create the Virtual machine",
-        :proc => Proc.new { |network| Chef::Config[:knife][:network] = network},
-        :default => "default"
+      option :ssh_gateway,
+        :short => "-w GATEWAY",
+        :long => "--ssh-gateway GATEWAY",
+        :description => "The ssh gateway server"
 
-      option :external_ip_address,
-        :short => "-e IPADDRESS",
-        :long => "--external-ip-address IPADDRESS",
-        :description => "A Static IP provided by Google",
-        :proc => Proc.new { |ipaddr| Chef::Config[:knife][:external_ip_address] = ipaddr},
-        :default => "ephemeral"
+      option :identity_file,
+        :short => "-i IDENTITY_FILE",
+        :long => "--identity-file IDENTITY_FILE",
+        :description => "The SSH identity file used for authentication"
 
-      option :internal_ip_address,
-        :short => "-P IPADDRESS",
-        :long => "--internal-ip-address IPADDRESS",
-        :description => "A Static IP provided by Google",
-        :proc => Proc.new { |ipaddr| Chef::Config[:knife][:internal_ip_address] = ipaddr}
+      option :prerelease,
+        :long => "--prerelease",
+        :description => "Install the pre-release chef gems"
 
-      option :project,
-        :short => "-p PROJECT",
-        :long => "--project_id PROJECT",
-        :description => "Google Compute Project",
-        :proc => Proc.new { |project| Chef::Config[:knife][:google_project] = project}
+      option :bootstrap_version,
+        :long => "--bootstrap-version VERSION",
+        :description => "The version of Chef to install"
 
-      def h
-        @highline ||= HighLine.new
-      end
-      
-      def locate_config_value(key)
-        key = key.to_sym
-        config[key] || Chef::Config[:knife][key]
-      end
+      option :distro,
+        :short => "-d DISTRO",
+        :long => "--distro DISTRO",
+        :description => "Bootstrap a distro using a template; default is 'chef-full'",
+        :default => 'chef-full'
 
-      def tcp_test_ssh(hostname, port)
-        tcp_socket = TCPSocket.new(hostname, port)
+      option :template_file,
+        :long => "--template-file TEMPLATE",
+        :description => "Full path to location of template to use",
+        :default => false
+
+      option :run_list,
+        :short => "-r RUN_LIST",
+        :long => "--run-list RUN_LIST",
+        :description => "Comma separated list of roles/recipes to apply",
+        :proc => lambda { |o| o.split(/[\s,]+/) }
+
+      option :json_attributes,
+        :short => "-j JSON",
+        :long => "--json-attributes JSON",
+        :description => "A JSON string to be added to the first run of chef-client",
+        :proc => lambda { |o| JSON.parse(o) }
+
+      option :host_key_verify,
+        :long => "--[no-]host-key-verify",
+        :description => "Verify host key, enabled by default.",
+        :boolean => true,
+        :default => true
+
+      option :compute_user_data,
+        :long => "--user-data USER_DATA_FILE",
+        :short => "-u USER_DATA_FILE",
+        :description => "The Google Compute User Data file to provision the server with"
+
+      option :hint,
+        :long => "--hint HINT_NAME[=HINT_FILE]",
+        :description => "Specify Ohai Hint to be set on the bootstrap target. Use multiple --hint options to specify multiple hints.",
+        :proc => Proc.new { |h|
+           Chef::Config[:knife][:hints] ||= {}
+           name, path = h.split("=")
+           Chef::Config[:knife][:hints][name] = path ? JSON.parse(::File.read(path)) : Hash.new
+        }
+
+      option :instance_connect_ip,
+        :long => "--google-compute-server-connect-ip PUBLIC",
+        :short => "-a PUBLIC",
+        :description => "Whether to use PUBLIC or PRIVATE address to connect; default is 'PUBLIC'",
+        :default => 'PUBLIC'
+
+      option :disks,
+        :long=> "--google-compute-disks DISK1,DISK2",
+        :proc => Proc.new { |metadata| metadata.split(',') },
+        :description => "Disks to be attached",
+        :default => []
+
+      option :public_ip,
+        :long=> "--google-compute-public-ip IP_ADDRESS",
+        :description => "EPHEMERAL or static IP address or NONE; default is 'EPHEMERAL'",
+        :default => "EPHEMERAL"
+
+      def tcp_test_ssh(hostname, ssh_port)
+        tcp_socket = TCPSocket.new(hostname, ssh_port)
         readable = IO.select([tcp_socket], nil, nil, 5)
         if readable
           Chef::Log.debug("sshd accepting connections on #{hostname}, banner is #{tcp_socket.gets}")
           yield
           true
         else
           false
         end
-      rescue Errno::ETIMEDOUT
-        false
-      rescue Errno::EPERM
-        false
-      rescue Errno::ECONNREFUSED
+      rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, IOError
         sleep 2
         false
-      rescue Errno::EHOSTUNREACH
-        sleep 2
+      rescue Errno::EPERM, Errno::ETIMEDOUT
         false
       ensure
         tcp_socket && tcp_socket.close
       end
 
-      def run
-        unless Chef::Config[:knife][:server_name]
-          ui.error("Server Name is a compulsory parameter")
-          exit 1
-        end
+      def wait_for_sshd(hostname)
+        config[:ssh_gateway] ? wait_for_tunnelled_sshd(hostname) : wait_for_direct_sshd(hostname, config[:ssh_port])
+      end
 
-        unless Chef::Config[:knife][:public_key_file]
-          ui.error("SSH public key file is a compulsory parameter")
-          exit 1
-        end
+      def wait_for_tunnelled_sshd(hostname)
+        print(".")
+        print(".") until tunnel_test_ssh(ssh_connect_host) {
+          sleep @initial_sleep_delay ||= 40
+          puts("done")
+        }
+      end
 
-        unless Chef::Config[:knife][:google_project]
-          ui.error("Project ID is a compulsory parameter")
-          exit 1
+      def tunnel_test_ssh(hostname, &block)
+        gw_host, gw_user = config[:ssh_gateway].split('@').reverse
+        gw_host, gw_port = gw_host.split(':')
+        gateway = Net::SSH::Gateway.new(gw_host, gw_user, :port => gw_port || 22)
+        status = false
+        gateway.open(hostname, config[:ssh_port]) do |local_tunnel_port|
+          status = tcp_test_ssh('localhost', local_tunnel_port, &block)
         end
+        status
+      rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, IOError
+        sleep 2
+        false
+      rescue Errno::EPERM, Errno::ETIMEDOUT
+        false
+      end
 
-        $stdout.sync = true
+      def wait_for_direct_sshd(hostname, ssh_port)
+        print(".") until tcp_test_ssh(ssh_connect_host, ssh_port) {
+          sleep @initial_sleep_delay ||=  40
+          puts("done")
+        }
+      end
 
-        project_id = Chef::Config[:knife][:google_project]
-        validate_project(project_id)
+      def ssh_connect_host
+        @ssh_connect_host ||= if config[:instance_connect_ip] == 'PUBLIC'
+                                public_ips(@instance).first
+        else
+           private_ips(@instance).first
+        end
+      end
 
-        server_name = Chef::Config[:knife][:server_name]
-        image = Chef::Config[:knife][:image]
-        key_file = locate_config_value(:public_key_file)
-        network = locate_config_value(:network)
-        flavor = locate_config_value(:flavor)
-        zone = locate_config_value(:availability_zone)
-        user = locate_config_value(:ssh_user)
-        external_ip_address = locate_config_value(:external_ip_address)
-        internal_ip_address = locate_config_value(:internal_ip_address) || nil
-        puts "\n#{ui.color("Waiting for the server to be Instantiated", :magenta)}"
-        cmd_add_instance = "#{@gcompute} addinstance #{server_name} --machine_type #{flavor} " +
-                             "--zone #{zone} --project_id #{project_id} --tags #{server_name} " +
-                             "--authorized_ssh_keys #{user}:#{key_file} --network #{network} " +
-                             "--external_ip_address #{external_ip_address} --print_json"
-        cmd_add_instance << " --internal_ip_address #{internal_ip_address}" if internal_ip_address 
+      def bootstrap_for_node(instance,ssh_host)
+        bootstrap = Chef::Knife::Bootstrap.new
+        bootstrap.name_args = [ssh_host]
+        bootstrap.config[:run_list] = config[:run_list]
+        bootstrap.config[:ssh_user] = config[:ssh_user]
+        bootstrap.config[:ssh_port] = config[:ssh_port]
+        bootstrap.config[:ssh_gateway] = config[:ssh_gateway]
+        bootstrap.config[:identity_file] = config[:identity_file]
+        bootstrap.config[:chef_node_name] = config[:chef_node_name] || instance.name
+        bootstrap.config[:prerelease] = config[:prerelease]
+        bootstrap.config[:bootstrap_version] = config[:bootstrap_version]
+        bootstrap.config[:first_boot_attributes] = config[:json_attributes]
+        bootstrap.config[:distro] = config[:distro]
+        bootstrap.config[:use_sudo] = true unless config[:ssh_user] == 'root'
+        bootstrap.config[:template_file] = config[:template_file]
+        bootstrap.config[:environment] = config[:environment]
+        # may be needed for vpc_mode
+        bootstrap.config[:host_key_verify] = config[:host_key_verify]
+        # Modify global configuration state to ensure hint gets set by
+        # knife-bootstrap
+        Chef::Config[:knife][:hints] ||= {}
+        Chef::Config[:knife][:hints]["google"] ||= {}
+        bootstrap
+      end
 
-        Chef::Log.debug 'Executing ' +  cmd_add_instance
-        create_server = exec_shell_cmd(cmd_add_instance)
+      def run
+        $stdout.sync = true
+        unless @name_args.size > 0
+          ui.error("Please provide the name of the new server")
+          exit 1
+        end
 
-        if create_server.stderr.downcase.scan("error").size > 0
-          ui.error("\nFailed to create server: #{create_server.stderr}")
+        begin
+          zone = client.zones.get(config[:zone]).self_link
+        rescue Google::Compute::ResourceNotFound
+          ui.error("Zone '#{config[:zone]}' not found")
           exit 1
         end
-        if create_server.stdout.downcase.scan("error").size > 0
-          output = to_json(create_server.stdout)
-          ui.error("\nFailed to create server: #{output["error"]}")
+        begin
+          machine_type = client.machine_types.get(config[:machine_type]).self_link
+        rescue Google::Compute::ResourceNotFound
+          ui.error("MachineType '#{config[:machine_type]}' not found")
           exit 1
         end
-                    
-        #Fetch server information
-        cmd_get_instance  = "#{@gcompute} getinstance #{server_name} --project_id #{project_id} --print_json "
-        Chef::Log.debug 'Executing ' +  cmd_get_instance
-        get_instance = exec_shell_cmd(cmd_get_instance)
+        begin
+          image = client.images.get(:project=>'google', :name=>config[:image]).self_link
+        rescue Google::Compute::ResourceNotFound
+          ui.error("Image '#{config[:image]}' not found")
+          exit 1
+        end
+        begin
+          network = client.networks.get(config[:network]).self_link
+        rescue Google::Compute::ResourceNotFound
+          ui.error("Network '#{config[:network]}' not found")
+          exit 1
+        end
 
-        if not get_instance.stderr.downcase.scan("error").empty?
-          ui.error("Failed to fetch server details.")
+        disks = config[:disks].collect{|disk| client.disks.get(:disk=>disk, :zone=>selflink2name(zone)).self_link}
+        metadata = config[:metadata].collect{|pair| Hash[*pair.split('=')] }
+        network_interface = {'network'=>network}
+
+        if config[:public_ip] == 'EPHEMERAL'
+          network_interface.merge!('accessConfigs' =>[{"name"=>"External NAT",
+                                  "type"=> "ONE_TO_ONE_NAT"}])
+        elsif config[:public_ip] =~ /\d+\.\d+\.\d+\.\d+/
+          network_interface.merge!('accessConfigs' =>[{"name"=>"External NAT",
+                  "type"=>"ONE_TO_ONE_NAT", "natIP"=>config[:public_ip] }])
+        elsif config[:public_ip] == 'NONE'
+          # do nothing
+        else
+          ui.error("Invalid public ip value : #{config[:public_ip]}")
           exit 1
         end
+        zone_operation = client.instances.create(:name=>@name_args.first, :zone=>selflink2name(zone),
+                                  :image=> image,
+                                  :machineType =>machine_type,
+                                  :disks=>disks,
+                                  :metadata=>{'items'=> metadata },
+                                  :networkInterfaces => [network_interface],
+                                  :tags=> config[:tags]
+                                )
 
-        server = to_json(get_instance.stdout)
-        private_ip = []
-        public_ip = []
-        server["networkInterfaces"].each do  |interface| 
-          private_ip << interface["networkIP"]
-          interface["accessConfigs"].select { |cfg| public_ip << cfg["natIP"] }
+        ui.info("Waiting for the create server operation to complete")
+        until zone_operation.progress.to_i == 100
+          ui.info(".")
+          sleep 1
+          zone_operation = client.zoneOperations.get(:name=>zone_operation, :operation=>zone_operation.name, :zone=>selflink2name(zone))
         end
+        ui.info("Waiting for the servers to be in running state")
 
-        puts "#{ui.color("Public IP Address", :cyan)}: #{public_ip[0]}"
-        puts "#{ui.color("Private IP Address", :cyan)}: #{private_ip[0]}"
-        puts "\n#{ui.color("Waiting for sshd.", :magenta)}"
-        puts(".") until tcp_test_ssh(public_ip[0], "22") { sleep @initial_sleep_delay ||= 10; puts("done") }
-        puts "\nBootstrapping #{h.color(server_name, :bold)}..."
-        bootstrap_for_node(server_name, public_ip[0]).run
-      end
+        @instance = client.instances.get(:name=>@name_args.first, :zone=>selflink2name(zone))
+        msg_pair("Instance Name", @instance.name)
+        msg_pair("MachineType", selflink2name(@instance.machine_type))
+        msg_pair("Image", selflink2name(@instance.image))
+        msg_pair("Zone", selflink2name(@instance.zone))
+        msg_pair("Tags", @instance.tags.has_key?("items") ? @instance.tags["items"].join(",") : "None")
+        until @instance.status == "RUNNING"
+          sleep 3
+          msg_pair("Status", @instance.status.downcase)
+          @instance = client.instances.get(:name=>@name_args.first, :zone=>selflink2name(zone))
+        end
 
-      def bootstrap_for_node(server_name, public_ip)
-        bootstrap = Chef::Knife::Bootstrap.new
-        bootstrap.name_args = [public_ip]
-        bootstrap.config[:run_list] = locate_config_value(:run_list)
-        bootstrap.config[:ssh_user] = locate_config_value(:ssh_user) || "root"
-        bootstrap.config[:identity_file] = locate_config_value(:private_key_file)
-        bootstrap.config[:chef_node_name] = locate_config_value(:chef_node_name) || server_name
-        bootstrap.config[:distro] = locate_config_value(:distro)
-        bootstrap.config[:bootstrap_version] = locate_config_value(:bootstrap_version)
-        bootstrap.config[:use_sudo] = true unless config[:ssh_user] == 'root'
-        bootstrap.config[:template_file] = locate_config_value(:template_file)
-        bootstrap
+        msg_pair("Public IP Address", public_ips(@instance)) unless public_ips(@instance).empty?
+        msg_pair("Private IP Address", private_ips(@instance))
+        ui.info("\n#{ui.color("Waiting for server", :magenta)}")
+
+        ui.info("\n")
+        ui.info(ui.color("Waiting for sshd", :magenta))
+        wait_for_sshd(ssh_connect_host)
+        bootstrap_for_node(@instance,ssh_connect_host).run
+        ui.info("\n")
+        ui.info("Complete!!")
       end
     end
   end
 end