ec2.rb in poolparty-1.4.0

- old
+ new

@@ -1,35 +1,19 @@
 =begin rdoc
   EC2 CloudProvider
   This serves as the basis for running PoolParty on Amazon's ec2 cloud.
 =end
-require "openssl"
-if OpenSSL::OPENSSL_VERSION_NUMBER < 0x00908000
-  warn "the ec2 cloud provider may not work with your version of ruby and OpenSSL.  Consider upgrading if you encoutner authentication errors."
-end
 begin
-  require 'right_aws'
+  require 'AWS'
 rescue LoadError
   puts <<-EOM
-Error: In order to use ec2, you need to install the right_aws gem
-
-Ec2 is the default cloud provider for PoolParty. If you intend on using
-a different provider, specify it with:
-
-using :provider_name
+  There was an error requiring AWS
 EOM
 end
 
-require "#{File.dirname(__FILE__)}/ec2_helpers"
-require "#{File.dirname(__FILE__)}/ec2_response"
-require "#{File.dirname(__FILE__)}/ec2_instance"
-
 module CloudProviders
   class Ec2 < CloudProvider
-    
-    include CloudProviders::Ec2Helpers
-    
     # Set the aws keys from the environment, or load from /etc/poolparty/env.yml if the environment variable is not set
     def self.default_access_key
       ENV['EC2_ACCESS_KEY'] || load_keys_from_file[:access_key]
     end
     
@@ -60,162 +44,293 @@
     def self.default_cloud_cert
      ENV['CLOUD_CERT'] || ENV['EUCALYPTUS_CERT'] || load_keys_from_file[:cloud_cert]
     end
     
     # Load the yaml file containing keys.  If the file does not exist, return an empty hash
-    def self.load_keys_from_file(filename='/etc/poolparty/env.yml', caching=true)
+    def self.load_keys_from_file(filename="#{ENV["HOME"]}/.poolparty/aws", caching=true)
       return @aws_yml if @aws_yml && caching==true
       return {} unless File.exists?(filename)
-      ddputs("Reading keys from file: #{filename}")
+      puts("Reading keys from file: #{filename}")
       @aws_yml = YAML::load( open(filename).read ) || {}
     end
     
-    default_options({
-        :image_id               => 'ami-bf5eb9d6',
-        :instance_type          => 'm1.small',
-        :addressing_type        => "public",
-        :availability_zone      => "us-east-1a",
-        :security_group         => ["default"],
-        :user_id                => default_user_id,
-        :private_key            => default_private_key,
-        :cert                   => default_cert,
-        :cloud_cert             => default_cloud_cert,
-        :access_key             => default_access_key,
-        :secret_access_key      => default_secret_access_key,
-        :ec2_url                => default_ec2_url,
-        :s3_url                 => default_s3_url,
-        :min_count              => 1,
-        :max_count              => 1,
-        :user_data              => '',
-        :addressing_type        => nil,
-        :kernel_id              => nil,
-        :ramdisk_id             => nil,
-        :availability_zone      => nil,
-        :block_device_mappings  => nil,
-        :elastic_ips            => [],  # An array of the elastic ips
-        :ebs_volumes            => []   # The volume id of an ebs volume # TODO: ensure this is consistent with :block_device_mappings
-      })
-      
-      
-    def ec2(o={})
-      @ec2 ||= Rightscale::Ec2.new(access_key, secret_access_key, o.merge(:logger => PoolParty::PoolPartyLog, :endpoint_url => ec2_url))
+    default_options(
+      :image_id               => 'ami-ed46a784',
+      :instance_type          => 'm1.small',
+      :addressing_type        => "public",
+      :availability_zones     => ["us-east-1a"],
+      :user_id                => default_user_id,
+      :private_key            => default_private_key,
+      :cert                   => default_cert,
+      :cloud_cert             => default_cloud_cert,
+      :access_key             => default_access_key,
+      :secret_access_key      => default_secret_access_key,
+      :ec2_url                => default_ec2_url,
+      :s3_url                 => default_s3_url,
+      :min_count              => 1,
+      :max_count              => 1,
+      :user_data              => '',
+      :addressing_type        => nil,
+      :kernel_id              => nil,
+      :ramdisk_id             => nil,
+      :block_device_mappings  => nil,
+      :elastic_ips            => [],  # An array of the elastic ips
+      :ebs_volumes            => []   # The volume id of an ebs volume # TODO: ensure this is consistent with :block_device_mappings
+    )
+    
+    # Called when the create command is called on the cloud
+    def create!
+      [:security_groups, :load_balancers].each do |type|
+        self.send(type).each {|ele| ele.create! }
+      end
     end
     
-    # Start a new instance with the given options
-    def run_instance(o={})
-      number_of_instances = o[:number_of_instances] || 1
-      set_vars_from_options o
-      raise StandardError.new("You must pass a keypair to launch an instance, or else you will not be able to login. options = #{o.inspect}") if !keypair_name
-      response_array = ec2(o).run_instances(image_id,
-                                      min_count,
-                                      number_of_instances,
-                                      security_group,
-                                      keypair.basename,
-                                      user_data,
-                                      addressing_type,
-                                      instance_type,
-                                      kernel_id,
-                                      ramdisk_id,
-                                      availability_zone,
-                                      block_device_mappings
-                                      )
-      instances = response_array.collect do |aws_response_hash|
-        Ec2Instance.new( Ec2Response.pp_format(aws_response_hash).merge(o) )
+    def run
+      puts "  for cloud: #{cloud.name}"
+      puts "  minimum_instances: #{minimum_instances}"
+      puts "  maximum_instances: #{maximum_instances}"
+      puts "  security_groups: #{security_group_names.join(", ")}"
+      puts "  running on keypair: #{keypair}"
+            
+      security_groups.each do |sg|
+        sg.run
       end
       
-      after_run_instance(instances)
+      unless load_balancers.empty?
+        load_balancers.each do |lb|
+          puts "    load balancer: #{lb.name}"
+          lb.run
+        end
+      end     
+      if autoscalers.empty?
+        puts "---- live, running instances (#{nodes.size}) ----"
+        if nodes.size < minimum_instances
+          expansion_count = minimum_instances - nodes.size
+          puts "-----> expanding the cloud because the minimum_instances is not satisified: #{expansion_count}"
+          expand_by(expansion_count)
+        elsif nodes.size > maximum_instances
+          contraction_count = nodes.size - maximum_instances
+          puts "-----> contracting the cloud because the instances count exceeds the maximum_instances by #{contraction_count}"
+          contract_by(contraction_count)
+        end
+        
+        progress_bar_until("Waiting for the instances to be launched") do
+          reset!
+          running_nodes = nodes.select {|n| n.running? }
+          running_nodes.size >= minimum_instances
+        end
+        reset!
+        # ELASTIC IPS
+        unless _elastic_ips.empty?
+          unused_elastic_ip_addresses = ElasticIp.unused_elastic_ips(self).map {|i| i.public_ip }
+          used_elastic_ip_addresses = ElasticIp.elastic_ips(self).map {|i| i.public_ip }
+
+          elastic_ip_objects = ElasticIp.unused_elastic_ips(self).select {|ip_obj| _elastic_ips.include?(ip_obj.public_ip) }
+
+          assignee_nodes = nodes.select {|n| !ElasticIp.elastic_ips(self).include?(n.public_ip) }
+
+          elastic_ip_objects.each_with_index do |eip, idx|
+            # Only get the nodes that do not have elastic ips associated with them
+            begin
+              if assignee_nodes[idx]
+                puts "Assigning elastic ip: #{eip.public_ip} to node: #{assignee_nodes[idx].instance_id}"
+                ec2.associate_address(:instance_id => assignee_nodes[idx].instance_id, :public_ip => eip.public_ip)
+              end
+            rescue Exception => e
+              p [:error, e.inspect]
+            end
+            reset!
+          end
+        end
+      else
+        autoscalers.each do |a|
+          puts "    autoscaler: #{a.name}"
+          puts "-----> The autoscaling groups will launch the instances"
+          a.run
+          
+          progress_bar_until("Waiting for autoscaler to launch instances") do
+            reset!
+            running_nodes = nodes.select {|n| n.running? }
+            running_nodes.size >= minimum_instances
+          end
+          reset!
+        end
+      end
       
-      instances.first
+      reset!
+      from_ports = security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten
+      
+      if from_ports.include?(22)
+        progress_bar_until("Waiting for the instances to be accessible by ssh") do
+          running_nodes = nodes.select {|n| n.running? }
+          accessible_count = running_nodes.map do |node|
+            node.accessible?
+          end.size
+          accessible_count == running_nodes.size
+        end
+      end
+      
     end
     
-    # Will select the first instance matching the provided criteria hash
-    def describe_instance(hash_of_criteria_to_select_instance_against)
-      describe_instances(hash_of_criteria_to_select_instance_against).first
+    def teardown
+      puts "------ Tearing down and cleaning up #{cloud.name} cloud"
+      unless autoscalers.empty?
+        puts "Tearing down autoscalers"
+      end
     end
     
-    # Describe instances
-    def describe_instances(o={})
-      instants = Ec2Response.describe_instances(ec2.describe_instances).select_with_hash(o)
-      return [] if instants.empty?
-      ec2_instances = instants.collect{|i| Ec2Instance.new(dsl_options.merge(i))}
-      ec2_instances.sort {|a,b| a[:launch_time].to_i <=> b[:launch_time].to_i }
+    def expand_by(num=1)
+      e = Ec2Instance.run!({
+        :image_id => image_id,
+        :min_count => num,
+        :max_count => num,
+        :key_name => keypair.basename,
+        :security_groups => security_groups,
+        :user_data => user_data,
+        :instance_type => instance_type,
+        :availability_zone => availability_zones.first,
+        :base64_encoded => true,
+        :cloud => cloud
+      })
+      reset!
+      e
     end
     
-    # Terminate an instance (or instances) by passing :instance_id and :instance_ids
-    def terminate_instance!(o={})
-      raise StandardError.new("You must pass an instance_id when terminating an instance with ec2") unless o[:instance_id] || o[:instance_ids]
-      instance_ids = o[:instance_ids] || [o[:instance_id]]
-      response = ec2.terminate_instances(instance_ids)
-      response.collect{|i| Ec2Instance.new(Ec2Response.pp_format(i)) }
+    def contract_by(num=1)
+      num.times do |i|
+        id = nodes[-num].instance_id
+        Ec2Instance.terminate!(:instance_id => id, :cloud => cloud)
+      end
+      reset!
     end
     
+    def bootstrap_nodes!(tmp_path=nil)
+      tmp_path ||= cloud.tmp_path
+      nodes.each do |node|
+        next unless node.in_service?
+        node.cloud_provider = self
+        node.rsync_dir(tmp_path)
+        node.bootstrap_chef!
+        node.run_chef!
+      end
+    end
     
-=begin rdoc
-  Helper methods for the Ec2 Cloud Provider. Helpers are not necessarily supported across all CloudProviders
-=end
-    # Are we running on amazon?
-    def amazon?
-      !['https://ec2.amazonaws.com', 
-       'https://us-east-1.ec2.amazonaws.com', 
-       'https://eu-west-1.ec2.amazonaws.com'
-       ].include?(ec2_url)
+    def configure_nodes!(tmp_path=nil)
+      tmp_path ||= cloud.tmp_path
+      nodes.each do |node|
+        next unless node.in_service?
+        node.cloud_provider = self
+        node.rsync_dir(tmp_path) if tmp_path
+        node.run_chef!
+      end
     end
     
-    # Callbacks
-    def before_compile(cld)
+    def nodes
+      all_nodes.select {|i| i.in_service? }#describe_instances.select {|i| i.in_service? && security_groups.include?(i.security_groups) }
     end
     
-    def after_compile(cld)
-      save_aws_env_to_yml(cld.tmp_path/"etc"/"poolparty"/"env.yml") rescue nil
+    def all_nodes
+      #TODO: need to sort by launch time
+      # 
+      @nodes ||= describe_instances.select {|i| security_group_names.include?(i.security_groups) }
     end
     
-    # Run after all the instances are run
-    def after_run_instance(instances_list)
-      instances_list.each do |inst|
-        associate_address(inst.instance_id) if next_unused_elastic_ip
-        attach_volume(inst.instance_id) if next_unused_volume
-      end
+    # Describe instances
+    # Describe the instances that are available on this cloud
+    # @params id (optional) if present, details about the instance
+    #   with the id given will be returned
+    #   if not given, details for all instances will be returned
+    def describe_instances(id=nil)
+      @describe_instances = ec2.describe_instances.reservationSet.item.map do |r|
+        r.instancesSet.item.map do |i|
+          inst_options = i.merge(r.merge(:cloud => cloud)).merge(cloud.cloud_provider.dsl_options)
+          Ec2Instance.new(inst_options)
+        end
+      end.flatten
+      # id.nil? ? @describe_instances : @describe_instances.select {|a| a.instance_id == id }.first
     end
     
-    # Read  yaml file and use it to set environment variables and local variables.
-    def set_aws_env_from_yml_file(filename='/etc/poolparty/env.yml')
-      aws = self.class.load_keys_from_file(filename)
-      aws.each{|k,v| ENV[k.upcase]=v.to_s}
-      set_vars_from_options aws
+    # Extras!
+    
+    def load_balancer(name=cloud.proper_name, o={}, &block)
+      load_balancers << ElasticLoadBalancer.new(name, sub_opts.merge(o || {}), &block)
     end
+    def autoscale(name=cloud.proper_name, o={}, &block)
+      autoscalers << ElasticAutoScaler.new(name, sub_opts.merge(o || {}), &block)
+    end
+    def security_group(name=cloud.proper_name, o={}, &block)
+      security_groups << SecurityGroup.new(name, sub_opts.merge(o || {}), &block)
+    end
+    def elastic_ip(*ips)
+      ips.each {|ip| _elastic_ips << ip}
+    end
+        
+    # Proxy to the raw Grempe amazon-aws @ec2 instance
+    def ec2
+      @ec2 ||= AWS::EC2::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
+    end
     
-    # Save aws keys and env variables to a yaml file
-    def save_aws_env_to_yml(filename='/etc/poolparty/env.yml')
-      hsh = aws_hash(default_options, "/etc/poolparty/ec2")
-      File.open(filename, 'w') {|f| f<<YAML::dump(hsh) }
+    # Proxy to the raw Grempe amazon-aws autoscaling instance
+    def as
+      @as = AWS::Autoscaling::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
     end
     
-    # Return a hash of the aws keys and environment variables
-    # If base_dir string is provided as second argument, replace path to 
-    # file based variables, such as cert, with the base_dir.
-    def aws_hash(opts={}, base_dir=nil)
-      aws={
-        :user_id            => user_id,
-        :private_key        => private_key,
-        :cert               => cert,
-        :access_key         => access_key,
-        :secret_access_key  => secret_access_key,
-        :ec2_url            => ec2_url,
-        :s3_url             => s3_url,
-        :cloud_cert    => cloud_cert
-      }.merge(opts)
-      if base_dir
-        aws[:cert] = "#{base_dir}/#{File.basename(cert)}" if cert
-        aws[:private_key] = "#{base_dir}/#{File.basename(private_key)}" if private_key
-        aws[:cloud_cert] = "#{base_dir}/#{File.basename(cloud_cert)}" if cloud_cert
-      end
-      aws.reject{|k,v| v.nil?}
+    # Proxy to the raw Grempe amazon-aws elastic_load_balancing instance
+    def elb
+      @elb ||= AWS::ELB::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
     end
+    def security_group_names
+      security_groups.map {|a| a.to_s }
+    end
+    def security_groups
+      @security_groups ||= []
+    end
+    def load_balancers
+      @load_balancers ||= []
+    end
+    def autoscalers
+      @autoscalers ||= []
+    end
     
-    # shortcut to 
-    # ec2-add-keypair name > ~./.ec2/kname
-    def create_keypair(kname, path='~/.ec2')
-      ` ec2-add-keypair #{kname} > #{path}/#{kname} &&  chmod 600 #{path}/#{kname}`
+    # Clear the cache
+    def reset!
+      @nodes = @describe_instances = nil
     end
     
+    private
+    # Helper to get the options with self as parent
+    def sub_opts
+      dsl_options.merge(:parent => self, :cloud => cloud)
+    end
+    def _elastic_ips
+      @_elastic_ips ||= []
+    end
+    def generate_keypair(n=nil)
+      puts "[EC2] generate_keypair is called with #{default_keypair_path/n}"
+      begin
+        hsh = ec2.create_keypair(:key_name => n)
+        string = hsh.keyMaterial
+        FileUtils.mkdir_p default_keypair_path unless File.directory?(default_keypair_path)
+        puts "[EC2] Generated keypair #{default_keypair_path/n}"
+        puts "[EC2] #{string}"
+        File.open(default_keypair_path/n, "w") {|f| f << string }
+        File.chmod 0600, default_keypair_path/n
+      rescue Exception => e
+        puts "[EC2] The keypair exists in EC2, but we cannot find the keypair locally: #{n} (#{e.inspect})"
+      end
+      keypair n
+    end
+    
   end
+end
+
+require "#{File.dirname(__FILE__)}/ec2_instance"
+require "#{File.dirname(__FILE__)}/helpers/ec2_helper"
+%w( security_group 
+    authorize 
+    elastic_auto_scaler 
+    elastic_block_store 
+    elastic_load_balancer 
+    elastic_ip
+    revoke).each do |lib|
+  require "#{File.dirname(__FILE__)}/helpers/#{lib}"
 end
\ No newline at end of file