lib/helpers/ec2.rb in gaptool-server-0.6.4 vs lib/helpers/ec2.rb in gaptool-server-0.6.6

@@ -1,11 +1,14 @@
 require 'aws-sdk'
 require 'securerandom'
+require 'logger'
+require 'airbrake'
 
 # encoding: utf-8
 module Gaptool
   module EC2
+    @@logger = Logger.new(STDERR)
 
     def self.configure_ec2 zone
       return if ENV['DRYRUN']
       id = ENV['AWS_ACCESS_KEY_ID']
       secret = ENV['AWS_SECRET_ACCESS_KEY']
@@ -43,29 +46,80 @@
       sg = ec2.security_groups.create(groupname)
       sg.authorize_ingress :tcp, 22, *internet
       return sg.id
     end
 
-    def self.create_ec2_instance(ec2opts, data)
+    def self.create_ec2_instance(ec2opts, data, retries=3, sleeptime=0.5)
       if ENV['DRYRUN']
         id = "i-test#{SecureRandom.hex(2)}"
         return {id: id,
                 hostname: "test-#{id}.#{data[:zone].chop}.compute.amazonaws.com",
+                instance: nil,
                 launch_time: Time.now.to_s}
       end
       configure_ec2 data[:zone].chop
       ec2 = AWS::EC2.new
 
-      instance = ec2.instances.create(ec2opts)
-      instance.add_tag('Name', value: "#{data[:role]}-#{data[:env]}-#{instance.id}")
-      instance.add_tag('gaptool', :value => "yes")
-      launch_time = instance.launch_time.to_s
-      launch_time = launch_time.empty? ? Time.now.to_s : launch_time
+      i = 0
+      begin
+        instance = ec2.instances.create(ec2opts)
+        @@logger.debug("Spawned instance #{instance.id}")
+      rescue => e
+        i += 1
+        raise if i > retries
+        @@logger.error("Error while creating instance: #{e}: sleeping #{sleeptime}s and retrying (#{i}/#{retries})")
+        sleep sleeptime
+        retry
+      end
+
+      begin
+        launch_time = instance.launch_time.to_s
+        launch_time = launch_time.empty? ? Time.now.to_s : launch_time
+      rescue
+        launch_time = Time.now.to_s
+      end
+
+      i = 0
+      begin
+        hostname = instance.public_dns_name
+      rescue => e
+        i += 1
+        if i > retries
+          @@logger.error("Could not get hostname for instance #{instance} after #{retries} retries, setting to nil")
+          hostname = nil
+          Airbrake.notify_or_ignore(
+            e,
+            error_class: "EC2 public dns fail",
+            parameters: {instance: instance[:id], role: data['role'], environment: data['environment'], hostname: nil}
+          )
+        else
+          @@logger.error("Error getting hostname for instance: #{e}: sleeping #{sleeptime}s and retrying (#{i}/#{retries})")
+          sleep sleeptime
+          retry
+        end
+      end
       {
         id: instance.id,
-        hostname: instance.public_dns_name,
+        instance: instance,
+        hostname: hostname,
         launch_time: launch_time
       }
+    end
+
+    def self.tag_ec2_instance(instance, key, value, retries=5, sleeptime=0.5)
+      return true if ENV['DRYRUN']
+      i = 0
+      begin
+        instance.add_tag(key, value: value)
+        @@logger.debug("Added tag #{key}=#{value} to #{instance.id}")
+        true
+      rescue => e
+        i += 1
+        raise if i > retries
+        @@logger.error("Error adding tag #{key} to #{instance.id}: #{e}: sleeping #{sleeptime}s and retrying (#{i}/#{retries})")
+        sleep sleeptime
+        retry
+      end
     end
 
     def self.terminate_ec2_instance(zone, id)
       return if ENV['DRYRUN']
       configure_ec2 zone