lib/zergrush_cf/init.rb in zergrush_cf-0.0.9 vs lib/zergrush_cf/init.rb in zergrush_cf-0.0.10

@@ -27,10 +27,11 @@
 require 'rbconfig'
 require 'awesome_print'
 require 'securerandom'
 require 'bunny'
 require 'time'
+require 'retries'
 require_relative 'renderer'
 
 class CloudFormation < ZergGemPlugin::Plugin "/driver"
     def rush hive_location, task_name, task_hash, debug
         aws_key_id = task_hash["vm"]["driver"]["driveroptions"][0]["access_key_id"] 
@@ -83,10 +84,14 @@
                     )
                 }
             end
         end
 
+        Excon.defaults[:connect_timeout] = 600
+        Excon.defaults[:read_timeout] = 600
+        Excon.defaults[:write_timeout] = 600
+
         cf = Fog::AWS::CloudFormation.new(
             :aws_access_key_id => aws_key_id,
             :aws_secret_access_key => aws_secret,
             :connection_options => {
                 :connect_timeout => 700,
@@ -110,23 +115,30 @@
         while outputs_info == nil do
             sleep 1 
             outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
         end
 
-        events = cf.describe_stack_events(stack_name).body['StackEvents']
-        while events == nil do
-            sleep 1 
+        events = nil
+        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
             events = cf.describe_stack_events(stack_name).body['StackEvents']
+        }
+        while events == nil do
+            sleep 3
+            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+                events = cf.describe_stack_events(stack_name).body['StackEvents']
+            }
         end
 
         event_counter = 0
         while outputs_info.body["Stacks"][0]["StackStatus"] == "CREATE_IN_PROGRESS" do
             logEvents(events.first(events.length - event_counter))
             logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
             event_counter = events.length
 
-            events = cf.describe_stack_events(stack_name).body['StackEvents']
+            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+                events = cf.describe_stack_events(stack_name).body['StackEvents']
+            }
             outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
             if outputs_info.body["Stacks"][0]["StackStatus"] == "CREATE_COMPLETE"
                 logEvents(events.first(events.length - event_counter))
                 logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
                 puts("Stack outputs:")
@@ -170,10 +182,14 @@
         puts("Cleaning task #{task_name} ...")
 
         # run fog cleanup on the stack.
         stack_name = "#{task_name}"
 
+        Excon.defaults[:connect_timeout] = 600
+        Excon.defaults[:read_timeout] = 600
+        Excon.defaults[:write_timeout] = 600
+        
         cf = Fog::AWS::CloudFormation.new(
             :aws_access_key_id => aws_key_id,
             :aws_secret_access_key => aws_secret,
             :connection_options => {
                 :connect_timeout => 700,
@@ -194,15 +210,20 @@
                 rabbit_objects[:connection].close unless rabbit_objects == nil
                 return 0
             end
         end
 
-        events = cf.describe_stack_events(stack_name).body['StackEvents']
+        events = nil
+        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+            events = cf.describe_stack_events(stack_name).body['StackEvents']
+        }
         while events == nil do
-            sleep 1
+            sleep 3
             begin
-                events = cf.describe_stack_events(stack_name).body['StackEvents']
+                with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+                    events = cf.describe_stack_events(stack_name).body['StackEvents']
+                }
             rescue Fog::AWS::CloudFormation::NotFound
                 rabbit_objects[:connection].close unless rabbit_objects == nil
                 return 0
             end
         end
@@ -212,10 +233,12 @@
             logEvents(events.first(events.length - event_counter))
             logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
 
             event_counter = events.length
             begin
-                events = cf.describe_stack_events(stack_name).body['StackEvents']
+                with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+                    events = cf.describe_stack_events(stack_name).body['StackEvents']
+                }
                 outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
             rescue Fog::AWS::CloudFormation::NotFound
                 logEvents(events.first(events.length - event_counter))
                 logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
                 rabbit_objects[:connection].close unless rabbit_objects == nil