pagerduty.rb in flapjack-1.6.0rc1

- old
+ new

@@ -14,10 +14,11 @@
   module Gateways
 
     class Pagerduty
       PAGERDUTY_EVENTS_API_URL   = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
       SEM_PAGERDUTY_ACKS_RUNNING = 'sem_pagerduty_acks_running'
+      SEM_PAGERDUTY_ACKS_RUNNING_TIMEOUT = 3600
 
       include Flapjack::Utility
 
       def initialize(opts = {})
         @config = opts[:config]
@@ -127,19 +128,20 @@
       def find_pagerduty_acknowledgements_if_safe
 
         # ensure we're the only instance of the pagerduty acknowledgement check running (with a naive
         # timeout of five minutes to guard against stale locks caused by crashing code) either in this
         # process or in other processes
-        if (@pagerduty_acks_started and @pagerduty_acks_started > (Time.now.to_i - 300)) or
+        if (@pagerduty_acks_started and @pagerduty_acks_started >
+          (Time.now.to_i - SEM_PAGERDUTY_ACKS_RUNNING_TIMEOUT)) or
             @redis.get(SEM_PAGERDUTY_ACKS_RUNNING) == 'true'
           @logger.debug("skipping looking for acks in pagerduty as this is already happening")
           return
         end
 
         @pagerduty_acks_started = Time.now.to_i
         @redis.set(SEM_PAGERDUTY_ACKS_RUNNING, 'true')
-        @redis.expire(SEM_PAGERDUTY_ACKS_RUNNING, 300)
+        @redis.expire(SEM_PAGERDUTY_ACKS_RUNNING, SEM_PAGERDUTY_ACKS_RUNNING_TIMEOUT)
 
         find_pagerduty_acknowledgements
 
         @redis.del(SEM_PAGERDUTY_ACKS_RUNNING)
         @pagerduty_acks_started = nil
@@ -198,26 +200,32 @@
           end
 
           # FIXME: try each set of credentials until one works (may have stale contacts turning up)
           options = ec_credentials.first.merge('check' => "#{entity_name}:#{check}")
 
-          acknowledged = pagerduty_acknowledged?(options)
-          if acknowledged.nil?
-            @logger.debug "#{entity_name}:#{check} is not acknowledged in pagerduty, skipping"
+          # check again that the check is still unacknowledged
+          if entity_check.in_unscheduled_maintenance?
+            # skip this one
+            @logger.warn "#{entity_name}:#{check} seems to have been acknowledged by " +
+              "some other process while I've been running. Cancelling acknowledgement creation"
             next
           end
 
-          # check again that the check is still unacknowledged in flapjack
-          unless Flapjack::Data::EntityCheck.unacknowledged_failing(:redis => @redis).map {|ec|
-              "#{ec.entity_name}:#{ec.check}"
-            }.include?("#{entity_name}:#{check}")
+          # check again that the check is still failing
+          unless entity_check.failed?
             # skip this one
-            @logger.warn "#{entity_name}:#{check} seems to have been acknowledged by " +
-              "some other process while I've been running, cancelling acknowledgement creation"
+            @logger.warn "#{entity_name}:#{check} seems to have recovered " +
+              "while I've been running. Cancelling acknowledgement creation"
             next
           end
 
+          acknowledged = pagerduty_acknowledged?(options)
+          if acknowledged.nil?
+            @logger.debug "#{entity_name}:#{check} is not acknowledged in pagerduty, skipping"
+            next
+          end
+
           pg_acknowledged_by = acknowledged[:pg_acknowledged_by]
           @logger.info "#{entity_name}:#{check} is acknowledged in pagerduty, creating flapjack acknowledgement... "
           who_text = ""
           if !pg_acknowledged_by.nil? && !pg_acknowledged_by['name'].nil?
             who_text = " by #{pg_acknowledged_by['name']}"
@@ -235,18 +243,20 @@
 
       end
 
       def pagerduty_acknowledged?(opts)
         subdomain   = opts['subdomain']
+        token       = opts['token']
         username    = opts['username']
         password    = opts['password']
         check       = opts['check']
 
-        unless subdomain && username && password && check
+        unless subdomain && (token || (username && password)) && check
           @logger.warn("pagerduty_acknowledged?: Unable to look for acknowledgements on pagerduty" +
-                       " as all of the following options are required:" +
-                       " subdomain (#{subdomain}), username (#{username}), password (#{password}), check (#{check})")
+                       " as the following options are required:" +
+                       " subdomain (#{subdomain}), token (#{token}) or" +
+                       " username (#{username}) and password (#{password}), check (#{check})")
           return nil
         end
 
         t = Time.now.utc
 
@@ -255,11 +265,17 @@
                   'since'        => (t - (60*60*24*7)).iso8601, # the last week
                   'until'        => (t + (60*60*24)).iso8601,   # 1 day in the future
                   'incident_key' => check,
                   'status'       => 'acknowledged' }
 
-        options = { :head  => { 'authorization' => [username, password] },
+        auth_header = if token && token.length > 0
+          "Token token=#{token}"
+        else
+          [username, password]
+        end
+
+        options = { :head  => { 'authorization' => auth_header },
                     :query => query }
 
         @logger.debug("pagerduty_acknowledged?: request to #{url}")
         @logger.debug("pagerduty_acknowledged?: query: #{query.inspect}")
         @logger.debug("pagerduty_acknowledged?: auth: #{options[:head].inspect}")
@@ -293,6 +309,5 @@
     end
 
   end
 
 end
-