<% require 'json' require 'set' services_blacklist = (ENV['EXCLUDE_SERVICES'] || 'consul-agent-http,mesos-slave,mesos-agent-watcher,mesos-exporter-slave').split(',') @current_time = Time.now.utc.iso8601 @last_service_info = {} @newest_index = 0 unless @newest_index cur_state = services.map do |service_name, _tags| next if services_blacklist.include?(service_name) snodes = service(service_name) cur_stats = { 'passing' => 0, 'warning' => 0, 'critical' => 0, 'total' => snodes.count } cur_index = snodes.endpoint.x_consul_index.to_i @newest_index = cur_index if @newest_index < cur_index @last_service_info[service_name] = { 'idx' => cur_index, 'stats' => cur_stats } snodes.each do |snode| case snode.status.downcase when 'passing' cur_stats['passing'] += 1 when 'warning' cur_stats['warning'] += 1 else cur_stats['critical'] += 1 end end instances = snodes .sort { |a, b| a['Node']['Node'] <=> b['Node']['Node'] } .map do |instance| ["#{instance['Node']['Node']}:#{instance['Service']['ID']}", { 'address' => instance.service_address, 'node' => instance['Node']['Node'], 'port' => instance['Service']['Port'], 'idx' => (instance['Service']['ModifyIndex'] || cur_index).to_i, 'status' => instance.status, 'stats' => cur_stats, 'checks' => instance['Checks'].map { |check| [check['CheckID'], { 'name' => check['Name'], 'status' => check['Status'], 'output' => check['Output'] }] }.to_h }] end.to_h [service_name, instances] end .compact .to_h old_state = if @previous_state @previous_state else cur_state end unless @events load File.expand_path(File.join(File.dirname(template_info['source']), 'ringbuffer.rb')) def diff(old, new_e) diff = OpenStruct.new diff.appeared = new_e - old diff.disappeared = old - new_e diff.stayed = new_e & old diff.all = new_e + old diff end @events = ::ConsulTimeline::SortedRingBuffer.new((ENV['CONSUL_TIMELINE_BUFFER'] || 10000).to_i, lambda {|a, b| a['idx'] <=> b['idx'] }) end @new_events = [] def log_event(line) puts "#{Time.now.to_i} #{line}" if ENV['DEBUG_TIMELINE'] end def store_event(service: nil, instance: nil, old_state: nil, new_state: nil, instance_info: nil, checks: []) @new_events << { 'service' => service, 'instance' => instance, 'old_state' => old_state, 'new_state' => new_state, 'ts' => @current_time, 'instance_info' => instance_info }.tap do |ev| ev['checks'] = checks if checks ev['stats'] = instance_info['stats'] ev['idx'] = instance_info['idx'] end end def compute_checks(old_state, cur_state, service_name, instance_name) old_checks = old_state.dig(service_name, instance_name, 'checks') || {} new_checks = cur_state.dig(service_name, instance_name, 'checks') || {} all_checks = Set.new(old_checks.keys + new_checks.keys) checks = [] all_checks.each do |check_id| old_status = old_state.dig(service_name, instance_name, 'checks', check_id, 'status') cur_status = cur_state.dig(service_name, instance_name, 'checks', check_id, 'status') next if old_status == cur_status check_name = cur_state.dig(service_name, instance_name, 'checks', check_id, 'name') check_name = old_state.dig(service_name, instance_name, 'checks', check_id, 'name') unless check_name check_name = check_id unless check_name checks << { 'id' => check_id, 'old' => old_status, 'new' => cur_status, 'name' => check_name }.tap do |check| check['output'] = (cur_state.dig(service_name, instance_name, 'checks', check_id, 'output') || '')[0..512] end end checks end service_diff = diff(old_state.keys, cur_state.keys) service_diff.disappeared.each do |service_name| old_state[service_name].each do |instance_name, instance_info| checks = compute_checks(old_state, cur_state, service_name, instance_name) the_info = {}.merge(instance_info) the_info['idx'] = @newest_index the_info['stats'] = { 'passing' => 0, 'warning' => 0, 'critical' => 0, 'total' => 0, } store_event(service: service_name, instance: instance_name, old_state: old_state[service_name][instance_name]['status'], new_state: nil, instance_info: the_info, checks: checks) end end def instances_are_equal(o_state, n_state) return true if o_state == n_state return false unless o_state.nil? || n_state.nil? %w[address node port status].each do |field| return false if o_state[field] != n_state[field] end true end (service_diff.stayed + service_diff.appeared).each do |service_name| instance_diff = diff((old_state[service_name] || {}).keys, cur_state[service_name].keys) instance_diff.disappeared.each do |instance_name| checks = compute_checks(old_state, cur_state, service_name, instance_name) the_info = {}.merge(old_state[service_name][instance_name]) the_info['idx'] = @last_service_info[service_name]['idx'] the_info['stats'] = @last_service_info[service_name]['stats'] store_event(service: service_name, old_state: old_state[service_name][instance_name]['status'], new_state: nil, instance: instance_name, instance_info: the_info, checks: checks) end instance_diff.appeared.each do |instance_name| checks = compute_checks(old_state, cur_state, service_name, instance_name) store_event(service: service_name, old_state: nil, new_state: cur_state[service_name][instance_name]['status'], instance: instance_name, instance_info: cur_state[service_name][instance_name], checks: checks) end instance_diff.stayed.each do |instance_name, _instance_info| checks = compute_checks(old_state, cur_state, service_name, instance_name) o_state = old_state[service_name][instance_name]['status'] n_state = cur_state[service_name][instance_name]['status'] next if instances_are_equal(o_state, n_state) && checks.empty? store_event(service: service_name, old_state: o_state, new_state: n_state, instance: instance_name, instance_info: cur_state[service_name][instance_name], checks: checks) end end sorted_events = @new_events.sort do |a, b| res = 0 %w[idx service instance].each do |f| res = a[f] <=> b[f] break if res != 0 end res end sorted_events.each { |e| @events.push e } @new_events.clear # We save the previous state only when we have a complete state once if template_info['was_rendered_once'] warn "First full rendering completed at #{@current_time} !" unless @previous_state @previous_state = cur_state end %><%= JSON.generate(@events.to_a) %>