#!/usr/bin/env ruby
# This script can be launched to get a uniq id for this instance
require 'consul/async/consul_template_engine'
require 'consul/async/process_handler'
require 'consul/async/debug'
require 'consul/async/version'
require 'optparse'
require 'optparse/uri'
def usage_text
"USAGE: #{__FILE__} [[options]]"
end
def compute_default_output(source)
dest = source.gsub(/\.erb$/, '')
raise "Source and destination cannot be the same in #{source}" if source == dest || dest.empty?
dest
end
options = {
erb: {
trim_mode: '-', # trim_mode for ERB
},
vault: {
debug: {
network: false
},
base_url: ENV['VAULT_ADDR'] || 'http://localhost:8200',
token: ENV['VAULT_TOKEN'] || nil,
token_renew: true,
retry_duration: 10,
lease_duration_factor: 0.5, # The time it waits before actualizing data based on the lease time: 2h lease * 0.5 = Fetch data every 1h
min_duration: 60,
max_retry_duration: 86_400, # Much higher than consul's because some dynamic secrets could be used for a day.
paths: {
'/v1/sys/mounts' => {
max_retry_duration: 600,
min_duration: 600
}
}
},
consul: {
debug: {
network: false
},
base_url: ENV['CONSUL_HTTP_ADDR'] || 'http://localhost:8500',
token: ENV['CONSUL_HTTP_TOKEN'] || nil,
retry_duration: 10, # On error, retry after n seconds
min_duration: 5, # On sucess and when differences are found
retry_on_non_diff: 3, # On success but when there are not differences
wait_duration: 600, # Delay to block in Consul
max_retry_duration: 600, # On consecutive errors, delay will increase, max value
missing_index_retry_time_on_diff: 15, # On endpoints without X-Consul-Index => next request
missing_index_retry_time_on_unchanged: 60, # Endpoints with X-Consul index and no diff
enable_gzip_compression: true,
paths: {
'/v1/catalog/services': {
min_duration: 15, # Since services change a lot, refresh services every 15 seconds
},
'/v1/catalog/nodes': {
min_duration: 15, # Do not wake up before 15 seconds when node appear/disappear
},
'/v1/catalog/datacenters': {
min_duration: 60, # Datacenters are not added every minute, right?
},
'/v1/agent/metrics': {
min_duration: 60, # Refresh metrics only minute max
},
'/v1/agent/self': {
min_duration: 60, # Refresh self info every minute max
}
}
}
}
consul_engine = Consul::Async::ConsulTemplateEngine.new
@programs = {}
cur_sig_reload = 'HUP'.freeze
cur_sig_term = 'TERM'.freeze
optparse = OptionParser.new do |opts|
opts.banner = usage_text
opts.on('-h', '--help', 'Show help') do
STDERR.puts opts
exit 0
end
opts.on('-v', '--version', 'Show Version') do
STDERR.puts Consul::Async::VERSION
exit 0
end
opts.on('-g', '--no-gzip-compression', 'Disable GZIP compression in HTTP requests') do
options[:consul][:enable_gzip_compression] = false
end
opts.on('-c', '--consul-addr=
', String, 'Address of Consul, eg: http://localhost:8500') do |consul_url|
options[:consul][:base_url] = consul_url
end
opts.on('-l', '--log-level=', String, "Log level, default=info, any of #{::Consul::Async::Debug.levels.join('|')}") do |log_level|
::Consul::Async::Debug.level = log_level
end
opts.on('--consul-token=', String, 'Use a token to connect to Consul') do |consul_token|
options[:consul][:token] = consul_token
end
opts.on('-V', '--vault-addr=', String, 'Address of Vault, eg: http://localhost:8200') do |vault_url|
options[:vault][:base_url] = vault_url
end
opts.on('-T', '--vault-token=', String, 'Token used to authenticate against vault.') do |vault_token|
options[:vault][:token] = vault_token
end
options[:vault][:token_renew] = true
opts.on('--[no-]vault-renew', 'Control auto-renewal of the Vault token. Default: activated') do |vault_renew|
options[:vault][:token_renew] = vault_renew
end
opts.on('--vault-lease-duration-factor=', Float, 'Wait at least * lease time before updating a Vault secret. Default: 0.5') do |factor|
options[:vault][:lease_duration_factor] = factor
end
opts.on('-w', '--wait=', Integer, 'Wait at least n seconds before each template generation') do |min_duration|
raise "--wait=#{min_duration} must be greater than 0" unless min_duration.positive?
consul_engine.template_frequency = min_duration
end
opts.on('-r', '--retry-delay=', Float, 'Min Retry delay on Error/Missing Consul Index') do |min_duration|
options[:consul][:min_duration] = min_duration
end
opts.on('-k', '--hot-reload=', String, 'Control hot reload behaviour, one of :'\
'[die (kill daemon on hot reload failure), '\
'keep (on error, keep running), '\
'disable (hot reload disabled)] ') do |hot_reload_behaviour|
consul_engine.hot_reload_failure = hot_reload_behaviour == 'die' ? nil : hot_reload_behaviour
end
def compute_signal(val, none_value)
valid_signals = Signal.list.keys
raise "Please specify a signal, use any of: #{valid_signals.inspect}" unless val
return nil if val == none_value
raise "Invalid signal #{val}, valid signals: #{valid_signals.inspect}" unless valid_signals.include? val
val
end
opts.on('-K', '--sig-term=kill_signal', String,
"Signal to send to next --exec command on kill, default=#{cur_sig_term}") do |sig|
cur_sig_term = compute_signal(sig, nil)
end
opts.on('-T', '--trim-mode=trim_mode', String,
"ERB Trim mode to use (#{options[:erb][:trim_mode]} by default)") do |trim_mode|
options[:erb][:trim_mode] = trim_mode
end
opts.on('-R', '--sig-reload=reload_signal', String,
"Signal to send to next --exec command on reload (NONE supported), default=#{cur_sig_reload}") do |sig|
cur_sig_reload = compute_signal(sig, 'NONE')
end
opts.on('-M', '--debug-memory-usage', 'Display messages when RAM grows') do
consul_engine.debug_memory = true
end
opts.on('-e', '--exec=', String, 'Execute the following command') do |cmd|
sig_reload = cur_sig_reload
sig_term = cur_sig_term
consul_engine.add_template_callback do |all_ready, template_manager, results|
if all_ready
modified = results.reduce(false) { |a, e| a || e.modified }
if @programs[cmd].nil?
STDERR.puts "[EXEC] Starting process: #{cmd}... on_reload=#{sig_reload ? sig_reload : 'NONE'} on_term=#{sig_term}"
@programs[cmd] = Consul::Async::ProcessHandler.new(cmd, sig_reload: sig_reload, sig_term: sig_term)
@programs[cmd].start
else
# At least one template has been modified
@programs[cmd].reload if modified
begin
@programs[cmd].process_status
rescue Consul::Async::ProcessDoesNotExist => e
STDERR.puts "[FATAL] The process is dead, aborting run: #{e.inspect}"
template_manager.terminate
EventMachine.stop
end
end
end
end
end
opts.on('-d', '--debug-network-usage', 'Debug the network usage') do
options[:consul][:debug][:network] = true
consul_engine.add_template_callback do |all_ready, template_manager, results|
if all_ready
mod = false
results = results.map do |res|
mod ||= res.modified
::Consul::Async::Debug.puts_info "Hot reload of template #{res.template_file} with success" if res.hot_reloaded
"#{res.modified ? 'WRITTEN' : 'UNCHANGED'}[#{res.output_file}]"
end.join(' ')
if mod
::Consul::Async::Debug.puts_info "File written: #{results} #{template_manager.net_info.inspect}"
else
::Consul::Async::Debug.print_debug "Files not changed #{results} #{template_manager.net_info.inspect}\r"
end
else
::Consul::Async::Debug.print_debug "Still waiting for data #{template_manager.net_info.inspect}...\r"
end
end
end
opts.on('-t', '--template erb_file:[output]:[command]:[params_file]', String, 'Add a erb template, its output and optional reload command') do |tpl|
splitted = tpl.split(':')
source = splitted[0]
dest = splitted[1]
unless dest
dest = compute_default_output(source)
STDERR.puts "-t --template #{tpl} : Since output has not been set, using #{dest}"
end
raise "Source and destination cannot be the same in #{tpl}" if source == dest || dest.empty?
command = splitted[2]
parameter_file = splitted[3]
params = {}
params = Consul::Async::Utilities.load_parameters_from_file(parameter_file) if parameter_file
consul_engine.add_template(source, dest, params)
if command
consul_engine.add_template_callback do |_all_ready, _template_manager, results|
results.each do |res|
next unless res.ready? && res.modified && res.output_file == dest && res.template_file == source
# Our template has been fully rendered
system(command)
end
end
end
end
opts.on('-o', '--once', 'Do not run the process as a daemon') do
consul_engine.add_template_callback do |all_ready, template_manager, _|
if all_ready
STDERR.puts '[INFO] Program ends since daemon mode has been disabled, file(s) has been written'
template_manager.terminate
EventMachine.stop
end
end
end
end
def kill_program
@programs.each do |k, v|
STDERR.puts "Killing process #{k}..."
v.kill
end
@programs = {}
exit 0
end
optparse.parse!
# Find the max descriptors for our system
def find_max_descriptors(max_descripors)
i = max_descripors
max_size = 1024
while i != max_size && i > 1024
max_size = EM.set_descriptor_table_size i
i /= 2 if max_size < i
end
max_size
end
%i[consul vault].each do |type|
unless options[type][:base_url].start_with? 'http', 'https'
options[type][:base_url] = "http://#{options[type][:base_url]}"
end
end
# Since we might be using a lots of descriptors, document this
new_size = find_max_descriptors(65_536)
STDERR.puts "Max number of descriptors set to #{new_size}" if options[:consul][:debug][:network]
# This is needed to avoid EM not to crash on some Linux Hosts
# When using a very large number of Consul Endpoints
# See https://github.com/eventmachine/eventmachine/issues/636#issuecomment-143313282
EM.epoll
consul_conf = Consul::Async::ConsulConfiguration.new(options[:consul])
vault_conf = Consul::Async::VaultConfiguration.new(options[:vault])
template_manager = Consul::Async::EndPointsManager.new(consul_conf, vault_conf, options[:erb][:trim_mode])
ARGV.each do |tpl|
dest = compute_default_output(tpl)
::Consul::Async::Debug.puts_info "Using #{dest} output for #{tpl}"
consul_engine.add_template(tpl, dest)
end
# Ensure to kill child process if any
%w[INT PIPE TERM].each do |sig|
Signal.trap(sig) do
STDERR.puts "[KILL] received #{sig}, stopping myself"
template_manager.terminate
kill_program
end
end
consul_engine.run(template_manager)
# Kill possible child process if consul_engine.run did stop
kill_program
exit 0