#!/usr/bin/env ruby # This script can be launched to get a uniq id for this instance require 'consul/async/consul_template_engine' require 'consul/async/process_handler' require 'consul/async/debug' require 'consul/async/version' require 'optparse' require 'optparse/uri' def usage_text "USAGE: #{__FILE__} [[options]]" end def compute_default_output(source) dest = source.gsub(/\.erb$/, '') raise "Source and destination cannot be the same in #{source}" if source == dest || dest.empty? dest end options = { erb: { trim_mode: '-', # trim_mode for ERB }, vault: { debug: { network: false }, base_url: ENV['VAULT_ADDR'] || 'http://localhost:8200', token: ENV['VAULT_TOKEN'] || nil, token_renew: true, retry_duration: 10, lease_duration_factor: 0.5, # The time it waits before actualizing data based on the lease time: 2h lease * 0.5 = Fetch data every 1h min_duration: 60, max_retry_duration: 86_400, # Much higher than consul's because some dynamic secrets could be used for a day. paths: { '/v1/sys/mounts' => { max_retry_duration: 600, min_duration: 600 } } }, consul: { debug: { network: false }, base_url: ENV['CONSUL_HTTP_ADDR'] || 'http://localhost:8500', token: ENV['CONSUL_HTTP_TOKEN'] || nil, retry_duration: 10, # On error, retry after n seconds min_duration: 5, # On sucess and when differences are found retry_on_non_diff: 3, # On success but when there are not differences wait_duration: 600, # Delay to block in Consul max_retry_duration: 600, # On consecutive errors, delay will increase, max value missing_index_retry_time_on_diff: 15, # On endpoints without X-Consul-Index => next request missing_index_retry_time_on_unchanged: 60, # Endpoints with X-Consul index and no diff enable_gzip_compression: true, paths: { '/v1/catalog/services': { min_duration: 15, # Since services change a lot, refresh services every 15 seconds }, '/v1/catalog/nodes': { min_duration: 15, # Do not wake up before 15 seconds when node appear/disappear }, '/v1/catalog/datacenters': { min_duration: 60, # Datacenters are not added every minute, right? }, '/v1/agent/metrics': { min_duration: 60, # Refresh metrics only minute max }, '/v1/agent/self': { min_duration: 60, # Refresh self info every minute max } } } } consul_engine = Consul::Async::ConsulTemplateEngine.new @programs = {} cur_sig_reload = 'HUP'.freeze cur_sig_term = 'TERM'.freeze optparse = OptionParser.new do |opts| opts.banner = usage_text opts.on('-h', '--help', 'Show help') do STDERR.puts opts exit 0 end opts.on('-v', '--version', 'Show Version') do STDERR.puts Consul::Async::VERSION exit 0 end opts.on('-g', '--no-gzip-compression', 'Disable GZIP compression in HTTP requests') do options[:consul][:enable_gzip_compression] = false end opts.on('-c', '--consul-addr=
', String, 'Address of Consul, eg: http://localhost:8500') do |consul_url| options[:consul][:base_url] = consul_url end opts.on('-l', '--log-level=', String, "Log level, default=info, any of #{::Consul::Async::Debug.levels.join('|')}") do |log_level| ::Consul::Async::Debug.level = log_level end opts.on('--consul-token=', String, 'Use a token to connect to Consul') do |consul_token| options[:consul][:token] = consul_token end opts.on('-V', '--vault-addr=
', String, 'Address of Vault, eg: http://localhost:8200') do |vault_url| options[:vault][:base_url] = vault_url end opts.on('-T', '--vault-token=', String, 'Token used to authenticate against vault.') do |vault_token| options[:vault][:token] = vault_token end options[:vault][:token_renew] = true opts.on('--[no-]vault-renew', 'Control auto-renewal of the Vault token. Default: activated') do |vault_renew| options[:vault][:token_renew] = vault_renew end opts.on('--vault-lease-duration-factor=', Float, 'Wait at least * lease time before updating a Vault secret. Default: 0.5') do |factor| options[:vault][:lease_duration_factor] = factor end opts.on('-w', '--wait=', Integer, 'Wait at least n seconds before each template generation') do |min_duration| raise "--wait=#{min_duration} must be greater than 0" unless min_duration.positive? consul_engine.template_frequency = min_duration end opts.on('-r', '--retry-delay=', Float, 'Min Retry delay on Error/Missing Consul Index') do |min_duration| options[:consul][:min_duration] = min_duration end opts.on('-k', '--hot-reload=', String, 'Control hot reload behaviour, one of :'\ '[die (kill daemon on hot reload failure), '\ 'keep (on error, keep running), '\ 'disable (hot reload disabled)] ') do |hot_reload_behaviour| consul_engine.hot_reload_failure = hot_reload_behaviour == 'die' ? nil : hot_reload_behaviour end def compute_signal(val, none_value) valid_signals = Signal.list.keys raise "Please specify a signal, use any of: #{valid_signals.inspect}" unless val return nil if val == none_value raise "Invalid signal #{val}, valid signals: #{valid_signals.inspect}" unless valid_signals.include? val val end opts.on('-K', '--sig-term=kill_signal', String, "Signal to send to next --exec command on kill, default=#{cur_sig_term}") do |sig| cur_sig_term = compute_signal(sig, nil) end opts.on('-T', '--trim-mode=trim_mode', String, "ERB Trim mode to use (#{options[:erb][:trim_mode]} by default)") do |trim_mode| options[:erb][:trim_mode] = trim_mode end opts.on('-R', '--sig-reload=reload_signal', String, "Signal to send to next --exec command on reload (NONE supported), default=#{cur_sig_reload}") do |sig| cur_sig_reload = compute_signal(sig, 'NONE') end opts.on('-M', '--debug-memory-usage', 'Display messages when RAM grows') do consul_engine.debug_memory = true end opts.on('-e', '--exec=', String, 'Execute the following command') do |cmd| sig_reload = cur_sig_reload sig_term = cur_sig_term consul_engine.add_template_callback do |all_ready, template_manager, results| if all_ready modified = results.reduce(false) { |a, e| a || e.modified } if @programs[cmd].nil? STDERR.puts "[EXEC] Starting process: #{cmd}... on_reload=#{sig_reload ? sig_reload : 'NONE'} on_term=#{sig_term}" @programs[cmd] = Consul::Async::ProcessHandler.new(cmd, sig_reload: sig_reload, sig_term: sig_term) @programs[cmd].start else # At least one template has been modified @programs[cmd].reload if modified begin @programs[cmd].process_status rescue Consul::Async::ProcessDoesNotExist => e STDERR.puts "[FATAL] The process is dead, aborting run: #{e.inspect}" template_manager.terminate EventMachine.stop end end end end end opts.on('-d', '--debug-network-usage', 'Debug the network usage') do options[:consul][:debug][:network] = true consul_engine.add_template_callback do |all_ready, template_manager, results| if all_ready mod = false results = results.map do |res| mod ||= res.modified ::Consul::Async::Debug.puts_info "Hot reload of template #{res.template_file} with success" if res.hot_reloaded "#{res.modified ? 'WRITTEN' : 'UNCHANGED'}[#{res.output_file}]" end.join(' ') if mod ::Consul::Async::Debug.puts_info "File written: #{results} #{template_manager.net_info.inspect}" else ::Consul::Async::Debug.print_debug "Files not changed #{results} #{template_manager.net_info.inspect}\r" end else ::Consul::Async::Debug.print_debug "Still waiting for data #{template_manager.net_info.inspect}...\r" end end end opts.on('-t', '--template erb_file:[output]:[command]:[params_file]', String, 'Add a erb template, its output and optional reload command') do |tpl| splitted = tpl.split(':') source = splitted[0] dest = splitted[1] unless dest dest = compute_default_output(source) STDERR.puts "-t --template #{tpl} : Since output has not been set, using #{dest}" end raise "Source and destination cannot be the same in #{tpl}" if source == dest || dest.empty? command = splitted[2] parameter_file = splitted[3] params = {} params = Consul::Async::Utilities.load_parameters_from_file(parameter_file) if parameter_file consul_engine.add_template(source, dest, params) if command consul_engine.add_template_callback do |_all_ready, _template_manager, results| results.each do |res| next unless res.ready? && res.modified && res.output_file == dest && res.template_file == source # Our template has been fully rendered system(command) end end end end opts.on('-o', '--once', 'Do not run the process as a daemon') do consul_engine.add_template_callback do |all_ready, template_manager, _| if all_ready STDERR.puts '[INFO] Program ends since daemon mode has been disabled, file(s) has been written' template_manager.terminate EventMachine.stop end end end end def kill_program @programs.each do |k, v| STDERR.puts "Killing process #{k}..." v.kill end @programs = {} exit 0 end optparse.parse! # Find the max descriptors for our system def find_max_descriptors(max_descripors) i = max_descripors max_size = 1024 while i != max_size && i > 1024 max_size = EM.set_descriptor_table_size i i /= 2 if max_size < i end max_size end %i[consul vault].each do |type| unless options[type][:base_url].start_with? 'http', 'https' options[type][:base_url] = "http://#{options[type][:base_url]}" end end # Since we might be using a lots of descriptors, document this new_size = find_max_descriptors(65_536) STDERR.puts "Max number of descriptors set to #{new_size}" if options[:consul][:debug][:network] # This is needed to avoid EM not to crash on some Linux Hosts # When using a very large number of Consul Endpoints # See https://github.com/eventmachine/eventmachine/issues/636#issuecomment-143313282 EM.epoll consul_conf = Consul::Async::ConsulConfiguration.new(options[:consul]) vault_conf = Consul::Async::VaultConfiguration.new(options[:vault]) template_manager = Consul::Async::EndPointsManager.new(consul_conf, vault_conf, options[:erb][:trim_mode]) ARGV.each do |tpl| dest = compute_default_output(tpl) ::Consul::Async::Debug.puts_info "Using #{dest} output for #{tpl}" consul_engine.add_template(tpl, dest) end # Ensure to kill child process if any %w[INT PIPE TERM].each do |sig| Signal.trap(sig) do STDERR.puts "[KILL] received #{sig}, stopping myself" template_manager.terminate kill_program end end consul_engine.run(template_manager) # Kill possible child process if consul_engine.run did stop kill_program exit 0