lib/stack.rb in stack-kicker-0.0.10 vs lib/stack.rb in stack-kicker-0.0.11
- old
+ new
@@ -26,14 +26,14 @@
require 'tempfile'
#
# This really needs to be converted into a class....
-#
+#
module Stack
- # Shadow the global constant Logger with Stack::Logger
+ # Shadow the global constant Logger with Stack::Logger
# (if you want access to the global constant, use ::Logger from inside the Stack module)
Logger = Logger.new(STDOUT)
Logger.level = ::Logger::INFO
Logger.datetime_format = "%Y-%m-%d %H:%M:%S"
Logger.formatter = proc do |severity, datetime, progname, msg|
@@ -58,11 +58,11 @@
Logger.info { "Stacks:" }
StackConfig::Stacks.each do |name, details|
Logger.info { " #{name}" }
end
end
-
+
def Stack.show_stack(config)
# syntax_check is a light weight check that doesn't talk to OpenStalk
Stack.syntax_check(config)
# generate an array of hostnames that this stack would create
hostnames = Stack.generate_server_names(config)
@@ -74,11 +74,11 @@
# our local config file
config_raw = File.read(stackfile)
eval(config_raw)
# if there is only one stack defined in the Stackfile, load it:
- if StackConfig::Stacks.count == 1 && stack_name.nil?
+ if StackConfig::Stacks.count == 1 && stack_name.nil?
stack_name = StackConfig::Stacks.keys[0]
Logger.info { "Defaulting to #{stack_name} as there is a single stack defined and no stack named" }
end
# returns a config object, injecting the name into the returned config
@@ -127,13 +127,13 @@
address_description
end
# check that all the required config items are set
def Stack.syntax_check(config)
- if config['REGION'].nil? || config['USERNAME'].nil? || config['PASSWORD'].nil? || config['AUTH_URL'].nil? || config['TENANT_NAME'].nil? &&
+ if config['REGION'].nil? || config['USERNAME'].nil? || config['PASSWORD'].nil? || config['AUTH_URL'].nil? || config['TENANT_NAME'].nil? &&
config['REGION'].empty? || config['USERNAME'].empty? || config['PASSWORD'].empty? || config['AUTH_URL'].empty? || config['TENANT_NAME'].empty?
- Logger.error { "REGION, USERNAME, PASSWORD, AUTH_URL & TENANT_NAME must all be set" }
+ Logger.error { "REGION, USERNAME, PASSWORD, AUTH_URL & TENANT_NAME must all be set" }
exit
end
# load defaults for any items not configured
Stack.populate_config(config)
@@ -144,11 +144,11 @@
# validation.pem to to exist
dot_chef_abs = File.absolute_path(config[:stackhome] + '/' + config[:dot_chef])
if !File.directory?(dot_chef_abs)
Logger.warn "#{dot_chef_abs} doesn't exist"
end
-
+
# Check we have a #{dot_chef_abs}/.chef/knife.rb
knife_rb_abs = dot_chef_abs + '/knife.rb'
if File.exists?(knife_rb_abs)
Logger.info "Found #{knife_rb_abs}, lets hope it contains something sensible"
else
@@ -173,11 +173,11 @@
Logger.info "Found #{config[:key_pair]} in the ssh-agent key list"
else
Logger.error "Couldn't find #{config[:key_pair]} key in the ssh-agent key list! Aborting!"
Logger.erroLogger.error "ssh_keys_loaded: #{ssh_keys_loaded}"
exit 2
- end
+ end
end
# populate the config & then walk through the AZs verifying the config
Stack.populate_config(config)
@@ -197,11 +197,11 @@
Logger.info "Checking that keypair #{config[:key_pair]} exists in #{az}...."
keypairs = os.keypairs()
if (keypairs[config[:key_pair]].nil? && keypairs[config[:key_pair].to_sym].nil?)
Logger.warn "#{config[:key_pair]} isn't available, uploading the key"
-
+
# upload the key
key = os.create_keypair({:name=> config[:key_pair], :public_key=> File.read(config[:key_public])})
Logger.warn "#{config[:key_pair]} fingerprint=#{key[:fingerprint]}"
else
Logger.info "#{config[:key_pair]} fingerprint=#{keypairs[config[:key_pair].to_sym][:fingerprint]}"
@@ -214,14 +214,14 @@
# extract the names
sg_names = security_groups.map { |secgroup, secgroup_details| secgroup_details[:name] }
config[:roles].each do |role, role_details|
# is does the secgroup exist?
- if sg_names.include?(role.to_s)
- Logger.info "security group #{role} exists in #{az}"
+ if sg_names.include?(role_details[:security_group])
+ Logger.info "security group #{role_details[:security_group]} exists in #{az}"
else
- Logger.error "security group #{role} is missing in #{az}"
+ Logger.error "security group #{role_details[:security_group]} is missing in #{az}"
end
end
end
end
@@ -249,11 +249,11 @@
Dir.mkdir(dot_chef_abs)
end
client_key = dot_chef_abs + '/' + config[:name] + '-' + ENV['USER'] + '.pem'
validation_key = dot_chef_abs + '/' + config[:name] + '-' + 'validation.pem'
-
+
Logger.debug "stackhome: #{config[:stackhome]}"
Logger.debug "Current user client key: #{client_key}"
Logger.debug "New Host Validation key: #{validation_key}"
knife_rb_template = %q{
@@ -288,26 +288,26 @@
Logger.debug config[:site_template]
Logger.debug role_details
Logger.debug role_details[:azs]
site = sprintf(config[:site_template], role_details[:azs][position-1].split('.')[0].sub(/-/, ''))
-
+
# generate the hostname
hostname = sprintf(config[:name_template], config[:global_service_name], site, role, position)
- hostname
+ hostname
end
def Stack.generate_server_names(config)
Stack.populate_config(config)
config[:hostnames] = config[:node_details].keys
config[:hostnames]
end
def Stack.populate_config(config)
# config[:role_details] contains built out role details with defaults filled in from stack defaults
- # config[:node_details] contains node details built out from role_details
+ # config[:node_details] contains node details built out from role_details
# set some sensible defaults to the stack-wide defaults if they haven't been set in the Stackfile.
if config[:provisioner].nil?
Logger.warn { "Defaulting to chef for config[:provisioner] "}
config[:provisioner] = 'chef'
@@ -328,11 +328,11 @@
config[:chef_validation_pem] = '.chef/validation.pem'
end
if config[:name_template].nil?
Logger.warn { "Defaulting to '%s-%s-%s%04d' for config[:name_template]" }
- config[:name_template] = '%s-%s-%s%04d'
+ config[:name_template] = '%s-%s-%s%04d'
end
if config[:site_template].nil?
Logger.warn { "Defaulting to '%s' for config[:site_template]" }
config[:site_template] = '%s'
@@ -341,28 +341,28 @@
if config[:global_service_name].nil?
Logger.error { "Defaulting to 'UNKNOWN' for config[:global_service_name]" }
config[:site_template] = 'UNKNOWN'
end
-
+
if config[:node_details].nil?
Logger.debug { "Initializing config[:node_details] and config[:azs]" }
config[:node_details] = Hash.new
config[:azs] = Array.new
- config[:roles].each do |role,role_details|
+ config[:roles].each do |role,role_details|
Logger.debug { "Setting defaults for #{role}" }
-
+
# default to 1 node of this role if :count isn't set
if role_details[:count].nil?
role_details[:count] = 1
end
-
+
if (role_details[:data_dir].nil?)
role_details[:data_dir] = '/dummy'
end
-
+
# Has the cloud_config_yaml been overridden?
if (role_details[:cloud_config_yaml])
role_details[:cloud_config_yaml] = Stack.find_file(config, role_details[:cloud_config_yaml])
else
role_details[:cloud_config_yaml] = Stack.find_file(config, 'cloud-config.yaml')
@@ -388,28 +388,32 @@
# (cwd is calculated relative to stackhome)
if role_details[:post_install_cwd].nil?
role_details[:post_install_cwd] = '/.'
end
+ if role_details[:post_install_args].nil?
+ role_details[:post_install_args] = ''
+ end
+
(1..role_details[:count]).each do |p|
Logger.debug { "Populating the config[:role_details][:azs] array with AZ" }
role_details[:azs] = Array.new if role_details[:azs].nil?
-
+
# is there an az set for this node?
if role_details[:azs][p-1].nil?
- # inherit the global az
+ # inherit the global az
Logger.debug { "Inheriting the AZ for #{role} (#{config['REGION']})" }
role_details[:azs][p-1] = config['REGION']
end
-
+
# add this AZ to the AZ list, we'll dedupe later
config[:azs] << role_details[:azs][p-1]
-
+
hostname = Stack.generate_hostname(config, role, p)
Logger.debug { "Setting node_details for #{hostname}, using element #{p}-1 from #{role_details[:azs]}" }
config[:node_details][hostname] = { :az => role_details[:azs][p-1], :region => role_details[:azs][p-1], :role => role }
- end
+ end
end
end
config[:azs].uniq!
# if set the region specific settings from the global settings if not already specified
@@ -427,22 +431,22 @@
# return a hash where key is the instance name, value is another hash containing :region, :id, :addresses
def Stack.get_all_instances(config, refresh = false)
if config[:all_instances].nil? || refresh
# we need to get the server list for each AZ mentioned in the config[:roles][:role][:azs], this is populated by Stack.populate_config
Stack.populate_config(config)
-
+
# get the current list of servers from OpenStack & generate a hash, keyed on name
servers = Hash.new
config[:azs].each do |az|
os = Stack.connect(config, az)
- os.servers.each do |server|
- servers[server[:name]] = {
+ os.servers.each do |server|
+ servers[server[:name]] = {
:region => az,
- :id => server[:id],
+ :id => server[:id],
:addresses => os.server(server[:id]).addresses
}
- end
+ end
end
config[:all_instances] = servers
end
config[:all_instances]
end
@@ -458,13 +462,13 @@
# Add an instance to the :all_instances hash, instead of having to poll the whole lot again
def Stack.add_instance(config, hostname, region, id, addresses)
config[:all_instances][hostname] = { :region => region, :id => id, :addresses => addresses}
end
-
+
def Stack.ssh(config, hostname = nil, user = ENV['USER'], command = nil)
- # ssh to a host, or all hosts
+ # ssh to a host, or all hosts
# get all running instances
servers = Stack.get_our_instances(config)
if hostname.nil?
@@ -521,35 +525,35 @@
def Stack.delete_all(config)
# this also populates out unspecified defaults, like az
Stack.populate_config(config)
- # get the list of nodes we consider 'ours', i.e. with hostnames that match
+ # get the list of nodes we consider 'ours', i.e. with hostnames that match
# those generated by this stack
ours = Stack.get_our_instances(config)
-
+
# do any of the list of servers in OpenStack match one of our hostnames?
ours.each do |node, node_details|
Logger.info "Deleting #{node}"
os = Stack.connect(config, config[:all_instances][node][:region])
d = os.get_server(config[:all_instances][node][:id])
d.delete!
end
end
-
+
def Stack.get_public_ip(config, hostname)
- # get a public address from the instance
+ # get a public address from the instance
# (could be either the dynamic or one of our floating IPs
config[:all_instances][hostname][:addresses].each do |address|
if address.label == 'public'
return address.address
end
end
end
def Stack.set_chef_server(config, chef_server)
- # set the private & public URLs for the chef server,
+ # set the private & public URLs for the chef server,
# called either after we create the Chef Server, or skip over it
Logger.debug "Setting :chef_server_hostname, chef_server_private & chef_server_public details (using #{chef_server})"
config[:chef_server_hostname] = chef_server
# get the internal IP of this instance....which we should have stored in config[:all_instances]
@@ -573,30 +577,30 @@
def Stack.secgroup_sync(config)
# 1) get all the IP information we have
# 2) generate the json to describe that to the "stackhelper secgroup-sync" tool
# 3) run "stackhelper secgroup-sync --some-file our-ips.json"
ours = Stack.get_our_instances(config)
-
+
secgroup_ips = Hash.new
# walk the list of hosts, dumping the IPs into role buckets
ours.each do |instance, instance_details|
secgroup_ips[instance_details[:role]] = Array.new if secgroup_ips[instance_details[:role]].nil?
#secgroup_ips[instance_details[:role]] << instance_details[:addresses].map { |address| address.address }
- secgroup_ips[instance_details[:role]] << instance_details[:addresses].map do |address|
- if (address.label == 'public')
- address.address
- else
- next
+ secgroup_ips[instance_details[:role]] << instance_details[:addresses].map do |address|
+ if (address.label == 'public')
+ address.address
+ else
+ next
end
end
# we potentially have an array of arrays, so flatten them
secgroup_ips[instance_details[:role]].flatten!
# delete any nil's that we collected due to skipping private ips
- secgroup_ips[instance_details[:role]].delete_if {|x| x.nil? }
+ secgroup_ips[instance_details[:role]].delete_if {|x| x.nil? }
end
# dump the json to a temp file
#sg_json = Tempfile.new(['secgroup_ips', '.json'])
sg_json = File.new('secgroup_ips.json', "w")
@@ -616,35 +620,35 @@
end
# if we're passed a role, only deploy this role.
def Stack.deploy_all(config, role_to_deploy = nil)
Stack.validate(config)
-
+
# this also populates out unspecified defaults, like az
node_details = Stack.populate_config(config)
# get info about all instances running in our account & AZs
servers = Stack.get_all_instances(config)
# this is our main loop iterator, generates each host
- config[:roles].each do |role,role_details|
+ config[:roles].each do |role,role_details|
Logger.debug { "Iterating over roles, this is #{role}, role_details = #{role_details}" }
(1..role_details[:count]).each do |p|
hostname = Stack.generate_hostname(config, role, p)
Logger.debug { "Iterating over nodes in #{role}, this is #{hostname}" }
-
+
# configure the global :chef_server details if this the chef server
if role_details[:chef_server]
Stack.set_chef_server(config, hostname)
end
# does this node already exist?
if (!servers[hostname].nil?)
Logger.info { "#{hostname} already exists, skipping.." }
next
end
-
+
Logger.debug { "Deploying #{role}, role_to_deploy = #{role_to_deploy}" }
if ((role_to_deploy.nil?) || (role_to_deploy.to_s == role.to_s))
if (role_details[:skip_chef_prereg] == true || role_details[:chef_server])
Logger.debug "Skipping Chef pre-reg for #{hostname}"
else
@@ -652,13 +656,13 @@
# 1) delete the client if it exists
knife_client_list = `knife client list | grep #{hostname}`
knife_client_list.sub!(/\s/,'')
if knife_client_list.length() > 0
# we should delete the client to make way for this new machine
- Logger.info `knife client delete --yes #{hostname}`
+ Logger.info `knife client delete --yes #{hostname}`
end
-
+
# knife node create -d --environment $CHEF_ENVIRONMENT $SERVER_NAME
# knife node run_list add -d --environment $CHEF_ENVIRONMENT $SERVER_NAME "role[${ROLE}]"
# this relies on .chef matching the stacks config (TODO: poke the Chef API directly?)
cmd = "EDITOR=\"perl -p -i -e 's/_default/#{config[:chef_environment]}/'\" knife node create --server-url #{config[:chef_server_public]} #{hostname}"
Logger.debug cmd
@@ -678,44 +682,51 @@
# (use config[:stackhome] for "project" config/scripts)
libdir = File.realpath(@@gemhome + '/lib')
multipart_cmd = "#{libdir}/write-mime-multipart #{role_details[:bootstrap]} #{role_details[:cloud_config_yaml]}"
Logger.debug { "multipart_cmd = #{multipart_cmd}" }
multipart = `#{multipart_cmd}`
+ Logger.debug { "multipart = #{multipart}" }
# 2) replace the tokens (CHEF_SERVER, CHEF_ENVIRONMENT, SERVER_NAME, ROLE)
+ Logger.debug { "Replacing %HOSTNAME% with #{hostname} in multipart" }
multipart.gsub!(%q!%HOSTNAME%!, hostname)
- Logger.info "Chef server is #{config[:chef_server_hostname]}, which is in #{config[:node_details][config[:chef_server_hostname]][:region]}"
- Logger.info "#{hostname}'s region is #{config[:node_details][hostname][:region]}"
- # if this host is in the same region/az, use the private URL, if not, use the public url
- if (config[:node_details][hostname][:region] == config[:node_details][config[:chef_server_hostname]][:region]) && !config[:chef_server_private].nil?
- multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_private])
- elsif !config[:chef_server_public].nil?
- multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_public])
+ if config[:chef_server_hostname].nil?
+ Logger.info "config[:chef_server_hostname] is nil, skipping chef server substitution"
else
- Logger.warn { "Not setting the chef url for #{hostname} as neither chef_server_private or chef_server_public are valid yet" }
+ Logger.info "Chef server is #{config[:chef_server_hostname]}, which is in #{config[:node_details][config[:chef_server_hostname]][:region]}"
+ Logger.info "#{hostname}'s region is #{config[:node_details][hostname][:region]}"
+ # if this host is in the same region/az, use the private URL, if not, use the public url
+ if (config[:node_details][hostname][:region] == config[:node_details][config[:chef_server_hostname]][:region]) && !config[:chef_server_private].nil?
+ multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_private])
+ elsif !config[:chef_server_public].nil?
+ multipart.gsub!(%q!%CHEF_SERVER%!, config[:chef_server_public])
+ else
+ Logger.warn { "Not setting the chef url for #{hostname} as neither chef_server_private or chef_server_public are valid yet" }
+ end
+ multipart.gsub!(%q!%CHEF_ENVIRONMENT%!, config[:chef_environment])
+ if File.exists?(config[:chef_validation_pem])
+ multipart.gsub!(%q!%CHEF_VALIDATION_PEM%!, File.read(config[:chef_validation_pem]))
+ else
+ Logger.warn "Skipping #{config[:chef_validation_pem]} substitution in user-data"
+ end
end
- multipart.gsub!(%q!%CHEF_ENVIRONMENT%!, config[:chef_environment])
- if File.exists?(config[:chef_validation_pem])
- multipart.gsub!(%q!%CHEF_VALIDATION_PEM%!, File.read(config[:chef_validation_pem]))
- else
- Logger.warn "Skipping #{config[:chef_validation_pem]} substitution in user-data"
- end
+
multipart.gsub!(%q!%SERVER_NAME%!, hostname)
multipart.gsub!(%q!%ROLE%!, role.to_s)
multipart.gsub!(%q!%DATA_DIR%!, role_details[:data_dir])
Logger.info "Creating #{hostname} in #{node_details[hostname][:az]} with role #{role}"
- # this will get put in /meta.js
+ # this will get put in /meta.js
metadata = { 'region' => node_details[hostname][:az], 'chef_role' => role }
os = Stack.connect(config, node_details[hostname][:az])
- newserver = os.create_server(:name => hostname,
- :imageRef => config[node_details[hostname][:az]]['image_id'],
+ newserver = os.create_server(:name => hostname,
+ :imageRef => config[node_details[hostname][:az]]['image_id'],
:flavorRef => config['flavor_id'],
:security_groups=>[role_details[:security_group]],
- :user_data => Base64.encode64(multipart),
+ :user_data => Base64.encode64(multipart),
:metadata => metadata,
:key_name => config[:key_pair])
# wait for the server to become ACTIVE before proceeding
while (newserver.status != 'ACTIVE') do
@@ -740,23 +751,23 @@
if role_details[:floating_ips] && role_details[:floating_ips][p-1]
floating_ip = role_details[:floating_ips][p-1]
Logger.info "Attaching #{floating_ip} to #{hostname}\n via 'nova add-floating-ip'"
# nova --os-region-name $REGION add-floating-ip $SERVER_NAME $FLOATING_IP
floating_ip_add = `nova --os-region-name #{node_details[hostname][:az]} add-floating-ip #{hostname} #{floating_ip}`
- Logger.info floating_ip_add
- end
-
+ Logger.info floating_ip_add
+ end
+
# refresh the secgroups ASAP
Stack.secgroup_sync(config)
# run any post-install scripts, these are run from the current host, not the nodes
if role_details[:post_install_script]
Logger.debug { "This role has a post-install script (#{role_details[:post_install_script]}), preparing to run" }
# convert when we got passed to an absolute path
post_install_script_abs = File.realpath(config[:stackhome] + '/' + role_details[:post_install_script])
post_install_cwd_abs = File.realpath(config[:stackhome] + '/' + role_details[:post_install_cwd])
-
+
# replace any tokens in the argument
public_ip = Stack.get_public_ip(config, hostname)
role_details[:post_install_args].sub!(%q!%PUBLIC_IP%!, public_ip)
# we system this, so that the script can give live feed back
@@ -764,19 +775,19 @@
system("cd #{post_install_cwd_abs} ; #{post_install_script_abs} #{role_details[:post_install_args]}")
end
else
Logger.info "Skipped role #{role}"
end
- end
+ end
end
end
def Stack.find_file(config, filename)
# find a file, using the standard path precedence
# 1) cwd
# 2) stackhome
# 3) gemhome/lib
- dirs = [ './' ]
+ dirs = [ './' ]
dirs.push(config[:stackhome])
dirs.push(@@gemhome + '/lib')
Logger.debug "find_file, looking for #{filename} in #{dirs}"
filename_fqp = ''