module Rudy
class UnknownInstance < RuntimeError; end
end
module Rudy
module Command
class NoCred < RuntimeError; end;
class Base < Drydock::Command
attr_reader :scm
attr_reader :rscripts
attr_reader :domains
attr_reader :machine_images
attr_reader :config
def init
raise "PRODUCTION ACCESS IS DISABLED IN DEBUG MODE" if @global.environment == "prod" && Drydock.debug?
@global.config ||= RUDY_CONFIG_FILE
unless File.exists?(@global.config)
init_config_dir
end
@config = Rudy::Config.new(@global.config, {:verbose => (@global.verbose > 0)} )
@config.look_and_load
raise "There is no machine group configured" if @config.machines.nil?
raise "There is no AWS info configured" if @config.awsinfo.nil?
@global.accesskey ||= @config.awsinfo.accesskey || ENV['AWS_ACCESS_KEY']
@global.secretkey ||= @config.awsinfo.secretkey || ENV['AWS_SECRET_KEY'] || ENV['AWS_SECRET_ACCESS_KEY']
@global.account ||= @config.awsinfo.account || ENV['AWS_ACCOUNT_NUMBER']
@global.cert ||= @config.awsinfo.cert || ENV['EC2_CERT']
@global.privatekey ||= @config.awsinfo.privatekey || ENV['EC2_PRIVATE_KEY']
@global.cert = File.expand_path(@global.cert || '')
@global.privatekey = File.expand_path(@global.privatekey || '')
@global.region ||= @config.defaults.region || DEFAULT_REGION
@global.zone ||= @config.defaults.zone || DEFAULT_ZONE
@global.environment ||= @config.defaults.environment || DEFAULT_ENVIRONMENT
@global.role ||= @config.defaults.role || DEFAULT_ROLE
@global.position ||= @config.defaults.position || DEFAULT_POSITION
@global.user ||= @config.defaults.user || DEFAULT_USER
@global.local_user = ENV['USER'] || :user
@global.local_hostname = Socket.gethostname || :host
check_keys
if @global.verbose > 1
puts "GLOBALS:"
@global.marshal_dump.each_pair do |n,v|
puts "#{n}: #{v}"
end
["machines", "routines"].each do |type|
puts "#{$/*2}#{type.upcase}:"
val = @config.send(type).find_deferred(@global.environment, @global.role)
puts val.to_hash.to_yaml
end
puts
end
# TODO: enforce home directory permissions
#if File.exists?(RUDY_CONFIG_DIR)
# puts "Checking #{check_environment} permissions..."
#end
if has_keys?
@ec2 = Rudy::AWS::EC2.new(@global.accesskey, @global.secretkey)
@sdb = Rudy::AWS::SimpleDB.new(@global.accesskey, @global.secretkey)
#@s3 = Rudy::AWS::SimpleDB.new(@global.accesskey, @global.secretkey)
end
end
protected :init
def machine_data
machine_data = {
# Give the machine an identity
:zone => @global.zone,
:environment => @global.environment,
:role => @global.role,
:position => @global.position,
# Add hosts to the /etc/hosts file
:hosts => {
:dbmaster => "127.0.0.1",
}
}
machine_data.to_hash
end
# Raises exceptions if the requested user does
# not have a valid keypair configured. (See: EC2_KEYPAIR_*)
def check_keys
raise "No SSH key provided for #{@global.user}! (check #{RUDY_CONFIG_FILE})" unless has_keypair?
raise "SSH key provided but cannot be found! (check #{RUDY_CONFIG_FILE})" unless File.exists?(keypairpath)
end
def has_pem_keys?
(@global.cert && File.exists?(@global.cert) &&
@global.privatekey && File.exists?(@global.privatekey))
end
def has_keys?
(@global.accesskey && !@global.accesskey.empty? && @global.secretkey && !@global.secretkey.empty?)
end
def keypairpath(name=nil)
name ||= @global.user
raise "No default user configured" unless name
kp = @config.machines.find(@global.environment, @global.role, :users, name, :keypair2)
kp ||= @config.machines.find(@global.environment, :users, name, :keypair)
kp ||= @config.machines.find(:users, name, :keypair)
kp &&= File.expand_path(kp)
kp
end
def has_keypair?(name=nil)
kp = keypairpath(name)
(!kp.nil? && File.exists?(kp))
end
# Opens an SSH session.
#
+host+ the hostname to connect to. Defaults to the machine specified
# by @global.environment, @global.role, @global.position.
# +b+ a block to execute on the host. Receives |session|
#
# ssh do |session|
# session.exec(cmd)
# end
#
# See Net::SSH
#
def ssh(host=nil, &b)
host ||= machine_hostname
raise "No host provided for SSH" unless host
raise "No block provided for SSH" unless b
Net::SSH.start(host, @global.user, :keys => [keypairpath]) do |session|
b.call(session)
end
end
# Secure copy.
#
# scp do |scp|
# # upload a file to a remote server
# scp.upload! "/local/path", "/remote/path"
#
# # upload from an in-memory buffer
# scp.upload! StringIO.new("some data to upload"), "/remote/path"
#
# # run multiple downloads in parallel
# d1 = scp.download("/remote/path", "/local/path")
# d2 = scp.download("/remote/path2", "/local/path2")
# [d1, d2].each { |d| d.wait }
# end
#
def scp(host=nil, &b)
host ||= machine_hostname
raise "No host provided for scp" unless host
raise "No block provided for scp" unless b
Net::SCP.start(host, @global.user, :keys => [keypairpath]) do |scp|
b.call(scp)
end
end
# +name+ the name of the remote user to use for the remainder of the command
# (or until switched again). If no name is provided, the user will be revert
# to whatever it was before the previous switch.
def switch_user(name=nil)
if name == nil && @switch_user_previous
@global.user = @switch_user_previous
elsif @global.user != name
puts "Remote commands will be run as #{name} user"
@switch_user_previous = @global.user
@global.user = name
end
end
# Returns a hash of info for the requested machine. If the requested machine
# is not running, it will raise an exception.
def find_current_machine
find_machine(machine_group)
end
def find_machine(group)
machine_list = @ec2.instances.list(group)
machine = machine_list.values.first # NOTE: Only one machine per group, for now...
raise "There's no machine running in #{group}" unless machine
raise "The primary machine in #{group} is not in a running state" unless machine[:aws_state] == 'running'
machine
end
def machine_hostname(group=nil)
group ||= machine_group
find_machine(group)[:dns_name]
end
def machine_group
[@global.environment, @global.role].join(RUDY_DELIM)
end
def machine_image
ami = @config.machines.find_deferred(@global.environment, @global.role, :ami)
raise "There is no AMI configured for #{machine_group}" unless ami
ami
end
def machine_address
@config.machines.find_deferred(@global.environment, @global.role, :address)
end
# TODO: fix machine_group to include zone
def machine_name
[@global.zone, machine_group, @global.position].join(RUDY_DELIM)
end
def instance_id?(id=nil)
(id && id[0,2] == "i-")
end
def image_id?(id=nil)
(id && id[0,4] == "ami-")
end
def volume_id?(id=nil)
(id && id[0,4] == "vol-")
end
def snapshot_id?(id=nil)
(id && id[0,5] == "snap-")
end
def wait_for_machine(id)
print "Waiting for #{id} to become available"
STDOUT.flush
while @ec2.instances.pending?(id)
sleep 2
print '.'
STDOUT.flush
end
machine = @ec2.instances.get(id)
puts " It's up!\a\a" # with bells
print "Waiting for SSH daemon at #{machine[:dns_name]}"
STDOUT.flush
while !Rudy::Utils.service_available?(machine[:dns_name], 22)
print '.'
STDOUT.flush
end
puts " It's up!\a\a\a"
end
def device_to_path(machine, device)
# /dev/sdr 10321208 154232 9642688 2% /rilli/app
dfoutput = ssh_command(machine[:dns_name], keypairpath, @global.user, "df #{device} | tail -1").chomp
dfvals = dfoutput.scan(/(#{device}).+\s(.+?)$/).flatten # ["/dev/sdr", "/rilli/app"]
dfvals.last
end
# +action+ is one of: :shutdown, :start, :deploy
# +machine+ is a right_aws machine instance hash
def execute_disk_routines(machines, action)
machines = [machines] unless machines.is_a?( Array)
puts "Running #{action.to_s.capitalize} DISK routines".att(:bright)
disks = @config.machines.find_deferred(@global.environment, @global.role, :disks)
routines = @config.routines.find(@global.environment, @global.role, action, :disks)
unless routines
puts "No #{action} disk routines for #{machine_group}"
return
end
switch_user("root")
machines.each do |machine|
unless machine[:aws_instance_id]
puts "Machine given has no instance ID. Skipping disks."
return
end
unless machine[:dns_name]
puts "Machine given has no DNS name. Skipping disks."
return
end
if routines.destroy
disk_paths = routines.destroy.keys
vols = @ec2.instances.volumes(machine[:aws_instance_id]) || []
puts "No volumes to destroy for (#{machine[:aws_instance_id]})" if vols.empty?
vols.each do |vol|
disk = Rudy::MetaData::Disk.find_from_volume(@sdb, vol[:aws_id])
if disk
this_path = disk.path
else
puts "No disk metadata for volume #{vol[:aws_id]}. Going old school..."
this_path = device_to_path(machine, vol[:aws_device])
end
dconf = disks[this_path]
unless dconf
puts "#{this_path} is not defined for this machine. Check your machines config."
next
end
if disk_paths.member?(this_path)
unless disks.has_key?(this_path)
puts "#{this_path} is not defined as a machine disk. Skipping..."
next
end
begin
puts "Unmounting #{this_path}..."
ssh_command machine[:dns_name], keypairpath, @global.user, "umount #{this_path}"
sleep 3
rescue => ex
puts "Error while unmounting #{this_path}: #{ex.message}"
puts ex.backtrace if Drydock.debug?
puts "We'll keep going..."
end
begin
if @ec2.volumes.attached?(disk.awsid)
puts "Detaching #{vol[:aws_id]}"
@ec2.volumes.detach(vol[:aws_id])
sleep 3 # TODO: replace with something like wait_for_machine
end
puts "Destroying #{this_path} (#{vol[:aws_id]})"
if @ec2.volumes.available?(disk.awsid)
@ec2.volumes.destroy(vol[:aws_id])
else
puts "Volume is still attached (maybe a web server of database is running?)"
end
if disk
puts "Deleteing metadata for #{disk.name}"
Rudy::MetaData::Disk.destroy(@sdb, disk)
end
rescue => ex
puts "Error while detaching volume #{vol[:aws_id]}: #{ex.message}"
puts ex.backtrace if Drydock.debug?
puts "Continuing..."
end
end
puts
end
end
if routines.mount
disk_paths = routines.mount.keys
vols = @ec2.instances.volumes(machine[:aws_instance_id]) || []
puts "No volumes to mount for (#{machine[:aws_instance_id]})" if vols.empty?
vols.each do |vol|
disk = Rudy::MetaData::Disk.find_from_volume(@sdb, vol[:aws_id])
if disk
this_path = disk.path
else
puts "No disk metadata for volume #{vol[:aws_id]}. Going old school..."
this_path = device_to_path(machine, vol[:aws_device])
end
next unless disk_paths.member?(this_path)
dconf = disks[this_path]
unless dconf
puts "#{this_path} is not defined for this machine. Check your machines config."
next
end
begin
unless @ec2.instances.attached_volume?(machine[:aws_instance_id], vol[:aws_device])
puts "Attaching #{vol[:aws_id]} to #{machine[:aws_instance_id]}".att(:bright)
@ec2.volumes.attach(machine[:aws_instance_id], vol[:aws_id],vol[:aws_device])
sleep 3
end
puts "Mounting #{this_path} to #{vol[:aws_device]}".att(:bright)
ssh_command machine[:dns_name], keypairpath, @global.user, "mkdir -p #{this_path} && mount -t ext3 #{vol[:aws_device]} #{this_path}"
sleep 1
rescue => ex
puts "There was an error mounting #{this_path}: #{ex.message}"
puts ex.backtrace if Drydock.debug?
end
puts
end
end
if routines.restore
routines.restore.each_pair do |path,props|
from = props[:from] || "unknown"
unless from.to_s == "backup"
puts "Sorry! You can currently only restore from backup. Check your routines config."
next
end
begin
puts "Restoring disk for #{path}"
dconf = disks[path]
unless dconf
puts "#{path} is not defined for this machine. Check your machines config."
next
end
zon = props[:zone] || @global.zone
env = props[:environment] || @global.environment
rol = props[:role] || @global.role
pos = props[:position] || @global.position
puts "Looking for backup from #{zon}-#{env}-#{rol}-#{pos}"
backup = find_most_recent_backup(zon, env, rol, pos, path)
unless backup
puts "No backups found"
next
end
puts "Found: #{backup.name}".att(:bright)
disk = Rudy::MetaData::Disk.new
disk.path = path
[:region, :zone, :environment, :role, :position].each do |n|
disk.send("#{n}=", @global.send(n)) if @global.send(n)
end
disk.device = dconf[:device]
size = (backup.size.to_i > dconf[:size].to_i) ? backup.size : dconf[:size]
disk.size = size.to_i
if Rudy::MetaData::Disk.is_defined?(@sdb, disk)
puts "The disk #{disk.name} already exists."
puts "You probably need to define when to destroy the disk."
puts "Skipping..."
next
end
if @ec2.instances.attached_volume?(machine[:aws_instance_id], disk.device)
puts "Skipping disk for #{disk.path} (device #{disk.device} is in use)"
next
end
# NOTE: It's important to use Caesars' hash syntax b/c the disk property
# "size" conflicts with Hash#size which is what we'll get if there's no
# size defined.
unless disk.size.kind_of?(Integer)
puts "Skipping disk for #{disk.path} (size not defined)"
next
end
if disk.path.nil?
puts "Skipping disk for #{disk.path} (no path defined)"
next
end
unless disk.valid?
puts "Skipping #{disk.name} (not enough info)"
next
end
puts "Creating volume... (from #{backup.awsid})".att(:bright)
volume = @ec2.volumes.create(@global.zone, disk.size, backup.awsid)
puts "Attaching #{volume[:aws_id]} to #{machine[:aws_instance_id]}".att(:bright)
@ec2.volumes.attach(machine[:aws_instance_id], volume[:aws_id], disk.device)
sleep 3
puts "Mounting #{disk.device} to #{disk.path}".att(:bright)
ssh_command machine[:dns_name], keypairpath, @global.user, "mkdir -p #{disk.path} && mount -t ext3 #{disk.device} #{disk.path}"
puts "Creating disk metadata for #{disk.name}"
disk.awsid = volume[:aws_id]
Rudy::MetaData::Disk.save(@sdb, disk)
sleep 1
rescue => ex
puts "There was an error creating #{path}: #{ex.message}"
puts ex.backtrace if Drydock.debug?
if disk
puts "Removing metadata for #{disk.name}"
Rudy::MetaData::Disk.destroy(@sdb, disk)
end
end
puts
end
end
if routines.create
routines.create.each_pair do |path,props|
begin
puts "Creating disk for #{path}"
dconf = disks[path]
unless dconf
puts "#{path} is not defined for this machine. Check your machines config."
next
end
disk = Rudy::MetaData::Disk.new
disk.path = path
[:region, :zone, :environment, :role, :position].each do |n|
disk.send("#{n}=", @global.send(n)) if @global.send(n)
end
[:device, :size].each do |n|
disk.send("#{n}=", dconf[n]) if dconf.has_key?(n)
end
if Rudy::MetaData::Disk.is_defined?(@sdb, disk)
puts "The disk #{disk.name} already exists."
puts "You probably need to define when to destroy the disk."
puts "Skipping..."
next
end
if @ec2.instances.attached_volume?(machine[:aws_instance_id], disk.device)
puts "Skipping disk for #{disk.path} (device #{disk.device} is in use)"
next
end
# NOTE: It's important to use Caesars' hash syntax b/c the disk property
# "size" conflicts with Hash#size which is what we'll get if there's no
# size defined.
unless disk.size.kind_of?(Integer)
puts "Skipping disk for #{disk.path} (size not defined)"
next
end
if disk.path.nil?
puts "Skipping disk for #{disk.path} (no path defined)"
next
end
unless disk.valid?
puts "Skipping #{disk.name} (not enough info)"
next
end
puts "Creating volume... (#{disk.size}GB in #{@global.zone})".att(:bright)
volume = @ec2.volumes.create(@global.zone, disk.size)
puts "Attaching #{volume[:aws_id]} to #{machine[:aws_instance_id]}".att(:bright)
@ec2.volumes.attach(machine[:aws_instance_id], volume[:aws_id], disk.device)
sleep 6
puts "Creating the filesystem (mkfs.ext3 -F #{disk.device})".att(:bright)
ssh_command machine[:dns_name], keypairpath, @global.user, "mkfs.ext3 -F #{disk.device}"
sleep 3
puts "Mounting #{disk.device} to #{disk.path}".att(:bright)
ssh_command machine[:dns_name], keypairpath, @global.user, "mkdir -p #{disk.path} && mount -t ext3 #{disk.device} #{disk.path}"
puts "Creating disk metadata for #{disk.name}"
disk.awsid = volume[:aws_id]
Rudy::MetaData::Disk.save(@sdb, disk)
sleep 1
rescue => ex
puts "There was an error creating #{path}: #{ex.message}"
if disk
puts "Removing metadata for #{disk.name}"
Rudy::MetaData::Disk.destroy(@sdb, disk)
end
end
puts
end
end
end
end
def find_most_recent_backup(zon, env, rol, pos, path)
criteria = [zon, env, rol, pos, path]
(Rudy::MetaData::Backup.list(@sdb, *criteria) || []).first
end
def execute_routines(machines, action, before_or_after)
machines = [machines] unless machines.is_a?( Array)
config = @config.routines.find_deferred(@global.environment, @global.role, :config) || {}
config[:global] = @global.marshal_dump
config[:global].reject! { |n,v| n == :cert || n == :privatekey }
# The config file contains settings from ~/.rudy/config
#
# routines do
# config do
# end
# end
#
config_file = "#{action}-config.yaml"
tf = Tempfile.new(config_file)
write_to_file(tf.path, config.to_hash.to_yaml, 'w')
puts "Running #{action.to_s.capitalize} #{before_or_after.to_s.upcase} routines".att(:bright)
machines.each do |machine|
puts "Machine Group: #{machine_group}"
puts "Hostname: #{machine[:dns_name]}"
rscripts = @config.routines.find_deferred(@global.environment, @global.role, action, before_or_after) || []
rscripts = [rscripts] unless rscripts.is_a?(Array)
puts "No scripts defined." if !rscripts || rscripts.empty?
rscripts.each do |rscript|
user, script = rscript.shift
switch_user(user) # scp and ssh will run as this user
puts "Transfering #{config_file}..."
scp do |scp|
scp.upload!(tf.path, "~/#{config_file}") do |ch, name, sent, total|
"#{name}: #{sent}/#{total}"
end
end
ssh do |session|
puts "Running #{script}...".att(:bright)
session.exec!("chmod 700 ~/#{config_file}")
session.exec!("chmod 700 #{script}")
puts session.exec!("#{script}")
puts "Removing remote copy of #{config_file}..."
session.exec!("rm ~/#{config_file}")
end
puts $/
end
end
tf.delete # remove local copy of config_file
#switch_user # return to the requested user
end
# Print a default header to the screen for every command.
# +cmd+ is the name of the command current running.
def print_header(cmd=nil)
title = "RUDY v#{Rudy::VERSION}" unless @global.quiet
now_utc = Time.now.utc.strftime("%Y-%m-%d %H:%M:%S")
criteria = []
[:zone, :environment, :role, :position].each do |n|
val = @global.send(n)
next unless val
criteria << "#{n.to_s.slice(0,1).att :normal}:#{val.att :bright}"
end
puts '%s -- %s UTC' % [title, now_utc] unless @global.quiet
puts '[%s]' % criteria.join(" ") unless @global.quiet
puts unless @global.quiet
if (@global.environment == "prod")
msg = without_indent %q(
=======================================================
=======================================================
!!!!!!!!! YOU ARE PLAYING WITH PRODUCTION !!!!!!!!!
=======================================================
=======================================================)
puts msg.colour(:red).bgcolour(:white).att(:bright), $/ unless @global.quiet
end
if Rudy.in_situ?
msg = %q(============ THIS IS EC2 ============)
puts msg.colour(:blue).bgcolour(:white).att(:bright), $/ unless @global.quiet
end
end
def print_footer
end
def group_metadata(env=@global.environment, role=@global.role)
query = "['environment' = '#{env}'] intersection ['role' = '#{role}']"
@sdb.query_with_attributes(RUDY_DOMAIN, query)
end
private
# Print info about a running instance
# +inst+ is a hash
def print_instance(inst)
puts '-'*60
puts "Instance: #{inst[:aws_instance_id].att(:bright)} (AMI: #{inst[:aws_image_id]})"
[:aws_state, :dns_name, :private_dns_name, :aws_availability_zone, :aws_launch_time, :ssh_key_name].each do |key|
printf(" %22s: %s#{$/}", key, inst[key]) if inst[key]
end
printf(" %22s: %s#{$/}", 'aws_groups', inst[:aws_groups].join(', '))
puts
end
def print_image(img)
puts '-'*60
puts "Image: #{img[:aws_id].att(:bright)}"
img.each_pair do |key, value|
printf(" %22s: %s#{$/}", key, value) if value
end
puts
end
def print_disk(disk, backups=[])
puts '-'*60
puts "Disk: #{disk.name.att(:bright)}"
puts disk.to_s
puts "#{backups.size} most recent backups:", backups.collect { |back| "#{back.nice_time} (#{back.awsid})" }
puts
end
def print_volume(vol, disk)
puts '-'*60
puts "Volume: #{vol[:aws_id].att(:bright)} (disk: #{disk.name if disk})"
vol.each_pair do |key, value|
printf(" %22s: %s#{$/}", key, value) if value
end
puts
end
# Print info about a a security group
# +group+ is an OpenStruct
def print_group(group)
puts '-'*60
puts "%12s: %s" % ['GROUP', group[:aws_group_name].att(:bright)]
puts
group_ip = {}
group[:aws_perms].each do |perm|
(group_ip[ perm[:cidr_ips] ] ||= []) << "#{perm[:protocol]}/#{perm[:from_port]}-#{perm[:to_port]}"
end
puts "%22s %s" % ["source address/mask", "protocol/ports (from, to)"]
group_ip.each_pair do |ip, perms|
puts "%22s %s" % [ip, perms.shift]
perms.each do |perm|
puts "%22s %s" % ['', perm]
end
puts
end
end
def init_config_dir
unless File.exists?(RUDY_CONFIG_DIR)
puts "Creating #{RUDY_CONFIG_DIR}"
Dir.mkdir(RUDY_CONFIG_DIR, 0700)
end
unless File.exists?(RUDY_CONFIG_FILE)
puts "Creating #{RUDY_CONFIG_FILE}"
rudy_config = without_indent %Q{
# Amazon Web Services
# Account access indentifiers.
awsinfo do
account ""
accesskey ""
secretkey ""
privatekey "~/path/2/pk-xxxx.pem"
cert "~/path/2/cert-xxxx.pem"
end
# Machine Configuration
# Specify your private keys here. These can be defined globally
# or by environment and role like in machines.rb.
machines do
users do
root :keypair => "path/2/root-private-key"
end
end
# Routine Configuration
# Define stuff here that you don't want to be stored in version control.
routines do
config do
# ...
end
end
# Global Defaults
# Define the values to use unless otherwise specified on the command-line.
defaults do
region "us-east-1"
zone "us-east-1b"
environment "stage"
role "app"
position "01"
user ENV['USER']
end
}
write_to_file(RUDY_CONFIG_FILE, rudy_config, 'w')
end
#puts "Creating SimpleDB domain called #{RUDY_DOMAIN}"
#@sdb.domains.create(RUDY_DOMAIN)
end
end
end
end