require 'rubygems' require 'json' require 'restclient' require 'ostruct' require 'ohai' require 'erb' ## IDEAS ## firewall until ready ## ruby 1.9 compatibility class Kuzushi def initialize(url) @base_url = File.dirname(url) @name = File.basename(url) @config_names = [] @configs = [] @packages = [] @tasks = [] load_config_stack(@name) @config = @configs.reverse.inject({}) { |i,c| i.merge(c) } end def init @init = true start end def start process_stack puts "----" @tasks.each do |t| puts "TASK: #{t[:description]}" t[:blk].call end puts "----" end protected def system ohai = Ohai::System.new ohai.all_plugins ohai end def load_config_stack(name) @config_names << name @configs << JSON.parse(RestClient.get("#{@base_url}/#{name}")) if import = @configs.last["import"] load_config_stack(import) end end def process_stack script get("before") process :packages process :local_packages process :gems process :volumes process :files process :users process :crontab script get("after") script get("init") if init? end ## magic goes here def process(type) ## if the file takes no args - just call it once if method("process_#{type}").arity == 0 send("process_#{type}") else ## else call it once per item get_array(type).each do |item| script item["before"] if item.is_a? Hash send("process_#{type}", OpenStruct.new(item)) else send("process_#{type}", item) end script item["after"] script item["init"] if init? end end end def process_packages @packages = get_array("packages") task "install packages" do shell "apt-get update && apt-get upgrade -y" shell "apt-get install -y #{@packages.join(" ")}" unless @packages.empty? end end def process_local_packages(p) package(p) do |file| task "install local package #{p}" do shell "dpkg -i #{file}" end end end def process_gems(gem) task "install gem #{gem}" do shell "gem install #{gem} --no-rdoc --no-ri" end end def process_volumes(v) handle_ebs v if v.media == "ebs" handle_raid v if v.media == "raid" set_readahead v if v.readahead set_scheduler v if v.scheduler handle_format v if v.format handle_mount v if v.mount end def handle_ebs(v) task "wait for volume #{v.device}" do wait_for_volume v.device end end def handle_raid(r) task "create raid #{r.device}", :init => true do shell "mdadm --create #{r.device} -n #{r.drives.size} -l #{r.level} -c #{r.chunksize || 64} #{r.drives.join(" ")}" end task "assemble raid #{r.device}" do ## assemble fails a lot with device busy - is udev to blame :( if not dev_exists? r.device shell "service stop udev" shell "mdadm --assemble #{r.device} #{r.drives.join(" ")}" shell "service start udev" end end add_package "mdadm" end def mount_options(m) o = [] o << m.options if m.options o << "size=#{m.size}M" if m.size and m.media == "tmpfs" o << "mode=#{m.mode}" if m.mode o << "noatime" if o.empty? o.join(",") end def handle_mount(m) task "mount #{m.mount}" do unless mounted?(m.mount) shell "mv #{m.mount} #{m.mout}.old" if File.exists?(m.mount) shell "mkdir -p #{m.mount} && mount -o #{mount_options(m)} -t #{m.format || m.media} #{m.device || m.media} #{m.mount}" shell "chown -R #{m.user}:#{m.group} #{m.mount}" if m.user or m.group end end end def system_arch system.kernel["machine"] end def mounted?(mount) ## cant use ohai here b/c it mashes drives together with none or tmpfs devices mount = mount.chop if mount =~ /\/$/ !!(File.read("/proc/mounts") =~ / #{mount} /) end def package_arch `dpkg --print-architecture`.chomp end def process_files(f) if f.template write_file("/templates/#{f.template}", f.file) do |file| @system = system t = ERB.new File.read(file), 0, '<>' t.result(binding) end else src = f.source || File.basename(f.file) write_file("/files/#{src}", f.file) do |file| File.read(file) end end end def write_file(src, dest, &blk) fetch(src) do |file| FileUtils.mkdir_p(File.dirname(dest)) File.open(dest,"w") { |f| f.write(blk.call(file)) } end end def handle_crontab(src, user = "root", &blk) ## FIXME - this is getting a little silly calling tmpfile twice - should be able to pass erb to fetch as an opton... ## unify code for process crontabs + process files fetch(src) do |file| tmpfile(blk.call(file)) do |tmp| task "process crontab for #{user}" do shell "crontab -u #{user} #{tmp}" end end end end def process_crontab(cron) if cron.template handle_crontab("/templates/#{cron.template}", cron.user) do |file| @system = system t = ERB.new File.read(file), 0, '<>' t.result(binding) end else src = cron.source || File.basename(cron.file) handle_crontab("/files/#{src}", cron.user) do |file| File.read(file) end end end def process_users(user) (user.authorized_keys || []).each do |key| task "add authorized_key for user #{user.name}" do shell "su - #{user.name} -c 'mkdir -p .ssh; echo \"#{key}\" >> .ssh/authorized_keys; chmod -R 0600 .ssh'" end end end def set_readahead(v) task "set readahead for #{v.device}" do shell "blockdev --setra #{v.readahead} #{v.device}" end end def set_scheduler(v) task "set scheduler for #{v.device}" do shell "echo #{v.scheduler} > /sys/block/#{File.basename(v.device)}/queue/scheduler" end end def handle_format(v) return if v.format == "tmpfs" task "formatting #{v.device}", :init => true do label = "-L " + v.label rescue "" shell "mkfs.#{v.format} #{label} #{v.device}" unless v.mount && mounted?(v.mount) end add_package "xfsprogs" if v.format == "xfs" end def add_package(p) @packages << p unless @packages.include? p end def package(p, &block) fetch("/packages/#{p}_#{package_arch}.deb") do |file| block.call(file) end end def inline_script(script) tmpfile(script) do |tmp| task "run inline script" do shell "#{tmp}" end end end def script(script) return if script.nil? return inline_script(script) if script =~ /^#!/ fetch("/scripts/#{script}") do |file| task "run script #{script}" do shell "#{file}" end end end def tmpfile(content, file = "tmp_#{rand(1_000_000_000)}", &block) tmp_dir = "/tmp/kuzushi" Dir.mkdir(tmp_dir) unless File.exists?(tmp_dir) file = "#{tmp_dir}/#{File.basename(file)}" File.open(file,"w") do |f| f.write(content) f.chmod(0700) end if content block.call(file) if block file end def fetch(file, &block) names = @config_names.clone begin ## its important that we try each name for the script - allows for polymorphic scripts tmpfile RestClient.get("#{@base_url}/#{names.first}#{file}"), file do |tmp| block.call(tmp) end rescue RestClient::ResourceNotFound names.shift retry unless names.empty? error("file not found: #{file}") rescue Object => e error("error fetching file: #{names.first}/#{file}", e) end end def error(message, exception = nil) puts "ERROR :#{message}" end def get(key) @config[key.to_s] end def get_array(key) [ get(key) || [] ].flatten end def wait_for_volume(vol) ## Maybe use ohai here instead -- FIXME until dev_exists? vol do puts "waiting for volume #{vol}" sleep 2 end end def shell(cmd) puts "# #{cmd}" puts Kernel.system cmd ## FIXME - need to handle/report exceptions here end def init? @init ||= false end def task(description, options = {}, &blk) return if options[:init] and not init? @tasks << { :description => description, :blk => blk } end def dev_exists?(dev) File.exists?("/sys/block/#{File.basename(dev)}") end end