require 'pkg-wizard/command' require 'pkg-wizard/rpm' require 'pkg-wizard/logger' require 'pkg-wizard/git' require 'pkg-wizard/mock' require 'pkg-wizard/utils' require 'tmpdir' require 'fileutils' require 'uri' require 'sinatra/base' require 'rufus/scheduler' require 'term/ansicolor' require 'pp' require 'yaml' require 'daemons' require 'singleton' module FakeColor def red; "#{self}"; end def blue; "#{self}"; end def yellow; "#{self}"; end def green; "#{self}"; end def bold; "#{self}"; end end module PKGWizard class NodeRunner @@logfile = '/dev/null' def self.logfile=(logfile) @@logfile = logfile end def self.available? not `which node`.strip.chomp.empty? end def self.kill Process.kill 15, @@proc.pid end def self.run public_dir = 'public' if not defined? @@proc puts '* starting NODE.JS...' @@proc = IO.popen("node public/server.js #{@@logfile}") end end end class BuildBotConfig include Singleton attr_accessor :mock_profile def initialize @mock_profile = "epel-5-x86_64" end end class BuildBot < Command registry << { :name => 'build-bot', :klass => self } option :help, :short => "-h", :long => "--help", :description => "Show this message", :on => :tail, :boolean => true, :show_options => true, :exit => 0 option :mock_profile, :short => '-m PROF', :long => '--mock-profile PROF', :description => 'Default Mock Profile' option :port, :short => '-p PORT', :long => '--port PORT', :default => 4567 option :daemonize, :long => '--daemonize', :default => false option :working_dir, :long => '--working-dir DIR' # not implemented option :log_format, :long => '--log-format FMT', :description => 'Log format to use (web, cli)', :default => 'cli' option :log_server_port, :long => '--log-server-port PORT', :description => 'log server port (60001 default)', :default => '60001' class Webapp < Sinatra::Base def find_job_path(name) (Dir["failed/job_*"] + Dir["success/job_*"]).find { |j| File.basename(j) == name } end get '/' do File.read(File.join(File.dirname(__FILE__), '/../../../resources/public/build-bot/index.html')) end post '/tag/:name' do name = params[:name] profile = params[:mock_profile] meta = { :name => name, :mock_profile => profile } if name.nil? or name.strip.chomp.empty? status 400 'Invalid tag' else File.open('tags/.tag', 'w') do |f| f.puts meta.to_yaml end "Tagging #{name}..." end end post '/createrepo' do FileUtils.touch 'repo/.createrepo' end post '/createsnapshot' do FileUtils.touch 'snapshot/.createsnapshot' end post '/job/clean' do dir = params[:dir] if dir == 'output' FileUtils.touch 'output/.clean' elsif dir == 'failed' FileUtils.touch 'failed/.clean' elsif dir.nil? FileUtils.touch 'output/.clean' FileUtils.touch 'failed/.clean' else $stderr.puts "WARNING: job/clean Unknown dir #{dir}. Ignoring." end end post '/build/' do pkg = params[:pkg] build_profile = params[:mock_profile] || BuildBotConfig.instance.mock_profile metadata = { :mock_profile => build_profile } if pkg.nil? Logger.instance.error '400: Invalid arguments. Needs pkg in post request' status 400 "Missing pkg parameter.\n" else incoming_file = "incoming/#{pkg[:filename]}" puts "* incoming file".ljust(40) + "#{pkg[:filename]}" FileUtils.cp pkg[:tempfile].path, incoming_file File.open("incoming/#{pkg[:filename]}.metadata", 'w') do |f| f.puts metadata.to_yaml end end end # log get '/log' do if NodeRunner.available? NodeRunner.run sleep 0.5 index = 'public/index.html' File.read index else 'node.js is not installed: Real time logs disabled :(' end end # list failed pkgs get '/job/failed' do max = params[:max] || 10 # Find failed jobs jobs = (Dir["failed/job_*"].sort { |a,b| a <=> b }).map { |j| File.basename(j) } # format job as job_XXXX_XXX (pkgname) jobs = jobs.map { |j| "#{j} (#{File.basename(Dir["failed/#{j}/*.src.rpm"].first(), '.src.rpm')})" } max = max.to_i if jobs.size > max.to_i jobs[- max..-1].to_yaml else jobs.to_yaml end end get '/server/stats' do fs = PKGWizard::Utils.filesystem_status { :filesystem => fs, }.to_yaml end get '/job/stats' do fjobs = Dir["failed/job*"].size snapshots = Dir["snapshot/snapshot*"].size sjobs = Dir["output/job*"].size qjobs = Dir["incoming/*.src.rpm"].size cjobs = Dir["workspace/job_*"].size total_jobs = fjobs + sjobs { :failed_jobs => fjobs, :successful_jobs => sjobs, :enqueued => qjobs, :total_jobs => total_jobs, :snapshots => snapshots, :building => cjobs }.to_yaml end # list successfully built pkgs get '/job/successful' do max = params[:max] || 10 jobs = (Dir["output/job_*"].sort { |a,b| a <=> b }).map { |j| File.basename(j) } max = max.to_i if jobs.size > max.to_i jobs[- max..-1].to_yaml else jobs.to_yaml end end # # Rebuild a previous job (It can be either successful or failed build) # post '/job/rebuild/:name' do name = params[:name] job = find_job_path(name) if job.nil? status 404 else puts "Rebuilding job [#{name}]".ljust(40) + File.basename(Dir["#{job}/*.rpm"].first) FileUtils.cp Dir["#{job}/*.rpm"].first, 'incoming/' FileUtils.rm_rf job end end # Get current building job (empty output if none) get '/job/current' do job = Dir['workspace/job*'].first if job File.read(job + '/meta.yml') else status 404 end end # Get job info get '/job/:name' do jname = params[:name] jobs = Dir['output/job_*'] + Dir['failed/job_*'] found = false meta = '' if jname == 'all' found = true metas = [] jobs.each do |j| mfile = j + '/meta.yml' if File.exist?(mfile) metas << YAML.load_file(mfile) else $stderr.puts "[WARNING] Meta file #{mfile} not found" end end meta = metas.to_yaml else jobs.each do |j| if File.basename(j) == jname found = true puts j + '/meta.yml' meta = File.read(j + '/meta.yml') break end end end status 404 if not found meta end end def self.perform cli = BuildBot.new cli.banner = "\nUsage: pkgwiz build-bot (options)\n\n" cli.parse_options ## Node.JS log server stuff public_dir = File.join(File.dirname(__FILE__), '/../../../resources/public/') node_port = cli.config[:log_server_port] if File.exist?('public') FileUtils.rm_rf 'public' end FileUtils.cp_r public_dir, 'public' html = File.read('public/index.html.tmpl').gsub('@@NODEJSPORT@@', node_port) serverjs = File.read('public/server.js.tmpl').gsub('@@NODEJSPORT@@', node_port) File.open 'public/index.html', 'w' do |f| f.puts html end File.open 'public/server.js', 'w' do |f| f.puts serverjs end if cli.config[:log_format] == 'web' String.class_eval do include FakeColor; end else String.class_eval do include Term::ANSIColor; end end pwd = cli.config[:working_dir] || Dir.pwd NodeRunner.logfile = (cli.config[:working_dir] || Dir.pwd) + '/build-bot.log' pwd = File.expand_path pwd if cli.config[:daemonize] umask = File.umask Daemons.daemonize :app_name => 'build-bot', :dir_mode => :normal, :dir => pwd Dir.chdir pwd log = File.new("build-bot.log", "a") $stdout.reopen(log) $stderr.reopen(log) $stdout.sync = true $stderr.sync = true File.umask umask else Dir.chdir pwd end mock_profile = cli.config[:mock_profile] BuildBotConfig.instance.mock_profile = mock_profile if not mock_profile $stderr.puts 'Invalid mock profile.' $stderr.puts cli.opt_parser.help exit 1 end if not File.exist? '/usr/bin/rpmbuild' $stderr.puts 'rpmbuild command not found. Install it first.' exit 1 end if not File.exist? '/usr/sbin/mock' $stderr.puts 'mock command not found. Install it first.' exit 1 end meta = { :mock_profile => mock_profile } Dir.mkdir 'incoming' if not File.exist?('incoming') Dir.mkdir 'output' if not File.exist?('output') Dir.mkdir 'workspace' if not File.exist?('workspace') Dir.mkdir 'archive' if not File.exist?('archive') Dir.mkdir 'failed' if not File.exist?('failed') Dir.mkdir 'snapshot' if not File.exist?('snapshot') Dir.mkdir 'tags' if not File.exist?('tags') FileUtils.ln_sf 'output', 'repo' if not File.exist?('repo') cleaner = Rufus::Scheduler.start_new cleaner.every '2s', :blocking => true do if File.exist?('failed/.clean') puts '* cleaning FAILED jobs' Dir["failed/job_*"].each do |d| FileUtils.rm_rf d end FileUtils.rm 'failed/.clean' FileUtils.rm_rf "failed/last" end if File.exist?('output/.clean') puts '* cleaning OUTPUT jobs' Dir["output/job_*"].each do |d| FileUtils.rm_rf d end FileUtils.rm 'output/.clean' FileUtils.rm_rf 'output/repodata' FileUtils.rm_rf 'output/last' end end # tag routine tag_sched = Rufus::Scheduler.start_new tag_sched.every '2s', :blocking => true do if File.exist?('tags/.tag') tag_meta = YAML.load_file 'tags/.tag' rescue nil if tag_meta tag_dir = "tags/#{tag_meta[:name]}" tag_mock_profile = tag_meta[:mock_profile] tag_pkgs = [] Dir["output/*/result/*.rpm"].sort.each do |rpm| p = YAML.load_file(File.join(File.dirname(rpm), '/../meta.yml'))[:mock_profile] if p == tag_mock_profile or tag_mock_profile.nil? tag_pkgs << rpm end end if not tag_pkgs.empty? puts "* create tag #{tag_meta[:name]} repo START" Dir.mkdir(tag_dir) if not File.exist?(tag_dir) tag_pkgs.each do |rpm| FileUtils.cp rpm, tag_dir end output = `createrepo -q -o #{tag_dir} --update -d #{tag_dir} 2>&1` if $? != 0 puts "create tag #{tag_meta[:name]} operation failed: #{output}".red.bold else puts "* create tag #{tag_meta[:name]} DONE" end else puts "* WARNING: trying to create a tag with no packages." end FileUtils.rm 'tags/.tag' else puts "* ERROR: error creating tag, could not parse tag metadata." end else end end # createrepo snapshot snapshot_sched = Rufus::Scheduler.start_new snapshot_sched.every '2s', :blocking => true do if File.exist?('snapshot/.createsnapshot') puts '* snapshot START' stamp = Time.now.strftime '%Y%m%d_%H%M%S' snapshot_dir = "snapshot/snapshot_#{stamp}" Dir.mkdir snapshot_dir begin Dir["output/*/result/*.rpm"].sort.each do |rpm| FileUtils.cp rpm, snapshot_dir end puts '* snapshot DONE' rescue Exception => e $stderr.puts "snapshot operation failed".red.bold ensure FileUtils.rm 'snapshot/.createsnapshot' end end end # createrepo scheduler createrepo_sched = Rufus::Scheduler.start_new createrepo_sched.every '2s', :blocking => true do if File.exist?('repo/.createrepo') puts '* createrepo START' begin output = `createrepo -q -o repo/ --update -d output/ 2>&1` if $? != 0 raise Exception.new(output) end puts '* createrepo DONE' rescue Exception => e $stderr.puts "createrepo operation failed".red.bold File.open('repo/createrepo.log', 'a') { |f| f.puts e.message } ensure FileUtils.rm 'repo/.createrepo' end end end # Build queue scheduler = Rufus::Scheduler.start_new scheduler.every '2s', :blocking => true do meta[:start_time] = Time.now meta[:mock_profile] = mock_profile queue = Dir['incoming/*.src.rpm'].sort_by {|filename| File.mtime(filename) } if not queue.empty? # Clean workspace first Dir["workspace/job_*"].each do |j| FileUtils.rm_rf j end job_dir = "workspace/job_#{Time.now.strftime '%Y%m%d_%H%M%S'}" imeta_file = "#{queue.first}.metadata" qfile = File.join(job_dir, File.basename(queue.first)) if File.exist?(imeta_file) begin imeta = YAML.load_file(imeta_file) if imeta m = YAML.load_file(imeta_file) meta[:mock_profile] = m[:mock_profile] || BuildBotConfig.mock_profile end rescue Exception puts "* ERROR: parsing #{queue.first} metadata" end FileUtils.rm imeta_file else puts "* WARNING: #{queue.first} does not have metadata!" end job_time = Time.now.strftime '%Y%m%d_%H%M%S' result_dir = job_dir + '/result' FileUtils.mkdir_p result_dir meta[:source] = File.basename(queue.first) meta[:status] = 'building' File.open("workspace/job_#{job_time}/meta.yml", 'w') do |f| f.puts meta.to_yaml end FileUtils.mv queue.first, qfile puts "Building pkg [job_#{job_time}][#{meta[:mock_profile]}] ".ljust(40).yellow.bold + "#{File.basename(qfile)}" rdir = nil begin PKGWizard::Mock.srpm :srpm => qfile, :profile => meta[:mock_profile], :resultdir => result_dir meta[:status] = 'ok' meta[:end_time] = Time.now meta[:build_time] = meta[:end_time] - meta[:start_time] puts "Build OK [job_#{job_time}][#{meta[:mock_profile]}] ".ljust(40).green.bold + "#{File.basename(qfile)}" rescue Exception => e meta[:status] = 'error' puts "Build FAILED [job_#{job_time}][#{meta[:mock_profile]}]".ljust(40).red.bold + "#{File.basename(qfile)}" File.open(job_dir + '/buildbot.log', 'w') do |f| f.puts "#{e.backtrace.join("\n")}" f.puts "#{e.message}" end ensure File.open(job_dir + '/meta.yml', 'w') do |f| f.puts meta.to_yaml end if meta[:status] == 'error' FileUtils.mv job_dir, 'failed/' FileUtils.rm_f 'failed/last' if File.exist?('failed/last') FileUtils.ln_sf "#{File.basename(job_dir)}", "failed/last" else FileUtils.mv job_dir, 'output/' FileUtils.rm_f 'output/last' if File.exist?('output/last') FileUtils.ln_sf "#{File.basename(job_dir)}", "output/last" end end end end Webapp.set :port => cli.config[:port] Webapp.set :public => 'public' Webapp.run! at_exit do NodeRunner.kill end end end end