require 'pkg-wizard/command' require 'pkg-wizard/rpm' require 'pkg-wizard/logger' require 'pkg-wizard/git' require 'pkg-wizard/mock' require 'pkg-wizard/utils' require 'tmpdir' require 'fileutils' require 'uri' require 'sinatra/base' require 'rufus/scheduler' require 'term/ansicolor' require 'pp' require 'yaml' require 'daemons' module FakeColor def red; "#{self}"; end def blue; "#{self}"; end def yellow; "#{self}"; end def green; "#{self}"; end def bold; "#{self}"; end end module PKGWizard class NodeRunner @@logfile = '/dev/null' def self.logfile=(logfile) @@logfile = logfile end def self.available? not `which node`.strip.chomp.empty? end def self.kill Process.kill 15, @@proc.pid end def self.run wslogview_dir = 'wslogview' if not defined? @@proc puts '* starting NODE.JS...' @@proc = IO.popen("node wslogview/server.js #{@@logfile}") end end end class BuildBot < Command registry << { :name => 'build-bot', :klass => self } option :help, :short => "-h", :long => "--help", :description => "Show this message", :on => :tail, :boolean => true, :show_options => true, :exit => 0 option :mock_profile, :short => '-m PROF', :long => '--mock-profile PROF' option :port, :short => '-p PORT', :long => '--port PORT', :default => 4567 option :daemonize, :long => '--daemonize', :default => false option :working_dir, :long => '--working-dir DIR' # not implemented option :log_format, :long => '--log-format FMT', :description => 'Log format to use (web, cli)', :default => 'cli' option :log_server_port, :long => '--log-server-port PORT', :description => 'log server port (60001 default)', :default => '60001' class Webapp < Sinatra::Base def find_job_path(name) (Dir["failed/job_*"] + Dir["success/job_*"]).find { |j| File.basename(j) == name } end post '/tag/:name' do name = params[:name] if name.nil? or name.strip.chomp.empty? status 400 'Invalid tag' else File.open('tags/.tag', 'w') do |f| f.puts name end "Tagging #{name}..." end end post '/createrepo' do FileUtils.touch 'repo/.createrepo' end post '/createsnapshot' do FileUtils.touch 'snapshot/.createsnapshot' end post '/job/clean' do dir = params[:dir] || 'output' if dir == 'output' FileUtils.touch 'output/.clean' elsif dir == 'failed' FileUtils.touch 'failed/.clean' else $stderr.puts "WARNING: job/clean Unknown dir #{dir}. Ignoring." end end post '/build/' do pkg = params[:pkg] if pkg.nil? Logger.instance.error '400: Invalid arguments. Needs pkg in post request' status 400 "Missing pkg parameter.\n" else incoming_file = "incoming/#{pkg[:filename]}" puts "* incoming file".ljust(40) + "#{pkg[:filename]}" FileUtils.cp pkg[:tempfile].path, incoming_file end end # log get '/log' do if NodeRunner.available? NodeRunner.run sleep 0.5 index = 'wslogview/index.html' File.read index else 'node.js is not installed: Real time logs disabled :(' end end # list failed pkgs get '/job/failed' do max = params[:max] || 10 # Find failed jobs jobs = (Dir["failed/job_*"].sort { |a,b| a <=> b }).map { |j| File.basename(j) } # format job as job_XXXX_XXX (pkgname) jobs = jobs.map { |j| "#{j} (#{File.basename(Dir["failed/#{j}/*.src.rpm"].first(), '.src.rpm')})" } max = max.to_i if jobs.size > max.to_i jobs[- max..-1].to_yaml else jobs.to_yaml end end get '/server/stats' do fs = PKGWizard::Utils.filesystem_status { :filesystem => fs, }.to_yaml end get '/job/stats' do fjobs = Dir["failed/job*"].size snapshots = Dir["snapshot/snapshot*"].size sjobs = Dir["output/job*"].size qjobs = Dir["incoming/*.src.rpm"].size cjobs = Dir["workspace/job_*"].size total_jobs = fjobs + sjobs { :failed_jobs => fjobs, :successful_jobs => sjobs, :enqueued => qjobs, :total_jobs => total_jobs, :snapshots => snapshots, :building => cjobs }.to_yaml end # list successfully built pkgs get '/job/successful' do max = params[:max] || 10 jobs = (Dir["output/job_*"].sort { |a,b| a <=> b }).map { |j| File.basename(j) } max = max.to_i if jobs.size > max.to_i jobs[- max..-1].to_yaml else jobs.to_yaml end end # # Rebuild a previous job (It can be either successful or failed build) # post '/job/rebuild/:name' do name = params[:name] job = find_job_path(name) if job.nil? status 404 else puts "Rebuilding job [#{name}]".ljust(40) + File.basename(Dir["#{job}/*.rpm"].first) FileUtils.cp Dir["#{job}/*.rpm"].first, 'incoming/' FileUtils.rm_rf job end end # Get current building job (empty output if none) get '/job/current' do job = Dir['workspace/job*'].first if job File.read(job + '/meta.yml') else status 404 end end # Get job info get '/job/:name' do jname = params[:name] jobs = Dir['output/job_*'] + Dir['failed/job_*'] found = false meta = '' if jname == 'all' found = true metas = [] jobs.each do |j| mfile = j + '/meta.yml' if File.exist?(mfile) metas << YAML.load_file(mfile) else $stderr.puts "[WARNING] Meta file #{mfile} not found" end end meta = metas.to_yaml else jobs.each do |j| if File.basename(j) == jname found = true puts j + '/meta.yml' meta = File.read(j + '/meta.yml') break end end end status 404 if not found meta end end def self.perform cli = BuildBot.new cli.banner = "\nUsage: pkgwiz build-bot (options)\n\n" cli.parse_options ## Node.JS log server stuff wslogview_dir = File.join(File.dirname(__FILE__), '/../../../resources/wslogview/') node_port = cli.config[:log_server_port] if not File.exist?('wslogview') FileUtils.cp_r wslogview_dir, 'wslogview' end html = File.read('wslogview/index.html.tmpl').gsub('@@NODEJSPORT@@', node_port) serverjs = File.read('wslogview/server.js.tmpl').gsub('@@NODEJSPORT@@', node_port) File.open 'wslogview/index.html', 'w' do |f| f.puts html end File.open 'wslogview/server.js', 'w' do |f| f.puts serverjs end if cli.config[:log_format] == 'web' String.class_eval do include FakeColor; end else String.class_eval do include Term::ANSIColor; end end pwd = cli.config[:working_dir] || Dir.pwd NodeRunner.logfile = (cli.config[:working_dir] || Dir.pwd) + '/build-bot.log' pwd = File.expand_path pwd if cli.config[:daemonize] umask = File.umask Daemons.daemonize :app_name => 'build-bot', :dir_mode => :normal, :dir => pwd Dir.chdir pwd log = File.new("build-bot.log", "a") $stdout.reopen(log) $stderr.reopen(log) $stdout.sync = true $stderr.sync = true File.umask umask else Dir.chdir pwd end mock_profile = cli.config[:mock_profile] if not mock_profile $stderr.puts 'Invalid mock profile.' $stderr.puts cli.opt_parser.help exit 1 end if not File.exist? '/usr/bin/rpmbuild' $stderr.puts 'rpmbuild command not found. Install it first.' exit 1 end if not File.exist? '/usr/sbin/mock' $stderr.puts 'mock command not found. Install it first.' exit 1 end meta = { :mock_profile => mock_profile } Dir.mkdir 'incoming' if not File.exist?('incoming') Dir.mkdir 'output' if not File.exist?('output') Dir.mkdir 'workspace' if not File.exist?('workspace') Dir.mkdir 'archive' if not File.exist?('archive') Dir.mkdir 'failed' if not File.exist?('failed') Dir.mkdir 'snapshot' if not File.exist?('snapshot') Dir.mkdir 'tags' if not File.exist?('tags') FileUtils.ln_sf 'output', 'repo' if not File.exist?('repo') cleaner = Rufus::Scheduler.start_new cleaner.every '2s', :blocking => true do if File.exist?('failed/.clean') puts '* cleaning FAILED jobs' Dir["failed/job_*"].each do |d| FileUtils.rm_rf d end FileUtils.rm 'failed/.clean' FileUtils.rm_rf "failed/last" end if File.exist?('output/.clean') puts '* cleaning OUTPUT jobs' Dir["output/job_*"].each do |d| FileUtils.rm_rf d end FileUtils.rm 'output/.clean' FileUtils.rm_rf 'output/repodata' FileUtils.rm_rf 'output/last' end end # tag routine tag_sched = Rufus::Scheduler.start_new tag_sched.every '2s', :blocking => true do if File.exist?('tags/.tag') tag = File.read('tags/.tag').strip.chomp tag_dir = "tags/#{tag}" Dir.mkdir(tag_dir) if not File.exist?(tag_dir) Dir["output/*/result/*.rpm"].sort.each do |rpm| FileUtils.cp rpm, tag_dir end puts "* create tag #{tag} repo START" output = `createrepo -q -o #{tag_dir} --update -d #{tag_dir} 2>&1` if $? != 0 puts "create tag #{tag} operation failed: #{output}".red.bold else puts "* create tag #{tag} DONE" end FileUtils.rm 'tags/.tag' end end # createrepo snapshot snapshot_sched = Rufus::Scheduler.start_new snapshot_sched.every '2s', :blocking => true do if File.exist?('snapshot/.createsnapshot') puts '* snapshot START' stamp = Time.now.strftime '%Y%m%d_%H%M%S' snapshot_dir = "snapshot/snapshot_#{stamp}" Dir.mkdir snapshot_dir begin Dir["output/*/result/*.rpm"].sort.each do |rpm| FileUtils.cp rpm, snapshot_dir end puts '* snapshot DONE' rescue Exception => e $stderr.puts "snapshot operation failed".red.bold ensure FileUtils.rm 'snapshot/.createsnapshot' end end end # createrepo scheduler createrepo_sched = Rufus::Scheduler.start_new createrepo_sched.every '2s', :blocking => true do if File.exist?('repo/.createrepo') puts '* createrepo START' begin output = `createrepo -q -o repo/ --update -d output/ 2>&1` if $? != 0 raise Exception.new(output) end puts '* createrepo DONE' rescue Exception => e $stderr.puts "createrepo operation failed".red.bold File.open('repo/createrepo.log', 'a') { |f| f.puts e.message } ensure FileUtils.rm 'repo/.createrepo' end end end # Build queue scheduler = Rufus::Scheduler.start_new scheduler.every '2s', :blocking => true do meta[:start_time] = Time.now queue = Dir['incoming/*.src.rpm'].sort_by {|filename| File.mtime(filename) } if not queue.empty? # Clean workspace first Dir["workspace/job_*"].each do |j| FileUtils.rm_rf j end job_dir = "workspace/job_#{Time.now.strftime '%Y%m%d_%H%M%S'}" qfile = File.join(job_dir, File.basename(queue.first)) job_time = Time.now.strftime '%Y%m%d_%H%M%S' result_dir = job_dir + '/result' FileUtils.mkdir_p result_dir meta[:source] = File.basename(queue.first) meta[:status] = 'building' File.open("workspace/job_#{job_time}/meta.yml", 'w') do |f| f.puts meta.to_yaml end FileUtils.mv queue.first, qfile puts "Building pkg [job_#{job_time}]".ljust(40).yellow.bold + "#{File.basename(qfile)}" rdir = nil begin PKGWizard::Mock.srpm :srpm => qfile, :profile => mock_profile, :resultdir => result_dir meta[:status] = 'ok' meta[:end_time] = Time.now meta[:build_time] = meta[:end_time] - meta[:start_time] puts "Build OK [job_#{job_time}] #{meta[:build_time].to_i}s ".ljust(40).green.bold + "#{File.basename(qfile)}" rescue Exception => e meta[:status] = 'error' puts "Build FAILED [job_#{job_time}]".ljust(40).red.bold + "#{File.basename(qfile)}" File.open(job_dir + '/buildbot.log', 'w') do |f| f.puts "#{e.backtrace.join("\n")}" f.puts "#{e.message}" end ensure File.open(job_dir + '/meta.yml', 'w') do |f| f.puts meta.to_yaml end if meta[:status] == 'error' FileUtils.mv job_dir, 'failed/' FileUtils.rm_f 'failed/last' if File.exist?('failed/last') FileUtils.ln_sf "#{File.basename(job_dir)}", "failed/last" else FileUtils.mv job_dir, 'output/' FileUtils.rm_f 'output/last' if File.exist?('output/last') FileUtils.ln_sf "#{File.basename(job_dir)}", "output/last" end end end end Webapp.set :port => cli.config[:port] Webapp.set :public => 'wslogview' Webapp.run! at_exit do NodeRunner.kill end end end end