#-----------------------------------------------------------------# # # # (C) Copyright Rubysophic Inc. 2007-2008 # # All rights reserved. # # # # Use, duplication or disclosure of the code is not permitted # # unless licensed. # # # # Last Updated: 7/18/08 # #-----------------------------------------------------------------# # # # RubyRunCommander__ is a module which handles the commands # # cmd_status, cmd_object_map, cmd_soft_kill and cmd_hard_kill # # # #-----------------------------------------------------------------# module RubyRunCommander__ # Use Thread.list to list show thread status, and native code to # display the last line # and function of the threads def dump_thread_status (unsupport_function; return) unless $rubyrun_native $rubyrun_thread_status_reporter = RubyRunHTMLWriter.new(@rubyrun_report_folder + '/' + File.basename($0, ".*") + '_' + $$.to_s + '_thread_status.html', nil, shift_age = 10, shift_size = 4096000) unless $rubyrun_thread_status_reporter start_time = Time.now th_data_hash = RubyRunNative__.get_all_top_stacks odd_row ||= true table_content = '' Thread.list.each {|th| thread_id = get_thread_id(th) table_content += sprintf("#{odd_row ? THREAD_STATUS_ODD_ROW : THREAD_STATUS_EVEN_ROW}", thread_id, th.status, get_top_stack(th_data_hash, thread_id)) odd_row = !odd_row } html_content = THREAD_STATUS_HTML.sub(/%START_TIMESTAMP%/,start_time.strftime("%H:%M:%S %m/%d/%Y")) html_content.sub!(/%THREAD_STATUS_ROW%/,table_content) $rubyrun_thread_status_reporter.info(html_content) end # Dump Controller/Actions response time metrics # metrics structure # metrics[0] Thread ID metrics[1] Timestamp of the request # metrics[2] URL metrics[3] Controller name # metrics[4] Action name metrics[5] Response time # metrics[6] Action time metrics[7] Database IO time # metrics[8] View time metrics[9] Uncaptured time # metrics[10] Dispatch wait time def dump_reports(dump_all_reports = false) buffer = return_and_switch_buffer buffer.each { |metrics| # Last element is 1, representing a request count of 1, used for calculating average response time for this controller/action update_perf_metrics(metrics[3], {metrics[4] => [metrics[5],metrics[6],metrics[7],metrics[8],metrics[9],metrics[10],1]}) } @rubyrun_req_count ||= 0 @rubyrun_req_count += buffer.length if $rubyrun_config['OUTPUT'].include?(RUBYRUN_OUTPUT_PERF_SUMMARY) && dump_all_reports create_rss_channels if (!$rubyrun_perf_summary_rss && $rubyrun_rails_env) add_perf_summary_rss_item(@rubyrun_req_count) @rubyrun_req_count = 0 end if $rubyrun_config['OUTPUT'].include?(RUBYRUN_OUTPUT_TXN_LOG) create_csv_files unless $rubyrun_txn_log_reporter add_txn_log_csv_item(buffer) end buffer.clear # Clear the buffer so that the main thread will push the data into a blank bucket end # The way to do soft/hard kill is to performa a thr.raise on the thread # from the thread monitor. Using the begin/rescue created around the block # in Thread.new by RubyRunInstrumentor__, the raise will be rescued # and $@ is then extracted to a global hash. # Softkill only kills non-main threads. Hardkill kills the main thread also # but as the last step. def kill_threads(monitor_thr) (unsupport_function; return) unless $rubyrun_native if !$rubyrun_thread_dump_reporter $rubyrun_thread_dump_reporter = Logger.new(@rubyrun_report_folder + '/' + File.basename($0, ".*") + '_' + $$.to_s + '_thread_dump.txt', shift_age = 10, shift_size = 4096000) $rubyrun_thread_dump_reporter.level = Logger::INFO class << $rubyrun_thread_dump_reporter include RubyRunUtils__ def format_message (severity, timestamp, progname, msg) "[#{timestamp.strftime("%Y-%m-%d %H:%M:%S")}.#{("%.3f" % timestamp.to_f).split('.')[1]}] #{get_thread_id} #{msg}\n" end end end th_data_hash = RubyRunNative__.get_all_top_stacks j_th_id = return_joined_thread(th_data_hash) Thread.list.each {|th| th_id = get_thread_id(th) if th.status == 'sleep' && th_id != get_thread_id && th_id != get_thread_id(monitor_thr) && th_id != get_thread_id(Thread.main) && th_id != j_th_id $rubyrun_thread_dump_reporter.info "*** Raising exception #{RUBYRUN_KILL_3_STRING} to #{get_thread_id(th)} ***" th.raise ThreadError, RUBYRUN_KILL_3_STRING end } sleep 3 back_trace_all(th_data_hash) hard_kill = hard_kill? remove_cmd_folder Thread.main.raise ThreadError, RUBYRUN_KILL_3_STRING if hard_kill end # Show the top 20 Ruby classes which have the largest no. of instances in memory # The snapshot is taken after a gc call is made def dump_object_map start_time = Time.now $rubyrun_obj_map_reporter = RubyRunHTMLWriter.new(@rubyrun_report_folder + '/' + File.basename($0, ".*") + '_' + $$.to_s + '_object_map.html', nil, shift_age = 10, shift_size = 4096000) unless $rubyrun_obj_map_reporter object_map = Hash.new ttl_object = 0 ObjectSpace.garbage_collect ObjectSpace.each_object { |obj| ttl_object += 1 object_map.has_key?(obj.class) ? object_map[obj.class] += 1 : object_map[obj.class] = 1 } results = object_map.sort{|a,b| a[1]<=>b[1]}.reverse! table_content = '' odd_row ||=true 20.times {|i| table_content += sprintf("#{odd_row ? OBJ_MAP_ODD_ROW : OBJ_MAP_EVEN_ROW}", results[i][0], results[i][1].to_s) odd_row = !odd_row } html_content = OBJ_MAP_HTML.sub(/%START_TIMESTAMP%/,start_time.strftime("%H:%M:%S %m/%d/%Y")) html_content.sub!(/%OBJ_MAP_ROW%/,table_content) $rubyrun_obj_map_reporter.info(html_content) end # metrics hash is a global collection point for all metrics (averaged) # for all actions by controller # Use serialization before updating this global hash # Structure of $rubyrun_metrics_hash: # controller_name => {action_name => [response_time, action_time, # db_io_time, view_time, # uncaptured_time, dispatch_wait_time, # request_count]} def update_perf_metrics(controller, action_metrics_hash) $rubyrun_metrics_hash[controller].merge!(action_metrics_hash) {|action, o_metrics, new_metrics| o_metrics.each_index { |x| (o_metrics[x] += new_metrics[x]; break) if x == (o_metrics.length-1) # Calculate the total request count for this controller/action o_metrics[x] = (o_metrics[x] * o_metrics.last + new_metrics[x])/(o_metrics.last + new_metrics.last).to_f } o_metrics } end # If a thread is joined this method returns the joining thread ID def return_joined_thread(th_data_hash) th_data_hash.each {|th, top_stack| if th.to_s.include?(get_thread_id(Thread.main)) top_stack[0] =~ /\*\*(.+?)\*\*/ return $1 end } end # Remove the cmd_kill-3 folder or file if any def remove_cmd_folder [RUBYRUN_CMD_SOFT_KILL, RUBYRUN_CMD_HARD_KILL].each { |cmd| path = $rubyrun_working_dir + cmd + '_' + Process.pid.to_s next unless File.exist?(path) File.directory?(path) ? Dir.delete(path) : File.delete(path) } end # If exists, indicate to the monitor thread that a thread # status report is requested def thread_status? File.exists?($rubyrun_working_dir + RUBYRUN_CMD_STATUS) end # If exists, indicate to the monitor thread that a soft kill # (kill all threads except the main thread) command is sent def soft_kill? File.exists?($rubyrun_working_dir + RUBYRUN_CMD_SOFT_KILL + '_' + Process.pid.to_s) end # If exists, indicate to the monitor thread that a hard kill # (kill all threads including the main thread) command is sent def hard_kill? File.exists?($rubyrun_working_dir + RUBYRUN_CMD_HARD_KILL + '_' + Process.pid.to_s) end # If exists, indicate to the montior thread that a object map is requested def object_map? File.exists?($rubyrun_working_dir + RUBYRUN_CMD_OBJECT_MAP) end # If exists, indicate to the monitor thread to exit def exit_monitor? File.exists?($rubyrun_working_dir + RUBYRUN_CMD_EXIT) end # Log if native library can't be loaded or not found def unsupport_function $rubyrun_logger.info "Native library not available. Function not supported." end end