module RequestLogAnalyzer # The RequestLogAnalyzer::Controller class creates a LogParser instance for the # requested file format, and connect it with sources and aggregators. # # Sources are streams or files from which the requests will be parsed. # Aggregators will handle every passed request to yield a meaningfull results. # # - Use the build-function to build a controller instance using command line arguments. # - Use add_aggregator to register a new aggregator # - Use add_source to register a new aggregator # - Use the run! method to start the parser and send the requests to the aggregators. # # Note that the order of sources can be imported if you have log files than succeed # eachother. Requests that span over succeeding files will be parsed correctly if the # sources are registered in the correct order. This can be helpful to parse requests # from several logrotated log files. class Controller attr_reader :source, :filters, :aggregators, :output, :options # Builds a RequestLogAnalyzer::Controller given parsed command line arguments # arguments A CommandLine::Arguments hash containing parsed commandline parameters. # report_with Width of the report. Defaults to 80. def self.build_from_arguments(arguments) options = {} # Copy fields options[:database] = arguments[:database] options[:reset_database] = arguments[:reset_database] options[:debug] = arguments[:debug] options[:yaml] = arguments[:dump] options[:parse_strategy] = arguments[:parse_strategy] options[:no_progress] = arguments[:no_progress] options[:format] = arguments[:format] options[:output] = arguments[:output] options[:file] = arguments[:file] options[:format] = arguments[:format] options[:after] = arguments[:after] options[:before] = arguments[:before] options[:reject] = arguments[:reject] options[:select] = arguments[:select] options[:boring] = arguments[:boring] options[:aggregator] = arguments[:aggregator] options[:report_width] = arguments[:report_width] options[:report_sort] = arguments[:report_sort] options[:report_amount] = arguments[:report_amount] # Apache format workaround if arguments[:rails_format] options[:format] = {:rails => arguments[:rails_format]} elsif arguments[:apache_format] options[:format] = {:apache => arguments[:apache_format]} end # Register sources if arguments.parameters.length == 1 file = arguments.parameters[0] if file == '-' || file == 'STDIN' options.store(:source_files, $stdin) elsif File.exist?(file) options.store(:source_files, file) else puts "File not found: #{file}" exit(0) end else options.store(:source_files, arguments.parameters) end build(options) end # Build a new controller using parameters (Base for new API) # source The source file # Options are passd on to the LogParser. # # Options # * :database Database file # * :reset_database # * :debug Enables echo aggregator. # * :yaml Output to YAML # * :parse_strategy # * :no_progress Do not display the progress bar # * :output :fixed_width, :html or Output class. Defaults to fixed width. # * :file Filestring or File or StringIO # * :format :rails, {:apache => 'FORMATSTRING'}, :merb, etcetera or Format Class. Defaults to :rails. # * :source_files File or STDIN # * :after Drop all requests after this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format) # * :before Drop all requests before this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format) # * :reject Reject specific {:field => :value} combination. Expects single hash. # * :select Select specific {:field => :value} combination. Expects single hash. # * :aggregator Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY!). Defaults to [:summarizer # * :boring Do not show color on STDOUT. Defaults to False. # * :report_width Width or reports in characters. Defaults to 80. # # TODO: # Check if defaults work (Aggregator defaults seem wrong). # Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization. def self.build(options) # Defaults options[:output] ||= 'fixed_width' options[:format] ||= :rails options[:aggregator] ||= [:summarizer] options[:report_width] ||= 80 options[:report_amount] ||= 20 options[:report_sort] ||= 'sum,mean' options[:boring] ||= false # Backwards compatibility if options[:dump] && options[:yaml].blank? warn "[DEPRECATION] `:dump` is deprecated. Please use `:yaml` instead." options[:yaml] = options[:dump] end # Set the output class output_args = {} output_object = nil if options[:output].is_a? Class output_class = options[:output] else output_class = RequestLogAnalyzer::Output::const_get(options[:output]) end output_sort = options[:report_sort].split(',').map { |s| s.to_sym } output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i if options[:file] output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+") output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount } elsif options[:mail] output_object = RequestLogAnalyzer::Mailer.new(arguments[:mail]) output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount } else output_object = STDOUT output_args = {:width => options[:report_width].to_i, :color => !options[:boring], :characters => (options[:boring] ? :ascii : :utf), :sort => output_sort, :amount => output_amount } end output_instance = output_class.new(output_object, output_args) # Create the controller with the correct file format if options[:format].kind_of?(Hash) file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0]) else file_format = RequestLogAnalyzer::FileFormat.load(options[:format]) end # Kickstart the controller controller = Controller.new( RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => options[:source_files]), { :output => output_instance, :database => options[:database], # FUGLY! :yaml => options[:yaml], :reset_database => options[:reset_database], :no_progress => options[:no_progress]}) # register filters if options[:after] || options[:before] filter_options = {} [:after, :before].each do |filter| case options[filter] when Date, DateTime, Time filter_options[filter] = options[filter] when String filter_options[filter] = DateTime.parse(options[filter]) end end controller.add_filter(:timespan, filter_options) end if options[:reject] options[:reject].each do |(field, value)| controller.add_filter(:field, :mode => :reject, :field => field, :value => value) end end if options[:reject] options[:select].each do |(field, value)| controller.add_filter(:field, :mode => :select, :field => field, :value => value) end end # register aggregators options[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) } controller.add_aggregator(:summarizer) if options[:aggregator].empty? controller.add_aggregator(:echo) if options[:debug] controller.add_aggregator(:database_inserter) if options[:database] && !options[:aggregator].include?('database') file_format.setup_environment(controller) return controller end # Builds a new Controller for the given log file format. # format Logfile format. Defaults to :rails # Options are passd on to the LogParser. # * :database Database the controller should use. # * :yaml Yaml Dump the contrller should use. # * :output All report outputs get << through this output. # * :no_progress No progress bar def initialize(source, options = {}) @source = source @options = options @aggregators = [] @filters = [] @output = options[:output] @interrupted = false # Register the request format for this session after checking its validity raise "Invalid file format!" unless @source.file_format.valid? # Install event handlers for wrnings, progress updates and source changes @source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } } @source.progress = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress] @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) } end # Progress function. # Expects :started with file, :progress with current line and :finished or :interrupted when done. # message Current state (:started, :finished, :interupted or :progress). # value File or current line. def handle_progress(message, value = nil) case message when :started @progress_bar = CommandLine::ProgressBar.new(File.basename(value), File.size(value), STDOUT) when :finished @progress_bar.finish @progress_bar = nil when :interrupted if @progress_bar @progress_bar.halt @progress_bar = nil end when :progress @progress_bar.set(value) end end # Source change handler def handle_source_change(change, filename) @aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) } end # Adds an aggregator to the controller. The aggregator will be called for every request # that is parsed from the provided sources (see add_source) def add_aggregator(agg) agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer::to_camelcase(agg)) if agg.kind_of?(Symbol) @aggregators << agg.new(@source, @options) end alias :>> :add_aggregator # Adds a request filter to the controller. def add_filter(filter, filter_options = {}) filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol) @filters << filter.new(source.file_format, @options.merge(filter_options)) end # Push a request through the entire filterchain (@filters). # request The request to filter. # Returns the filtered request or nil. def filter_request(request) @filters.each do |filter| request = filter.filter(request) return nil if request.nil? end return request end # Push a request to all the aggregators (@aggregators). # request The request to push to the aggregators. def aggregate_request(request) return false unless request @aggregators.each { |agg| agg.aggregate(request) } return true end # Runs RequestLogAnalyzer # 1. Call prepare on every aggregator # 2. Generate requests from source object # 3. Filter out unwanted requests # 4. Call aggregate for remaning requests on every aggregator # 4. Call finalize on every aggregator # 5. Call report on every aggregator # 6. Finalize Source def run! # @aggregators.each{|agg| p agg} @aggregators.each { |agg| agg.prepare } install_signal_handlers @source.each_request do |request| break if @interrupted aggregate_request(filter_request(request)) end @aggregators.each { |agg| agg.finalize } @output.header @aggregators.each { |agg| agg.report(@output) } @output.footer @source.finalize if @output.io.kind_of?(File) puts puts "Report written to: " + File.expand_path(@output.io.path) puts "Need an expert to analyze your application?" puts "Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com" puts "Thanks for using request-log-analyzer!" @output.io.close elsif @output.io.kind_of?(RequestLogAnalyzer::Mailer) @output.io.mail end end def install_signal_handlers Signal.trap("INT") do handle_progress(:interrupted) puts "Caught interrupt! Stopping parsing..." @interrupted = true end end end end