require 'minitest/reporters' require 'tmpdir' require 'yaml' module Minitest module Reporters # This reporter creates a report providing the average (mean), minimum and # maximum times for a test to run. Running this for all your tests will # allow you to: # # 1) Identify the slowest running tests over time as potential candidates # for improvements or refactoring. # 2) Identify (and fix) regressions in test run speed caused by changes to # your tests or algorithms in your code. # 3) Provide an abundance of statistics to enjoy. # # This is achieved by creating a (configurable) 'previous runs' statistics # file which is parsed at the end of each run to provide a new # (configurable) report. These statistics can be reset at any time by using # a simple rake task: # # rake reset_statistics # class MeanTimeReporter < Minitest::Reporters::DefaultReporter class InvalidOrder < StandardError; end class InvalidSortColumn < StandardError; end # Reset the statistics file for this reporter. Called via a rake task: # # rake reset_statistics # # @return [Boolean] def self.reset_statistics! new.reset_statistics! end # @param options [Hash] # @option previous_runs_filename [String] Contains the times for each test # by description. Defaults to '/tmp/minitest_reporters_previous_run'. # @option report_filename [String] Contains the parsed results for the # last test run. Defaults to '/tmp/minitest_reporters_report'. # @option show_count [Fixnum] The number of tests to show in the report # summary at the end of the test run. Default is 15. # @option show_progress [Boolean] If true it prints pass/skip/fail marks. # Default is true. # @option show_all_runs [Boolean] If true it shows all recorded suit results. # Default is true. # @option sort_column [Symbol] One of :avg (default), :min, :max, :last. # Determines the column by which the report summary is sorted. # @option order [Symbol] One of :desc (default), or :asc. By default the # report summary is listed slowest to fastest (:desc). :asc will order # the report summary as fastest to slowest. # @return [Minitest::Reporters::MeanTimeReporter] def initialize(options = {}) super @all_suite_times = [] end # Copies the suite times from the # {Minitest::Reporters::DefaultReporter#after_suite} method, making them # available to this class. # # @return [Hash Float>] def after_suite(suite) super @all_suite_times = @suite_times end # Runs the {Minitest::Reporters::DefaultReporter#report} method and then # enhances it by storing the results to the 'previous_runs_filename' and # outputs the parsed results to both the 'report_filename' and the # terminal. # def report super create_or_update_previous_runs! create_new_report! write_to_screen! end def on_start super if options[:show_progress] end def on_record(test) super if options[:show_progress] end def on_report super if options[:show_progress] end # Resets the 'previous runs' file, essentially removing all previous # statistics gathered. # # @return [void] def reset_statistics! File.open(previous_runs_filename, 'w+') { |f| f.write('') } end protected attr_accessor :all_suite_times private # @return [Hash Float>] def current_run Hash[all_suite_times] end # @return [Hash] Sets default values for the filenames used by this class, # and the number of tests to output to output to the screen after each # run. def defaults { order: :desc, show_count: 15, show_progress: true, show_all_runs: true, sort_column: :avg, previous_runs_filename: File.join(Dir.tmpdir, 'minitest_reporters_previous_run'), report_filename: File.join(Dir.tmpdir, 'minitest_reporters_report'), } end # Added to the top of the report file and to the screen output. # # @return [String] def report_title "\n\e[4mMinitest Reporters: Mean Time Report\e[24m " \ "(Samples: #{samples}, Order: #{sort_column.inspect} " \ "#{order.inspect})\n" end # The report itself. Displays statistics about all runs, ideal for use # with the Unix 'head' command. Listed in slowest average descending # order. # # @return [String] def report_body order_sorted_body.each_with_object([]) do |result, obj| rating = rate(result[:last], result[:min], result[:max]) obj << "#{avg_label} #{result[:avg].to_s.ljust(12)} " \ "#{min_label} #{result[:min].to_s.ljust(12)} " \ "#{max_label} #{result[:max].to_s.ljust(12)} " \ "#{run_label(rating)} #{result[:last].to_s.ljust(12)} " \ "#{des_label} #{result[:desc]}\n" end.join end # @return [String] All of the column-sorted results sorted by the :order # option. (Defaults to :desc). def order_sorted_body if desc? column_sorted_body.reverse elsif asc? column_sorted_body end end # @return [Array String>>] All of the results sorted by # the :sort_column option. (Defaults to :avg). def column_sorted_body runs = options[:show_all_runs] ? previous_run : current_run runs.keys.each_with_object([]) do |description, obj| timings = previous_run[description] size = Array(timings).size sum = Array(timings).inject { |total, x| total + x } obj << { avg: (sum / size).round(9), min: Array(timings).min.round(9), max: Array(timings).max.round(9), last: Array(timings).last.round(9), desc: description, } end.sort_by { |k| k[sort_column] } end # @return [Hash] def options defaults.merge!(@options) end # @return [Fixnum] The number of tests to output to output to the screen # after each run. def show_count options[:show_count] end # @return [Hash Array] def previous_run @previous_run ||= YAML.load_file(previous_runs_filename) end # @return [String] The path to the file which contains all the durations # for each test run. The previous runs file is in YAML format, using the # test name for the key and an array containing the time taken to run # this test for values. def previous_runs_filename options[:previous_runs_filename] end # Returns a boolean indicating whether a previous runs file exists. # # @return [Boolean] def previously_ran? File.exist?(previous_runs_filename) end # @return [String] The path to the file which contains the parsed test # results. The results file contains a line for each test with the # average time of the test, the minimum time the test took to run, # the maximum time the test took to run and a description of the test # (which is the test name as emitted by Minitest). def report_filename options[:report_filename] end # A barbaric way to find out how many runs are in the previous runs file; # this method takes the first test listed, and counts its samples # trusting (naively) all runs to be the same number of samples. This will # produce incorrect averages when new tests are added, so it is advised # to restart the statistics by removing the 'previous runs' file. A rake # task is provided to make this more convenient. # # rake reset_statistics # # @return [Fixnum] def samples return 1 unless previous_run.first[1].is_a?(Array) previous_run.first[1].size end # Creates a new 'previous runs' file, or updates the existing one with # the latest timings. # # @return [void] def create_or_update_previous_runs! if previously_ran? current_run.each do |description, elapsed| new_times = if previous_run[description.to_s] Array(previous_run[description.to_s]) << elapsed else Array(elapsed) end previous_run.store(description.to_s, new_times) end File.write(previous_runs_filename, previous_run.to_yaml) else File.write(previous_runs_filename, current_run.to_yaml) end end # Creates a new report file in the 'report_filename'. This file contains # a line for each test of the following example format: (this is a single # line despite explicit wrapping) # # Avg: 0.0555555 Min: 0.0498765 Max: 0.0612345 Last: 0.0499421 # Description: The test name # # Note however the timings are to 9 decimal places, and padded to 12 # characters and each label is coloured, Avg (yellow), Min (green), # Max (red), Last (multi), and Description (blue). It looks pretty! # # The 'Last' label is special in that it will be colour coded depending # on whether the last run was faster (bright green) or slower (bright red) # or inconclusive (purple). This helps to identify changes on a per run # basis. # # @return [void] def create_new_report! File.write(report_filename, report_title + report_body) end # Writes a number of tests (configured via the 'show_count' option) to the # screen after creating the report. See '#create_new_report!' for example # output information. # # @return [void] def write_to_screen! puts report_title puts report_body.lines.take(show_count) end # @return [String] A yellow 'Avg:' label. def avg_label ANSI::Code.yellow('Avg:') end # @return [String] A blue 'Description:' label. def des_label ANSI::Code.blue('Description:') end # @return [String] A red 'Max:' label. def max_label ANSI::Code.red('Max:') end # @return [String] A green 'Min:' label. def min_label ANSI::Code.green('Min:') end # @param rating [Symbol] One of :faster, :slower or :inconclusive. # @return [String] A purple 'Last:' label. def run_label(rating) case rating when :faster then ANSI::Code.green('Last:') when :slower then ANSI::Code.red('Last:') else ANSI::Code.magenta('Last:') end end # @param run [Float] The last run time. # @param min [Float] The minimum run time. # @param max [Float] The maximum run time. # @return [Symbol] One of :faster, :slower or :inconclusive. def rate(run, min, max) if run == min :faster elsif run == max :slower else :inconclusive end end # @return [Boolean] Whether the given :order option is :asc. def asc? order == :asc end # @return [Boolean] Whether the given :order option is :desc (default). def desc? order == :desc end # @raise [Minitest::Reporters::MeanTimeReporter::InvalidOrder] # When the given :order option is invalid. # @return [Symbol] The :order option, or by default; :desc. def order orders = [:desc, :asc] if orders.include?(options[:order]) options[:order] else fail Minitest::Reporters::MeanTimeReporter::InvalidOrder, "`:order` option must be one of #{orders.inspect}." end end # @raise [Minitest::Reporters::MeanTimeReporter::InvalidSortColumn] # When the given :sort_column option is invalid. # @return [Symbol] The :sort_column option, or by default; :avg. def sort_column sort_columns = [:avg, :min, :max, :last] if sort_columns.include?(options[:sort_column]) options[:sort_column] else fail Minitest::Reporters::MeanTimeReporter::InvalidSortColumn, "`:sort_column` option must be one of #{sort_columns.inspect}." end end end end end