tasks/evaluation.rake in nucleus-0.1.0 vs tasks/evaluation.rake in nucleus-0.2.0

- old
+ new

@@ -1,162 +1,162 @@ -require 'rspec/core/rake_task' -require 'oj' - -namespace :evaluation do - namespace :requests do - namespace :count do - desc 'Load the data that will be used in the request evaluation tasks' - task load: :environment do - # TODO: choose API version - api_version = 'v1' - - adapter_dao = Nucleus::API::DB::AdapterDao.instance(api_version) - endpoint_dao = Nucleus::API::DB::EndpointDao.instance(api_version) - provider_dao = Nucleus::API::DB::ProviderDao.instance(api_version) - vendor_dao = Nucleus::API::DB::VendorDao.instance(api_version) - @vendor_results = {} - - adapter_dao.all.each do |adapter_index_entry| - vendor_name = vendor_dao.get(provider_dao.get(endpoint_dao.get(adapter_index_entry.id).provider).vendor).name - # from camel to sneak case - adapter_file_name = vendor_name.gsub(/\s/, '_').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') - .gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase - next if @vendor_results.key?(vendor_name) - adaper_results = {} - - method_recordings_dir = File.join('spec', 'adapter', 'recordings', api_version, adapter_file_name, - 'vcr_cassettes', 'with_valid_credentials', 'is_compliant_and') - Find.find(method_recordings_dir) do |file| - next if File.directory?(file) || File.basename(file) == '.DS_Store' - test = Pathname.new(file).relative_path_from(Pathname.new(method_recordings_dir)).to_s - # Load contents with the serializer, oj - cassette = ::Oj.load(File.read(file)) - adaper_results[test] = cassette['http_interactions'].length - end - @vendor_results[vendor_name] = adaper_results - end - - @all_tests = @vendor_results.flat_map { |_name, tests| tests.keys }.uniq - end - - task markdown: :load do - # table header - puts "Test / Vendor|#{@vendor_results.keys.join('|')}" - - # column styles - alignment = ':--' - @vendor_results.length.times { |_time| alignment << '|:-:' } - puts alignment - - lines = [] - @all_tests.sort.each_with_index do |test_name, line| - @vendor_results.each do |_vendor, results| - lines[line] = "#{test_name}" unless lines[line] - lines[line] << "|#{results[test_name]}" - end - end - - lines.each do |line| - puts line - end - - # table header - puts '|' * @vendor_results.length - puts "Test / Vendor|#{@vendor_results.keys.join('|')}" - puts '|' * @vendor_results.length - puts "Tested methods|#{@vendor_results.collect { |_name, tests| tests.length }.join('|')}" - puts "Total vendor API requests|#{@vendor_results.collect do |_name, tests| - tests.map { |_key, value| value }.sum - end.join('|')}" - puts "Avg. vendor API requests per tested method|#{@vendor_results.collect do |_name, tests| - ((tests.map { |_key, value| value }.sum) / tests.length.to_f).round(2) - end.join('|')}" - end - - task :latex, [:save_to] => :load do |_t, args| - tests_with_wait = %w(app-actions/lifecycle/restart/succeeds - app-actions/lifecycle/start/succeeds_for_app_all_if_currently_stopped - app-actions/lifecycle/start/succeeds_for_app_min_if_currently_stopped - app-actions/lifecycle/stop/succeeds_for_app_all_if_currently_running - app-actions/lifecycle/stop/succeeds_for_app_min_if_currently_running - app-data/deploy/succeeds/and_app_with - app-data/rebuild/changes_the_release_version_property - app/web access/for_app_with_min_properties) - ignore_tests_with_wait = 0 - ignore_req_with_wait = {} - @vendor_results.each { |vendor, _| ignore_req_with_wait[vendor] = 0 } - - table = [] - table << '\\scriptsize' - table << "\\begin{longtable}{|L{12cm}|#{'c|' * @vendor_results.length}}" - next_line = ' \\multicolumn{1}{l}{\\Large{\\textbf{recorded test cassette}}}' - @vendor_results.keys.each do |vendor| - next_line += " & \\multicolumn{1}{l}{\\turn{60}{#{vendor}}}" - end - next_line += ' \\\\\\hline' - table << next_line - table << ' \\endhead' - table << ' \\rowcolor{white}' - table << ' \\caption{Requests per method test case for all vendors}'\ - '\\label{table:evaluation_request_count}%' - table << ' \\endlastfoot' - - lines = [] - @all_tests.sort.each_with_index do |t_name, line| - @vendor_results.each do |v, results| - unless lines[line] - lines[line] = "#{t_name}" - if tests_with_wait.any? { |name| t_name.start_with?(name) } - ignore_tests_with_wait += 1 - lines[line] = "\\rowcolor{failedtablebg}#{lines[line]}" - end - end - ignore_req_with_wait[v] += results[t_name].to_i if tests_with_wait.any? { |name| t_name.start_with?(name) } - lines[line] << " & #{results[t_name]}" - end - end - - # format and print all lines - lines.each_with_index do |line, index| - if index != lines.length - 1 - table << " #{line.gsub(/_/, '\_')} \\\\\\hline" - else - # special treatment for the last line - table << " #{line.gsub(/_/, '\_')} \\\\\\hhline{|=|#{'=|' * @vendor_results.length}}" - end - end - - # print general statistics - table << " Tested methods & #{@vendor_results.collect { |_n, tests| tests.length }.join(' & ')} \\\\\\hline" - table << " Total vendor API requests & #{@vendor_results.collect do |_name, tests| - tests.map { |_key, value| value }.sum - end.join(' & ')} \\\\\\hline" - table << " Avg. vendor API requests per tested method & #{@vendor_results.collect do |_name, tests| - ((tests.map { |_key, value| value }.sum) / tests.length.to_f).round(2) - end.join(' & ')} \\\\\\hhline{|=|#{'=|' * @vendor_results.length}}" - - # sanitize stats, exclude methods with rspec wait repetitions - table << " Tested methods without repeated requests & #{@vendor_results.collect do |_n, tests| - tests.length - ignore_tests_with_wait - end.join(' & ')} \\\\\\hline" - - table << " Total vendor API requests without repeated requests & #{@vendor_results.collect do |name, tests| - tests.map { |_key, value| value }.sum - ignore_req_with_wait[name] - end.join(' & ')} \\\\\\hline" - table << ' Avg. vendor API requests per tested method without repeated requests & '\ - "#{@vendor_results.collect do |name, tests| - ((tests.map { |_key, value| value }.sum - ignore_req_with_wait[name]) / - (tests.length - ignore_tests_with_wait).to_f).round(2) - end.join(' & ')} \\\\\\hline" - - table << '\\end{longtable}' - table << '\\normalsize' - - # print lines - table.each { |line| puts line } - - # and save to file if requested - File.open(args.save_to, 'w') { |file| file.write table.join("\n") } if args.save_to - end - end - end -end +require 'rspec/core/rake_task' +require 'oj' + +namespace :evaluation do + namespace :requests do + namespace :count do + desc 'Load the data that will be used in the request evaluation tasks' + task load: :environment do + # TODO: choose API version + api_version = 'v1' + + adapter_dao = Nucleus::API::DB::AdapterDao.instance(api_version) + endpoint_dao = Nucleus::API::DB::EndpointDao.instance(api_version) + provider_dao = Nucleus::API::DB::ProviderDao.instance(api_version) + vendor_dao = Nucleus::API::DB::VendorDao.instance(api_version) + @vendor_results = {} + + adapter_dao.all.each do |adapter_index_entry| + vendor_name = vendor_dao.get(provider_dao.get(endpoint_dao.get(adapter_index_entry.id).provider).vendor).name + # from camel to sneak case + adapter_file_name = vendor_name.gsub(/\s/, '_').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') + .gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase + next if @vendor_results.key?(vendor_name) + adaper_results = {} + + method_recordings_dir = File.join('spec', 'adapter', 'recordings', api_version, adapter_file_name, + 'vcr_cassettes', 'with_valid_credentials', 'is_compliant_and') + Find.find(method_recordings_dir) do |file| + next if File.directory?(file) || File.basename(file) == '.DS_Store' + test = Pathname.new(file).relative_path_from(Pathname.new(method_recordings_dir)).to_s + # Load contents with the serializer, oj + cassette = ::Oj.load(File.read(file)) + adaper_results[test] = cassette['http_interactions'].length + end + @vendor_results[vendor_name] = adaper_results + end + + @all_tests = @vendor_results.flat_map { |_name, tests| tests.keys }.uniq + end + + task markdown: :load do + # table header + puts "Test / Vendor|#{@vendor_results.keys.join('|')}" + + # column styles + alignment = ':--' + @vendor_results.length.times { |_time| alignment << '|:-:' } + puts alignment + + lines = [] + @all_tests.sort.each_with_index do |test_name, line| + @vendor_results.each do |_vendor, results| + lines[line] = test_name.to_s unless lines[line] + lines[line] << "|#{results[test_name]}" + end + end + + lines.each do |line| + puts line + end + + # table header + puts '|' * @vendor_results.length + puts "Test / Vendor|#{@vendor_results.keys.join('|')}" + puts '|' * @vendor_results.length + puts "Tested methods|#{@vendor_results.collect { |_name, tests| tests.length }.join('|')}" + puts "Total vendor API requests|#{@vendor_results.collect do |_name, tests| + tests.map { |_key, value| value }.sum + end.join('|')}" + puts "Avg. vendor API requests per tested method|#{@vendor_results.collect do |_name, tests| + (tests.map { |_key, value| value }.sum / tests.length.to_f).round(2) + end.join('|')}" + end + + task :latex, [:save_to] => :load do |_t, args| + tests_with_wait = %w(app-actions/lifecycle/restart/succeeds + app-actions/lifecycle/start/succeeds_for_app_all_if_currently_stopped + app-actions/lifecycle/start/succeeds_for_app_min_if_currently_stopped + app-actions/lifecycle/stop/succeeds_for_app_all_if_currently_running + app-actions/lifecycle/stop/succeeds_for_app_min_if_currently_running + app-data/deploy/succeeds/and_app_with + app-data/rebuild/changes_the_release_version_property + app/web access/for_app_with_min_properties) + ignore_tests_with_wait = 0 + ignore_req_with_wait = {} + @vendor_results.each { |vendor, _| ignore_req_with_wait[vendor] = 0 } + + table = [] + table << '\\scriptsize' + table << "\\begin{longtable}{|L{12cm}|#{'c|' * @vendor_results.length}}" + next_line = ' \\multicolumn{1}{l}{\\Large{\\textbf{recorded test cassette}}}' + @vendor_results.keys.each do |vendor| + next_line += " & \\multicolumn{1}{l}{\\turn{60}{#{vendor}}}" + end + next_line += ' \\\\\\hline' + table << next_line + table << ' \\endhead' + table << ' \\rowcolor{white}' + table << ' \\caption{Requests per method test case for all vendors}'\ + '\\label{table:evaluation_request_count}%' + table << ' \\endlastfoot' + + lines = [] + @all_tests.sort.each_with_index do |t_name, line| + @vendor_results.each do |v, results| + unless lines[line] + lines[line] = t_name.to_s + if tests_with_wait.any? { |name| t_name.start_with?(name) } + ignore_tests_with_wait += 1 + lines[line] = "\\rowcolor{failedtablebg}#{lines[line]}" + end + end + ignore_req_with_wait[v] += results[t_name].to_i if tests_with_wait.any? { |name| t_name.start_with?(name) } + lines[line] << " & #{results[t_name]}" + end + end + + # format and print all lines + lines.each_with_index do |line, index| + table << if index != lines.length - 1 + " #{line.gsub(/_/, '\_')} \\\\\\hline" + else + # special treatment for the last line + " #{line.gsub(/_/, '\_')} \\\\\\hhline{|=|#{'=|' * @vendor_results.length}}" + end + end + + # print general statistics + table << " Tested methods & #{@vendor_results.collect { |_n, tests| tests.length }.join(' & ')} \\\\\\hline" + table << " Total vendor API requests & #{@vendor_results.collect do |_name, tests| + tests.map { |_key, value| value }.sum + end.join(' & ')} \\\\\\hline" + table << " Avg. vendor API requests per tested method & #{@vendor_results.collect do |_name, tests| + (tests.map { |_key, value| value }.sum / tests.length.to_f).round(2) + end.join(' & ')} \\\\\\hhline{|=|#{'=|' * @vendor_results.length}}" + + # sanitize stats, exclude methods with rspec wait repetitions + table << " Tested methods without repeated requests & #{@vendor_results.collect do |_n, tests| + tests.length - ignore_tests_with_wait + end.join(' & ')} \\\\\\hline" + + table << " Total vendor API requests without repeated requests & #{@vendor_results.collect do |name, tests| + tests.map { |_key, value| value }.sum - ignore_req_with_wait[name] + end.join(' & ')} \\\\\\hline" + table << ' Avg. vendor API requests per tested method without repeated requests & '\ + "#{@vendor_results.collect do |name, tests| + ((tests.map { |_key, value| value }.sum - ignore_req_with_wait[name]) / + (tests.length - ignore_tests_with_wait).to_f).round(2) + end.join(' & ')} \\\\\\hline" + + table << '\\end{longtable}' + table << '\\normalsize' + + # print lines + table.each { |line| puts line } + + # and save to file if requested + File.open(args.save_to, 'w') { |file| file.write table.join("\n") } if args.save_to + end + end + end +end