# frozen_string_literal: true module GitlabQuality module TestTooling module Report # Uses the API to create GitLab issues for any failed test coming from JSON test reports. # # - Takes the JSON test reports like rspec-*.json # - Takes a project where failed test issues should be created # - For every passed test in the report: # - Find issue by test hash or create a new issue if no issue was found # - Add a failure report in the "Failure reports" note class FailedTestIssue < ReportAsIssue include Concerns::GroupAndCategoryLabels include Concerns::IssueReports IDENTITY_LABELS = ['test', 'automation:bot-authored'].freeze NEW_ISSUE_LABELS = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze SEARCH_LABELS = ['test'].freeze FOUND_IN_MR_LABEL = '~"found:in MR"' FOUND_IN_MASTER_LABEL = '~"found:master"' REPORTS_DISCUSSION_HEADER = '### Failure reports' REPORT_SECTION_HEADER = '#### Failure reports' FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?.+))|(?.+))}m ISSUE_STACKTRACE_REGEX = /##### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*/m DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15 MultipleNotesFound = Class.new(StandardError) def initialize( token:, input_files:, base_issue_labels: nil, dry_run: false, project: nil, max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION, **_kwargs) super(token: token, input_files: input_files, project: project, dry_run: dry_run) @base_issue_labels = Set.new(base_issue_labels) @max_diff_ratio = max_diff_ratio.to_f end private attr_reader :base_issue_labels, :max_diff_ratio def run! puts "Reporting failed tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`." TestResults::Builder.new(files).test_results_per_file do |test_results| puts "=> Reporting #{test_results.count} tests in #{test_results.path}" process_test_results(test_results) end end def process_test_results(test_results) test_results.each do |test| next unless test_is_applicable?(test) puts " => Reporting failure for test '#{test.name}'..." issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS) issues << create_issue(test) if issues.empty? update_reports(issues, test) collect_issues(test, issues) end end def test_is_applicable?(test) test.status == 'failed' end def up_to_date_labels(test:, issue: nil, new_labels: Set.new) (base_issue_labels + super).to_a end def update_reports(issues, test) issues.each do |issue| puts " => Adding the failed test to the existing issue: #{issue.web_url}" add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue])) end end def add_report_to_issue(issue:, test:, related_issues:) # rubocop:disable Metrics/AbcSize: reports_discussion = find_or_create_reports_discussion(issue: issue) current_reports_note = find_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion) new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test) note_body = [ new_reports_list.to_s, identity_labels_quick_action, relate_issues_quick_actions(related_issues) ].join("\n") if current_reports_note gitlab.edit_issue_note( issue_iid: issue.iid, note_id: current_reports_note.id, note: note_body ) else gitlab.add_note_to_issue_discussion_as_thread( iid: issue.iid, discussion_id: reports_discussion.id, note: note_body ) end rescue MultipleNotesFound => e warn(e.message) end def find_or_create_reports_discussion(issue:) reports_discussion = existing_reports_discussion(issue: issue) return reports_discussion if reports_discussion gitlab.create_issue_discussion(iid: issue.iid, note: REPORTS_DISCUSSION_HEADER) end def existing_reports_discussion(issue:) gitlab.find_issue_discussions(iid: issue.iid).find do |discussion| next if discussion.individual_note next unless discussion.notes.first discussion.notes.first.body.start_with?(REPORTS_DISCUSSION_HEADER) end end def find_failure_discussion_note(issue:, test:, reports_discussion:) return unless reports_discussion relevant_notes = find_relevant_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion) return if relevant_notes.empty? best_matching_note, smaller_diff_ratio = relevant_notes.min_by { |_, diff_ratio| diff_ratio } raise(MultipleNotesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!)) unless relevant_notes.values.count(smaller_diff_ratio) == 1 # Re-instantiate a `Gitlab::ObjectifiedHash` object after having converted it to a hash in #find_relevant_failure_issues above. best_matching_note = Gitlab::ObjectifiedHash.new(best_matching_note) test.failure_issue ||= "#{issue.web_url}#note_#{best_matching_note.id}" best_matching_note end def find_relevant_failure_discussion_note(issue:, test:, reports_discussion:) return [] unless reports_discussion.notes.size > 1 clean_test_stacktrace = cleaned_stack_trace_from_test(test: test) # We're skipping the first note of the discussion as this is the "non-collapsible note", aka # the "header note", which doesn't contain any stack trace. reports_discussion.notes[1..].each_with_object({}) do |note, memo| clean_note_stacktrace = cleaned_stack_trace_from_note(issue: issue, note: note) diff_ratio = diff_ratio_between_test_and_note_stacktraces( issue: issue, note: note, test_stacktrace: clean_test_stacktrace, note_stacktrace: clean_note_stacktrace) memo[note.to_h] = diff_ratio if diff_ratio end end def cleaned_stack_trace_from_test(test:) sanitize_stacktrace(stacktrace: test.full_stacktrace, regex: FAILURE_STACKTRACE_REGEX) || test.full_stacktrace end def cleaned_stack_trace_from_note(issue:, note:) note_stacktrace = sanitize_stacktrace(stacktrace: note.body, regex: ISSUE_STACKTRACE_REGEX) return note_stacktrace if note_stacktrace puts " => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}#note_#{note.id}!" end def sanitize_stacktrace(stacktrace:, regex:) stacktrace_match = stacktrace.match(regex) if stacktrace_match stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip else puts " => [DEBUG] Stacktrace doesn't match the regex (#{regex})!" end end def diff_ratio_between_test_and_note_stacktraces(issue:, note:, test_stacktrace:, note_stacktrace:) return if note_stacktrace.nil? stack_trace_comparator = StackTraceComparator.new(test_stacktrace, note_stacktrace) if stack_trace_comparator.lower_or_equal_to_diff_ratio?(max_diff_ratio) puts " => [DEBUG] Note #{issue.web_url}#note_#{note.id} has an acceptable diff ratio of #{stack_trace_comparator.diff_percent}%." # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key. # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key. # See: # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995 # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494 stack_trace_comparator.diff_ratio else puts " => [DEBUG] Found note #{issue.web_url}#note_#{note.id} but stacktraces are too different (#{stack_trace_comparator.diff_percent}%).\n" puts " => [DEBUG] Issue stacktrace:\n----------------\n#{note_stacktrace}\n----------------\n" puts " => [DEBUG] Failure stacktrace:\n----------------\n#{test_stacktrace}\n----------------\n" end end def add_report_for_test(current_reports_content:, test:) increment_reports( current_reports_content: current_reports_content, test: test, reports_section_header: REPORT_SECTION_HEADER, item_extra_content: found_label, reports_extra_content: "##### Stack trace\n\n```\n#{test.full_stacktrace}\n```" ) end def found_label if ENV.key?('CI_MERGE_REQUEST_IID') FOUND_IN_MR_LABEL else FOUND_IN_MASTER_LABEL end end def identity_labels_quick_action labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ') %(/label #{labels_list}) end def relate_issues_quick_actions(issues) issues.map do |issue| "/relate #{issue.web_url}" end.join("\n") end end end end end