# frozen_string_literal: true # Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Auto-generated by gapic-generator-ruby. DO NOT EDIT! module Google module Cloud module DiscoveryEngine module V1beta # An evaluation is a single execution (or run) of an evaluation process. It # encapsulates the state of the evaluation and the resulting data. # @!attribute [rw] name # @return [::String] # Identifier. The full resource name of the # {::Google::Cloud::DiscoveryEngine::V1beta::Evaluation Evaluation}, in the format # of `projects/{project}/locations/{location}/evaluations/{evaluation}`. # # This field must be a UTF-8 encoded string with a length limit of 1024 # characters. # @!attribute [rw] evaluation_spec # @return [::Google::Cloud::DiscoveryEngine::V1beta::Evaluation::EvaluationSpec] # Required. The specification of the evaluation. # @!attribute [r] quality_metrics # @return [::Google::Cloud::DiscoveryEngine::V1beta::QualityMetrics] # Output only. The metrics produced by the evaluation, averaged across all # {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuery SampleQuery}s in the # {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuerySet SampleQuerySet}. # # Only populated when the evaluation's state is SUCCEEDED. # @!attribute [r] state # @return [::Google::Cloud::DiscoveryEngine::V1beta::Evaluation::State] # Output only. The state of the evaluation. # @!attribute [r] error # @return [::Google::Rpc::Status] # Output only. The error that occurred during evaluation. Only populated when # the evaluation's state is FAILED. # @!attribute [r] create_time # @return [::Google::Protobuf::Timestamp] # Output only. Timestamp the # {::Google::Cloud::DiscoveryEngine::V1beta::Evaluation Evaluation} was created # at. # @!attribute [r] end_time # @return [::Google::Protobuf::Timestamp] # Output only. Timestamp the # {::Google::Cloud::DiscoveryEngine::V1beta::Evaluation Evaluation} was completed # at. # @!attribute [r] error_samples # @return [::Array<::Google::Rpc::Status>] # Output only. A sample of errors encountered while processing the request. class Evaluation include ::Google::Protobuf::MessageExts extend ::Google::Protobuf::MessageExts::ClassMethods # Describes the specification of the evaluation. # @!attribute [rw] search_request # @return [::Google::Cloud::DiscoveryEngine::V1beta::SearchRequest] # Required. The search request that is used to perform the evaluation. # # Only the following fields within SearchRequest are supported; if any # other fields are provided, an UNSUPPORTED error will be returned: # # * {::Google::Cloud::DiscoveryEngine::V1beta::SearchRequest#serving_config SearchRequest.serving_config} # * {::Google::Cloud::DiscoveryEngine::V1beta::SearchRequest#branch SearchRequest.branch} # * {::Google::Cloud::DiscoveryEngine::V1beta::SearchRequest#canonical_filter SearchRequest.canonical_filter} # * {::Google::Cloud::DiscoveryEngine::V1beta::SearchRequest#query_expansion_spec SearchRequest.query_expansion_spec} # * {::Google::Cloud::DiscoveryEngine::V1beta::SearchRequest#spell_correction_spec SearchRequest.spell_correction_spec} # * {::Google::Cloud::DiscoveryEngine::V1beta::SearchRequest#content_search_spec SearchRequest.content_search_spec} # * {::Google::Cloud::DiscoveryEngine::V1beta::SearchRequest#user_pseudo_id SearchRequest.user_pseudo_id} # @!attribute [rw] query_set_spec # @return [::Google::Cloud::DiscoveryEngine::V1beta::Evaluation::EvaluationSpec::QuerySetSpec] # Required. The specification of the query set. class EvaluationSpec include ::Google::Protobuf::MessageExts extend ::Google::Protobuf::MessageExts::ClassMethods # Describes the specification of the query set. # @!attribute [rw] sample_query_set # @return [::String] # Required. The full resource name of the # {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuerySet SampleQuerySet} # used for the evaluation, in the format of # `projects/{project}/locations/{location}/sampleQuerySets/{sampleQuerySet}`. class QuerySetSpec include ::Google::Protobuf::MessageExts extend ::Google::Protobuf::MessageExts::ClassMethods end end # Describes the state of an evaluation. module State # The evaluation is unspecified. STATE_UNSPECIFIED = 0 # The service is preparing to run the evaluation. PENDING = 1 # The evaluation is in progress. RUNNING = 2 # The evaluation completed successfully. SUCCEEDED = 3 # The evaluation failed. FAILED = 4 end end # Describes the metrics produced by the evaluation. # @!attribute [rw] doc_recall # @return [::Google::Cloud::DiscoveryEngine::V1beta::QualityMetrics::TopkMetrics] # Recall per document, at various top-k cutoff levels. # # Recall is the fraction of relevant documents retrieved out of all # relevant documents. # # Example (top-5): # * For a single # {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuery SampleQuery}, If 3 out # of 5 relevant documents are retrieved in the top-5, recall@5 = 3/5 = 0.6 # @!attribute [rw] doc_precision # @return [::Google::Cloud::DiscoveryEngine::V1beta::QualityMetrics::TopkMetrics] # Precision per document, at various top-k cutoff levels. # # Precision is the fraction of retrieved documents that are relevant. # # Example (top-5): # * For a single # {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuery SampleQuery}, If 4 out # of 5 retrieved documents in the top-5 are relevant, precision@5 = 4/5 = # 0.8 # @!attribute [rw] doc_ndcg # @return [::Google::Cloud::DiscoveryEngine::V1beta::QualityMetrics::TopkMetrics] # Normalized discounted cumulative gain (NDCG) per document, at various top-k # cutoff levels. # # NDCG measures the ranking quality, giving higher relevance to top # results. # # Example (top-3): # Suppose {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuery SampleQuery} # with three retrieved documents (D1, D2, D3) and binary relevance # judgements (1 for relevant, 0 for not relevant): # # Retrieved: [D3 (0), D1 (1), D2 (1)] # Ideal: [D1 (1), D2 (1), D3 (0)] # # Calculate NDCG@3 for each # {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuery SampleQuery}: # * DCG@3: 0/log2(1+1) + 1/log2(2+1) + 1/log2(3+1) = 1.13 # * Ideal DCG@3: 1/log2(1+1) + 1/log2(2+1) + 0/log2(3+1) = 1.63 # * NDCG@3: 1.13/1.63 = 0.693 # @!attribute [rw] page_recall # @return [::Google::Cloud::DiscoveryEngine::V1beta::QualityMetrics::TopkMetrics] # Recall per page, at various top-k cutoff levels. # # Recall is the fraction of relevant pages retrieved out of all relevant # pages. # # Example (top-5): # * For a single # {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuery SampleQuery}, if 3 out # of 5 relevant pages are retrieved in the top-5, recall@5 = 3/5 = 0.6 # @!attribute [rw] page_ndcg # @return [::Google::Cloud::DiscoveryEngine::V1beta::QualityMetrics::TopkMetrics] # Normalized discounted cumulative gain (NDCG) per page, at various top-k # cutoff levels. # # NDCG measures the ranking quality, giving higher relevance to top # results. # # Example (top-3): # Suppose {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuery SampleQuery} # with three retrieved pages (P1, P2, P3) and binary relevance judgements (1 # for relevant, 0 for not relevant): # # Retrieved: [P3 (0), P1 (1), P2 (1)] # Ideal: [P1 (1), P2 (1), P3 (0)] # # Calculate NDCG@3 for # {::Google::Cloud::DiscoveryEngine::V1beta::SampleQuery SampleQuery}: # * DCG@3: 0/log2(1+1) + 1/log2(2+1) + 1/log2(3+1) = 1.13 # * Ideal DCG@3: 1/log2(1+1) + 1/log2(2+1) + 0/log2(3+1) = 1.63 # * NDCG@3: 1.13/1.63 = 0.693 class QualityMetrics include ::Google::Protobuf::MessageExts extend ::Google::Protobuf::MessageExts::ClassMethods # Stores the metric values at specific top-k levels. # @!attribute [rw] top_1 # @return [::Float] # The top-1 value. # @!attribute [rw] top_3 # @return [::Float] # The top-3 value. # @!attribute [rw] top_5 # @return [::Float] # The top-5 value. # @!attribute [rw] top_10 # @return [::Float] # The top-10 value. class TopkMetrics include ::Google::Protobuf::MessageExts extend ::Google::Protobuf::MessageExts::ClassMethods end end end end end end