feat: [generativelanguage] Adds search grounding (#5828)

* feat: Add GenerateContentRespoinse.model_version feat: Add GroundingMetadata.web_search_queries docs: Small updates PiperOrigin-RevId: 698047489 Source-Link: googleapis/googleapis@c70bd1e Source-Link: googleapis/googleapis-gen@a7fd007 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWFpLWdlbmVyYXRpdmVsYW5ndWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiYTdmZDAwN2M1NDdmYTA3N2M5Nzg0NmU2MDg3YjdiNTdkM2EzN2YzZiJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: Adds search grounding feat: Adds `GenerateContentResponse.model_version` output docs: Some small updates. PiperOrigin-RevId: 698047851 Source-Link: googleapis/googleapis@7466a5b Source-Link: googleapis/googleapis-gen@edb373c Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWFpLWdlbmVyYXRpdmVsYW5ndWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiZWRiMzczYzQ2ZTc4NTQ4ZDNjMzJjNWE3YjU5NDlkZDhiNzI3ZmVlOCJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update gapic_cache_service_v1beta.ts --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: sofisl <[email protected]>
googleapis · Nov 21, 2024 · 0b27dc2 · 0b27dc2
1 parent 39dc74e
commit 0b27dc2
Show file tree

Hide file tree

Showing 10 changed files with 3,293 additions and 123 deletions.
diff --git a/...gle-ai-generativelanguage/protos/google/ai/generativelanguage/v1/generative_service.proto b/...gle-ai-generativelanguage/protos/google/ai/generativelanguage/v1/generative_service.proto
@@ -62,6 +62,10 @@ service GenerativeService {
     option (google.api.http) = {
       post: "/v1/{model=models/*}:streamGenerateContent"
       body: "*"
+      additional_bindings {
+        post: "/v1/{model=tunedModels/*}:streamGenerateContent"
+        body: "*"
+      }
     };
     option (google.api.method_signature) = "model,contents";
   }
@@ -260,7 +264,7 @@ message GenerationConfig {
   // values will cause the model to start repeating a common token  until it
   // hits the
   // [max_output_tokens][google.ai.generativelanguage.v1.GenerationConfig.max_output_tokens]
-  // limit: "...the the the the the...".
+  // limit.
   optional float frequency_penalty = 16
       [(google.api.field_behavior) = OPTIONAL];
 
@@ -340,6 +344,9 @@ message GenerateContentResponse {
 
   // Output only. Metadata on the generation requests' token usage.
   UsageMetadata usage_metadata = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The model version used to generate the response.
+  string model_version = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // A response candidate generated from the model.
@@ -412,7 +419,13 @@ message Candidate {
   // Output only. Token count for this candidate.
   int32 token_count = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
 
-  // Output only.
+  // Output only. Grounding metadata for the candidate.
+  //
+  // This field is populated for `GenerateContent` calls.
+  GroundingMetadata grounding_metadata = 9
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Average log probability score of the candidate.
   double avg_logprobs = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. Log-likelihood scores for the response tokens and top tokens
@@ -448,6 +461,99 @@ message LogprobsResult {
   repeated Candidate chosen_candidates = 2;
 }
 
+// Metadata related to retrieval in the grounding flow.
+message RetrievalMetadata {
+  // Optional. Score indicating how likely information from google search could
+  // help answer the prompt. The score is in the range [0, 1], where 0 is the
+  // least likely and 1 is the most likely. This score is only populated when
+  // google search grounding and dynamic retrieval is enabled. It will be
+  // compared to the threshold to determine whether to trigger google search.
+  float google_search_dynamic_retrieval_score = 2
+      [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Metadata returned to client when grounding is enabled.
+message GroundingMetadata {
+  // Optional. Google search entry for the following-up web searches.
+  optional SearchEntryPoint search_entry_point = 1
+      [(google.api.field_behavior) = OPTIONAL];
+
+  // List of supporting references retrieved from specified grounding source.
+  repeated GroundingChunk grounding_chunks = 2;
+
+  // List of grounding support.
+  repeated GroundingSupport grounding_supports = 3;
+
+  // Metadata related to retrieval in the grounding flow.
+  optional RetrievalMetadata retrieval_metadata = 4;
+
+  // Web search queries for the following-up web search.
+  repeated string web_search_queries = 5;
+}
+
+// Google search entry point.
+message SearchEntryPoint {
+  // Optional. Web content snippet that can be embedded in a web page or an app
+  // webview.
+  string rendered_content = 1 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Base64 encoded JSON representing array of <search term, search
+  // url> tuple.
+  bytes sdk_blob = 2 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Grounding chunk.
+message GroundingChunk {
+  // Chunk from the web.
+  message Web {
+    // URI reference of the chunk.
+    optional string uri = 1;
+
+    // Title of the chunk.
+    optional string title = 2;
+  }
+
+  // Chunk type.
+  oneof chunk_type {
+    // Grounding chunk from the web.
+    Web web = 1;
+  }
+}
+
+// Segment of the content.
+message Segment {
+  // Output only. The index of a Part object within its parent Content object.
+  int32 part_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Start index in the given Part, measured in bytes. Offset from
+  // the start of the Part, inclusive, starting at zero.
+  int32 start_index = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. End index in the given Part, measured in bytes. Offset from
+  // the start of the Part, exclusive, starting at zero.
+  int32 end_index = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The text corresponding to the segment from the response.
+  string text = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// Grounding support.
+message GroundingSupport {
+  // Segment of the content this support belongs to.
+  optional Segment segment = 1;
+
+  // A list of indices (into 'grounding_chunk') specifying the
+  // citations associated with the claim. For instance [1,3,4] means
+  // that grounding_chunk[1], grounding_chunk[3],
+  // grounding_chunk[4] are the retrieved content attributed to the claim.
+  repeated int32 grounding_chunk_indices = 2;
+
+  // Confidence score of the support references. Ranges from 0 to 1. 1 is the
+  // most confident. This list must have the same size as the
+  // grounding_chunk_indices.
+  repeated float confidence_scores = 3;
+}
+
 // Request containing the `Content` for the model to embed.
 message EmbedContentRequest {
   // Required. The model's resource name. This serves as an ID for the Model to

diff --git a/...ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/generative_service.proto b/...ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/generative_service.proto
@@ -74,6 +74,10 @@ service GenerativeService {
     option (google.api.http) = {
       post: "/v1beta/{model=models/*}:streamGenerateContent"
       body: "*"
+      additional_bindings {
+        post: "/v1beta/{model=tunedModels/*}:streamGenerateContent"
+        body: "*"
+      }
     };
     option (google.api.method_signature) = "model,contents";
   }
@@ -327,7 +331,7 @@ message GenerationConfig {
   // values will cause the model to start repeating a common token  until it
   // hits the
   // [max_output_tokens][google.ai.generativelanguage.v1beta.GenerationConfig.max_output_tokens]
-  // limit: "...the the the the the...".
+  // limit.
   optional float frequency_penalty = 16
       [(google.api.field_behavior) = OPTIONAL];
 
@@ -434,6 +438,9 @@ message GenerateContentResponse {
 
   // Output only. Metadata on the generation requests' token usage.
   UsageMetadata usage_metadata = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The model version used to generate the response.
+  string model_version = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // A response candidate generated from the model.
@@ -519,7 +526,7 @@ message Candidate {
   GroundingMetadata grounding_metadata = 9
       [(google.api.field_behavior) = OUTPUT_ONLY];
 
-  // Output only.
+  // Output only. Average log probability score of the candidate.
   double avg_logprobs = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. Log-likelihood scores for the response tokens and top tokens
@@ -624,6 +631,9 @@ message GroundingMetadata {
 
   // Metadata related to retrieval in the grounding flow.
   optional RetrievalMetadata retrieval_metadata = 4;
+
+  // Web search queries for the following-up web search.
+  repeated string web_search_queries = 5;
 }
 
 // Google search entry point.

diff --git a/...ogle-ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/model_service.proto b/...ogle-ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/model_service.proto
@@ -257,9 +257,9 @@ message UpdateTunedModelRequest {
   // Required. The tuned model to update.
   TunedModel tuned_model = 1 [(google.api.field_behavior) = REQUIRED];
 
-  // Required. The list of fields to update.
+  // Optional. The list of fields to update.
   google.protobuf.FieldMask update_mask = 2
-      [(google.api.field_behavior) = REQUIRED];
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Request to delete a TunedModel.

diff --git a/...google-ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/tuned_model.proto b/...google-ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/tuned_model.proto
@@ -219,16 +219,16 @@ message Hyperparameters {
 message Dataset {
   // Inline data or a reference to the data.
   oneof dataset {
-    // Optional. Inline examples.
+    // Optional. Inline examples with simple input/output text.
     TuningExamples examples = 1 [(google.api.field_behavior) = OPTIONAL];
   }
 }
 
 // A set of tuning examples. Can be training or validation data.
 message TuningExamples {
-  // Required. The examples. Example input can be for text or discuss, but all
-  // examples in a set must be of the same type.
-  repeated TuningExample examples = 1 [(google.api.field_behavior) = REQUIRED];
+  // The examples. Example input can be for text or discuss, but all examples
+  // in a set must be of the same type.
+  repeated TuningExample examples = 1;
 }
 
 // A single example for tuning.