From 4fb48c0d356d83ced7b7d90422cb9d6908d5868b Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Mon, 29 Apr 2024 14:08:20 +0200
Subject: [PATCH 01/13] feat(vertexai): Multimodal snippets using Gemini 1.5

---
 .../MultimodalAudioInputSummarization.java    | 64 +++++++++++++++++++
 .../MultimodalAudioInputTranscription.java    | 63 ++++++++++++++++++
 .../gemini/MultimodalVideoAudioInput.java     | 64 +++++++++++++++++++
 3 files changed, 191 insertions(+)
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputSummarization.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputTranscription.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputSummarization.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputSummarization.java
new file mode 100644
index 00000000000..05a9f1fb98d
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputSummarization.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_audio_summarization]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.generativeai.ContentMaker;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.PartMaker;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+
+import java.io.IOException;
+
+public class MultimodalAudioInputSummarization {
+
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "your-google-cloud-project-id";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-pro-preview-0409";
+
+    summarizeAudio(projectId, location, modelName);
+  }
+
+  // Analyzes the given audio input.
+  public static String summarizeAudio(String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      String audioUri = "gs://cloud-samples-data/generative-ai/audio/pixel.mp3";
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
+      GenerateContentResponse response = model.generateContent(
+          ContentMaker.fromMultiModalData(
+              "Please provide a summary for the audio.\n" +
+                  "Provide chapter titles with timestamps, be concise and short, no need to provide chapter summaries.\n" +
+                  "Do not make up any information that is not part of the audio and do not be verbose.",
+              PartMaker.fromMimeTypeAndData("audio/mp3", audioUri)
+          ));
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_audio_summarization]
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputTranscription.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputTranscription.java
new file mode 100644
index 00000000000..e3e6bbdb467
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputTranscription.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_audio_transcription]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.generativeai.ContentMaker;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.PartMaker;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+
+import java.io.IOException;
+
+public class MultimodalAudioInputTranscription {
+
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "your-google-cloud-project-id";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-pro-preview-0409";
+
+    transcribeAudio(projectId, location, modelName);
+  }
+
+  // Analyzes the given audio input.
+  public static String transcribeAudio(String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      String audioUri = "gs://cloud-samples-data/generative-ai/audio/pixel.mp3";
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
+      GenerateContentResponse response = model.generateContent(
+          ContentMaker.fromMultiModalData(
+              "Can you transcribe this interview, in the format of timecode, speaker, caption.\n" +
+                  "Use speaker A, speaker B, etc. to identify speakers.",
+              PartMaker.fromMimeTypeAndData("audio/mp3", audioUri)
+          ));
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_audio_transcription]
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java
new file mode 100644
index 00000000000..05b280e5357
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_video_with_audio]
+
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.generativeai.ContentMaker;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.PartMaker;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+
+import java.io.IOException;
+
+public class MultimodalVideoAudioInput {
+
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "your-google-cloud-project-id";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-pro-preview-0409";
+
+    multimodalVideoAudioInput(projectId, location, modelName);
+  }
+
+  // Analyzes the given audio input.
+  public static String multimodalVideoAudioInput(String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      String videoUri = "gs://cloud-samples-data/generative-ai/video/pixel8.mp4";
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
+      GenerateContentResponse response = model.generateContent(
+          ContentMaker.fromMultiModalData(
+              "Provide a description of the video.\n" +
+                  "The description should also contain anything important which people say in the video.",
+              PartMaker.fromMimeTypeAndData("video/mp4", videoUri)
+          ));
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_video_with_audio]

From 3462f9a472e7bfcd4edea254d672bd7e1bd887e9 Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Mon, 29 Apr 2024 14:31:30 +0200
Subject: [PATCH 02/13] feat(vertexai): Multimodal snippets using Gemini 1.5
 (checkstyle and header tweaks)

---
 .../MultimodalAudioInputSummarization.java    | 64 -------------------
 .../MultimodalAudioInputTranscription.java    | 63 ------------------
 .../gemini/MultimodalVideoAudioInput.java     | 11 ++--
 3 files changed, 5 insertions(+), 133 deletions(-)
 delete mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputSummarization.java
 delete mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputTranscription.java

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputSummarization.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputSummarization.java
deleted file mode 100644
index 05a9f1fb98d..00000000000
--- a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputSummarization.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2023 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package vertexai.gemini;
-
-// [START generativeaionvertexai_gemini_audio_summarization]
-import com.google.cloud.vertexai.VertexAI;
-import com.google.cloud.vertexai.api.GenerateContentResponse;
-import com.google.cloud.vertexai.generativeai.ContentMaker;
-import com.google.cloud.vertexai.generativeai.GenerativeModel;
-import com.google.cloud.vertexai.generativeai.PartMaker;
-import com.google.cloud.vertexai.generativeai.ResponseHandler;
-
-import java.io.IOException;
-
-public class MultimodalAudioInputSummarization {
-
-  public static void main(String[] args) throws IOException {
-    // TODO(developer): Replace these variables before running the sample.
-    String projectId = "your-google-cloud-project-id";
-    String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
-
-    summarizeAudio(projectId, location, modelName);
-  }
-
-  // Analyzes the given audio input.
-  public static String summarizeAudio(String projectId, String location, String modelName)
-      throws IOException {
-    // Initialize client that will be used to send requests. This client only needs
-    // to be created once, and can be reused for multiple requests.
-    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
-      String audioUri = "gs://cloud-samples-data/generative-ai/audio/pixel.mp3";
-
-      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
-      GenerateContentResponse response = model.generateContent(
-          ContentMaker.fromMultiModalData(
-              "Please provide a summary for the audio.\n" +
-                  "Provide chapter titles with timestamps, be concise and short, no need to provide chapter summaries.\n" +
-                  "Do not make up any information that is not part of the audio and do not be verbose.",
-              PartMaker.fromMimeTypeAndData("audio/mp3", audioUri)
-          ));
-
-      String output = ResponseHandler.getText(response);
-      System.out.println(output);
-
-      return output;
-    }
-  }
-}
-// [END generativeaionvertexai_gemini_audio_summarization]
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputTranscription.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputTranscription.java
deleted file mode 100644
index e3e6bbdb467..00000000000
--- a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAudioInputTranscription.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright 2023 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package vertexai.gemini;
-
-// [START generativeaionvertexai_gemini_audio_transcription]
-import com.google.cloud.vertexai.VertexAI;
-import com.google.cloud.vertexai.api.GenerateContentResponse;
-import com.google.cloud.vertexai.generativeai.ContentMaker;
-import com.google.cloud.vertexai.generativeai.GenerativeModel;
-import com.google.cloud.vertexai.generativeai.PartMaker;
-import com.google.cloud.vertexai.generativeai.ResponseHandler;
-
-import java.io.IOException;
-
-public class MultimodalAudioInputTranscription {
-
-  public static void main(String[] args) throws IOException {
-    // TODO(developer): Replace these variables before running the sample.
-    String projectId = "your-google-cloud-project-id";
-    String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
-
-    transcribeAudio(projectId, location, modelName);
-  }
-
-  // Analyzes the given audio input.
-  public static String transcribeAudio(String projectId, String location, String modelName)
-      throws IOException {
-    // Initialize client that will be used to send requests. This client only needs
-    // to be created once, and can be reused for multiple requests.
-    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
-      String audioUri = "gs://cloud-samples-data/generative-ai/audio/pixel.mp3";
-
-      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
-      GenerateContentResponse response = model.generateContent(
-          ContentMaker.fromMultiModalData(
-              "Can you transcribe this interview, in the format of timecode, speaker, caption.\n" +
-                  "Use speaker A, speaker B, etc. to identify speakers.",
-              PartMaker.fromMimeTypeAndData("audio/mp3", audioUri)
-          ));
-
-      String output = ResponseHandler.getText(response);
-      System.out.println(output);
-
-      return output;
-    }
-  }
-}
-// [END generativeaionvertexai_gemini_audio_transcription]
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java
index 05b280e5357..3688aff7c86 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Google LLC
+ * Copyright 2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@
 import com.google.cloud.vertexai.generativeai.GenerativeModel;
 import com.google.cloud.vertexai.generativeai.PartMaker;
 import com.google.cloud.vertexai.generativeai.ResponseHandler;
-
 import java.io.IOException;
 
 public class MultimodalVideoAudioInput {
@@ -35,11 +34,11 @@ public static void main(String[] args) throws IOException {
     String location = "us-central1";
     String modelName = "gemini-1.5-pro-preview-0409";
 
-    multimodalVideoAudioInput(projectId, location, modelName);
+    videoAudioInput(projectId, location, modelName);
   }
 
   // Analyzes the given audio input.
-  public static String multimodalVideoAudioInput(String projectId, String location, String modelName)
+  public static String videoAudioInput(String projectId, String location, String modelName)
       throws IOException {
     // Initialize client that will be used to send requests. This client only needs
     // to be created once, and can be reused for multiple requests.
@@ -49,8 +48,8 @@ public static String multimodalVideoAudioInput(String projectId, String location
       GenerativeModel model = new GenerativeModel(modelName, vertexAI);
       GenerateContentResponse response = model.generateContent(
           ContentMaker.fromMultiModalData(
-              "Provide a description of the video.\n" +
-                  "The description should also contain anything important which people say in the video.",
+              "Provide a description of the video.\n The description should also "
+                  + "contain anything important which people say in the video.",
               PartMaker.fromMimeTypeAndData("video/mp4", videoUri)
           ));
 

From fde6d50f7badbe0fb321cb1e20332412f635adf6 Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Mon, 29 Apr 2024 18:19:22 +0200
Subject: [PATCH 03/13] feat(vertexai): Multimodal snippets using Gemini 1.5
 (incorporating feedback, fixing region tags, and clarifying comments)

---
 .../gemini/MultimodalVideoAudioInput.java     | 63 -------------------
 1 file changed, 63 deletions(-)
 delete mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java
deleted file mode 100644
index 3688aff7c86..00000000000
--- a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoAudioInput.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright 2024 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package vertexai.gemini;
-
-// [START generativeaionvertexai_gemini_video_with_audio]
-
-import com.google.cloud.vertexai.VertexAI;
-import com.google.cloud.vertexai.api.GenerateContentResponse;
-import com.google.cloud.vertexai.generativeai.ContentMaker;
-import com.google.cloud.vertexai.generativeai.GenerativeModel;
-import com.google.cloud.vertexai.generativeai.PartMaker;
-import com.google.cloud.vertexai.generativeai.ResponseHandler;
-import java.io.IOException;
-
-public class MultimodalVideoAudioInput {
-
-  public static void main(String[] args) throws IOException {
-    // TODO(developer): Replace these variables before running the sample.
-    String projectId = "your-google-cloud-project-id";
-    String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
-
-    videoAudioInput(projectId, location, modelName);
-  }
-
-  // Analyzes the given audio input.
-  public static String videoAudioInput(String projectId, String location, String modelName)
-      throws IOException {
-    // Initialize client that will be used to send requests. This client only needs
-    // to be created once, and can be reused for multiple requests.
-    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
-      String videoUri = "gs://cloud-samples-data/generative-ai/video/pixel8.mp4";
-
-      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
-      GenerateContentResponse response = model.generateContent(
-          ContentMaker.fromMultiModalData(
-              "Provide a description of the video.\n The description should also "
-                  + "contain anything important which people say in the video.",
-              PartMaker.fromMimeTypeAndData("video/mp4", videoUri)
-          ));
-
-      String output = ResponseHandler.getText(response);
-      System.out.println(output);
-
-      return output;
-    }
-  }
-}
-// [END generativeaionvertexai_gemini_video_with_audio]

From b1274d9d61042830e4db21610afbd0cd318b9d4d Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Fri, 24 May 2024 18:44:28 +0200
Subject: [PATCH 04/13] feat(vertexai): Use gemini-1.5-flash-001 instead of the
 preview model

---
 .../gemini/AudioInputSummarization.java       |  2 +-
 .../gemini/AudioInputTranscription.java       |  2 +-
 .../vertexai/gemini/MultimodalAllInput.java   |  2 +-
 .../main/java/vertexai/gemini/PdfInput.java   |  2 +-
 .../vertexai/gemini/VideoInputWithAudio.java  |  2 +-
 .../gemini/WithSystemInstruction.java         |  2 +-
 .../test/java/vertexai/gemini/SnippetsIT.java | 22 +++++++++----------
 7 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/AudioInputSummarization.java b/vertexai/snippets/src/main/java/vertexai/gemini/AudioInputSummarization.java
index cc9f7a1423d..8c8ca317848 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/AudioInputSummarization.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/AudioInputSummarization.java
@@ -31,7 +31,7 @@ public static void main(String[] args) throws IOException {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
+    String modelName = "gemini-1.5-flash-001";
 
     summarizeAudio(projectId, location, modelName);
   }
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/AudioInputTranscription.java b/vertexai/snippets/src/main/java/vertexai/gemini/AudioInputTranscription.java
index 292f36519d7..b4c61384228 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/AudioInputTranscription.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/AudioInputTranscription.java
@@ -31,7 +31,7 @@ public static void main(String[] args) throws IOException {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
+    String modelName = "gemini-1.5-flash-001";
 
     transcribeAudio(projectId, location, modelName);
   }
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAllInput.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAllInput.java
index f1166cd3a3a..909f82ba3bf 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAllInput.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalAllInput.java
@@ -32,7 +32,7 @@ public static void main(String[] args) throws IOException {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
+    String modelName = "gemini-1.5-flash-001";
 
     multimodalAllInput(projectId, location, modelName);
   }
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/PdfInput.java b/vertexai/snippets/src/main/java/vertexai/gemini/PdfInput.java
index 18d87638371..cd83f6419ff 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/PdfInput.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/PdfInput.java
@@ -32,7 +32,7 @@ public static void main(String[] args) throws IOException {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
+    String modelName = "gemini-1.5-flash-001";
 
     pdfInput(projectId, location, modelName);
   }
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/VideoInputWithAudio.java b/vertexai/snippets/src/main/java/vertexai/gemini/VideoInputWithAudio.java
index 134d3efdca2..75566fe5df4 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/VideoInputWithAudio.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/VideoInputWithAudio.java
@@ -32,7 +32,7 @@ public static void main(String[] args) throws IOException {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
+    String modelName = "gemini-1.5-flash-001";
 
     videoAudioInput(projectId, location, modelName);
   }
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/WithSystemInstruction.java b/vertexai/snippets/src/main/java/vertexai/gemini/WithSystemInstruction.java
index 605e5304bbe..4ae867f0842 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/WithSystemInstruction.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/WithSystemInstruction.java
@@ -29,7 +29,7 @@ public static void main(String[] args) throws Exception {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.5-pro-preview-0409";
+    String modelName = "gemini-1.5-flash-001";
 
     String output = translateToFrench(projectId, location, modelName);
     System.out.println(output);
diff --git a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
index c998c30642f..b91686af77d 100644
--- a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
+++ b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
@@ -42,14 +42,13 @@
 @RunWith(JUnit4.class)
 public class SnippetsIT {
 
-  public static final String GEMINI_ULTRA_VISION = "gemini-1.0-ultra-vision";
   private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT");
   private static final String LOCATION = "us-central1";
   private static final String GEMINI_PRO_VISION = "gemini-1.0-pro-vision-001";
   private static final String GEMINI_PRO = "gemini-1.0-pro-002";
-  private static final String GEMINI_PRO_1_5 = "gemini-1.5-pro-preview-0409";
+  private static final String GEMINI_FLASH = "gemini-1.5-flash-001";
   private static final int MAX_ATTEMPT_COUNT = 3;
-  private static final int INITIAL_BACKOFF_MILLIS = 120000; // 2 minutes
+  private static final int INITIAL_BACKOFF_MILLIS = 12000; // 2 minutes
 
   @Rule
   public final MultipleAttemptsRule multipleAttemptsRule =
@@ -160,10 +159,9 @@ public void testMultimodalVideoInput() throws IOException {
     assertThat(bout.toString()).contains("Zoo");
   }
 
-  @Ignore("Don't test until ultra launch")
   @Test
   public void testMultiTurnMultimodal() throws IOException {
-    MultiTurnMultimodal.multiTurnMultimodal(PROJECT_ID, LOCATION, GEMINI_ULTRA_VISION);
+    MultiTurnMultimodal.multiTurnMultimodal(PROJECT_ID, LOCATION, GEMINI_FLASH);
     assertThat(bout.toString()).contains("scones");
   }
 
@@ -236,7 +234,7 @@ public void testFunctionCalling() throws Exception {
 
   @Test
   public void testAudioInputSummary() throws IOException {
-    String output = AudioInputSummarization.summarizeAudio(PROJECT_ID, LOCATION, GEMINI_PRO_1_5);
+    String output = AudioInputSummarization.summarizeAudio(PROJECT_ID, LOCATION, GEMINI_FLASH);
 
     assertThat(output).ignoringCase().contains("Pixel");
     assertThat(output).ignoringCase().contains("feature");
@@ -244,7 +242,7 @@ public void testAudioInputSummary() throws IOException {
 
   @Test
   public void testAudioInputTranscription() throws IOException {
-    String output = AudioInputTranscription.transcribeAudio(PROJECT_ID, LOCATION, GEMINI_PRO_1_5);
+    String output = AudioInputTranscription.transcribeAudio(PROJECT_ID, LOCATION, GEMINI_FLASH);
 
     assertThat(output).ignoringCase().contains("Pixel");
     assertThat(output).ignoringCase().contains("feature");
@@ -252,7 +250,7 @@ public void testAudioInputTranscription() throws IOException {
 
   @Test
   public void testVideoAudioInput() throws IOException {
-    String output = VideoInputWithAudio.videoAudioInput(PROJECT_ID, LOCATION, GEMINI_PRO_1_5);
+    String output = VideoInputWithAudio.videoAudioInput(PROJECT_ID, LOCATION, GEMINI_FLASH);
 
     assertThat(output).ignoringCase().contains("Pixel");
     assertThat(output).ignoringCase().contains("Tokyo");
@@ -260,21 +258,21 @@ public void testVideoAudioInput() throws IOException {
 
   @Test
   public void testAllModalityInputs() throws IOException {
-    String output = MultimodalAllInput.multimodalAllInput(PROJECT_ID, LOCATION, GEMINI_PRO_1_5);
+    String output = MultimodalAllInput.multimodalAllInput(PROJECT_ID, LOCATION, GEMINI_FLASH);
 
-    assertThat(output).ignoringCase().contains("0:49");
+    assertThat(output).ignoringCase().contains("0:4");
   }
 
   @Test
   public void testPdfInput() throws IOException {
-    String output = PdfInput.pdfInput(PROJECT_ID, LOCATION, GEMINI_PRO_1_5);
+    String output = PdfInput.pdfInput(PROJECT_ID, LOCATION, GEMINI_FLASH);
 
     assertThat(output).ignoringCase().contains("Gemini");
   }
 
   @Test
   public void testSystemInstruction() throws Exception {
-    String output = WithSystemInstruction.translateToFrench(PROJECT_ID, LOCATION, GEMINI_PRO_1_5);
+    String output = WithSystemInstruction.translateToFrench(PROJECT_ID, LOCATION, GEMINI_FLASH);
 
     assertThat(output).ignoringCase().contains("bagels");
     assertThat(output).ignoringCase().contains("aime");

From f9b72065391a4e03df7d907949dbcad26fc366c8 Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Fri, 24 May 2024 19:03:46 +0200
Subject: [PATCH 05/13] Reset the initial backoff time to its original setting

---
 vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
index b91686af77d..69254d634c8 100644
--- a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
+++ b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
@@ -48,7 +48,7 @@ public class SnippetsIT {
   private static final String GEMINI_PRO = "gemini-1.0-pro-002";
   private static final String GEMINI_FLASH = "gemini-1.5-flash-001";
   private static final int MAX_ATTEMPT_COUNT = 3;
-  private static final int INITIAL_BACKOFF_MILLIS = 12000; // 2 minutes
+  private static final int INITIAL_BACKOFF_MILLIS = 120000; // 2 minutes
 
   @Rule
   public final MultipleAttemptsRule multipleAttemptsRule =

From 46cea5f98f22189ea3b15b04585749e54feb104a Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Tue, 4 Jun 2024 11:01:57 +0200
Subject: [PATCH 06/13] feat(vertexai): Updated to use Gemini 1.5 Flash and new
 samples

- added some advanced variants of existing snippets
- added grounding with private data and with Google Search web results
- added new multimodality streaming/non-streaming examples
- updated some samples with Gemini 1.5 Flash
- updated/added some region tags for integration in the documentation
---
 .../java/vertexai/gemini/FunctionCalling.java |  2 +-
 .../java/vertexai/gemini/GetTokenCount.java   | 56 ++++++++++--
 .../vertexai/gemini/GroundingWithData.java    | 91 +++++++++++++++++++
 .../main/java/vertexai/gemini/Multimodal.java | 63 +++++++++++++
 .../vertexai/gemini/MultimodalVideoInput.java |  2 +-
 .../java/vertexai/gemini/QuestionAnswer.java  |  2 +-
 .../vertexai/gemini/StreamingMultimodal.java  | 58 ++++++++++++
 .../gemini/StreamingQuestionAnswer.java       |  4 +-
 .../test/java/vertexai/gemini/SnippetsIT.java | 44 ++++++++-
 9 files changed, 306 insertions(+), 16 deletions(-)
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithData.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/Multimodal.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/StreamingMultimodal.java

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/FunctionCalling.java b/vertexai/snippets/src/main/java/vertexai/gemini/FunctionCalling.java
index dd9cabdec11..3d8f875515e 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/FunctionCalling.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/FunctionCalling.java
@@ -38,7 +38,7 @@ public static void main(String[] args) throws IOException {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.0-pro-002";
+    String modelName = "gemini-1.5-flash-001";
 
     String promptText = "What's the weather like in Paris?";
 
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java b/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
index 7603c775fe3..da5ee96100f 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
@@ -17,36 +17,78 @@
 package vertexai.gemini;
 
 // [START aiplatform_gemini_token_count]
+// [START generativeaionvertexai_gemini_token_count_advanced]
 import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.Content;
 import com.google.cloud.vertexai.api.CountTokensResponse;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.generativeai.ContentMaker;
 import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.PartMaker;
 import java.io.IOException;
 // [END aiplatform_gemini_token_count]
+// [END generativeaionvertexai_gemini_token_count_advanced]
 
 public class GetTokenCount {
   public static void main(String[] args) throws IOException {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.0-pro-vision-001";
+    String modelName = "gemini-1.5-flash-001";
 
-    String textPrompt = "Why is the sky blue?";
-    getTokenCount(projectId, location, modelName, textPrompt);
+    getTokenCount(projectId, location, modelName);
+    getMediaTokenCount(projectId, location, modelName);
   }
 
   // [START aiplatform_gemini_token_count]
-  public static int getTokenCount(String projectId, String location, String modelName,
-                                  String textPrompt)
+  public static int getTokenCount(String projectId, String location, String modelName)
       throws IOException {
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       GenerativeModel model = new GenerativeModel(modelName, vertexAI);
+
+      String textPrompt = "Why is the sky blue?";
       CountTokensResponse response = model.countTokens(textPrompt);
 
+      int promptTokenCount = response.getTotalTokens();
+      int promptCharCount = response.getTotalBillableCharacters();
+
+      System.out.println("Prompt token Count: " + promptTokenCount);
+      System.out.println("Prompt billable character count: " + promptCharCount);
+
+      GenerateContentResponse contentResponse = model.generateContent(textPrompt);
+
+      int tokenCount = contentResponse.getUsageMetadata().getPromptTokenCount();
+      int candidateTokenCount = contentResponse.getUsageMetadata().getCandidatesTokenCount();
+      int totalTokenCount = contentResponse.getUsageMetadata().getTotalTokenCount();
+
+      System.out.println("Prompt token Count: " + tokenCount);
+      System.out.println("Candidate Token Count: " + candidateTokenCount);
+      System.out.println("Total token Count: " + totalTokenCount);
+
+      return promptTokenCount;
+    }
+  }
+  // [END aiplatform_gemini_token_count]
+
+  // [START generativeaionvertexai_gemini_token_count_advanced]
+  public static int getMediaTokenCount(String projectId, String location, String modelName)
+      throws IOException {
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
+
+      Content content = ContentMaker.fromMultiModalData(
+          "Provide a description of the video.",
+          PartMaker.fromMimeTypeAndData(
+              "video/mp4", "gs://cloud-samples-data/generative-ai/video/pixel8.mp4")
+      );
+
+      CountTokensResponse response = model.countTokens(content);
+
       int tokenCount = response.getTotalTokens();
-      System.out.println("There are " + tokenCount + " tokens in the prompt.");
+      System.out.println("Token count: " + tokenCount);
 
       return tokenCount;
     }
   }
-  // [END aiplatform_gemini_token_count]
+  // [END generativeaionvertexai_gemini_token_count_advanced]
 }
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithData.java b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithData.java
new file mode 100644
index 00000000000..27add469260
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithData.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_grounding_private_data_basic]
+// [START generativeaionvertexai_grounding_public_data_basic]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.api.GoogleSearchRetrieval;
+import com.google.cloud.vertexai.api.GroundingMetadata;
+import com.google.cloud.vertexai.api.Retrieval;
+import com.google.cloud.vertexai.api.Tool;
+import com.google.cloud.vertexai.api.VertexAISearch;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+import java.io.IOException;
+import java.util.Collections;
+// [END generativeaionvertexai_grounding_private_data_basic]
+// [END generativeaionvertexai_grounding_public_data_basic]
+
+public class GroundingWithData {
+
+  public static String groundWithPublicData(String projectId, String location, String modelName)
+      throws IOException {
+    // [START generativeaionvertexai_grounding_public_data_basic]
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      Tool googleSearchTool = Tool.newBuilder()
+          .setGoogleSearchRetrieval(GoogleSearchRetrieval.newBuilder().setDisableAttribution(false))
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI).withTools(
+          Collections.singletonList(googleSearchTool)
+      );
+
+      GenerateContentResponse response = model.generateContent("Why is the sky blue?");
+
+      GroundingMetadata groundingMetadata = response.getCandidates(0).getGroundingMetadata();
+      String answer = ResponseHandler.getText(response);
+
+      System.out.println("Answer: " + answer);
+      System.out.println("Grounding metadata: " + groundingMetadata);
+      // [END generativeaionvertexai_grounding_public_data_basic]
+
+      return answer;
+    }
+  }
+
+  public static String groundWithPrivateData(String projectId, String location, String modelName,
+                                             String datastoreId)
+      throws IOException {
+    // [START generativeaionvertexai_grounding_private_data_basic]
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      Tool datastoreTool = Tool.newBuilder()
+          .setRetrieval(
+              Retrieval.newBuilder()
+                  .setVertexAiSearch(VertexAISearch.newBuilder().setDatastore(datastoreId))
+                  .setDisableAttribution(false))
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI).withTools(
+          Collections.singletonList(datastoreTool)
+      );
+
+      GenerateContentResponse response = model.generateContent(
+          "How do I make an appointment to renew my driver's license?");
+
+      GroundingMetadata groundingMetadata = response.getCandidates(0).getGroundingMetadata();
+      String answer = ResponseHandler.getText(response);
+
+      System.out.println("Answer: " + answer);
+      System.out.println("Grounding metadata: " + groundingMetadata);
+      // [END generativeaionvertexai_grounding_private_data_basic]
+
+      return answer;
+    }
+  }
+}
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/Multimodal.java b/vertexai/snippets/src/main/java/vertexai/gemini/Multimodal.java
new file mode 100644
index 00000000000..816e371f80b
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/Multimodal.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_non_stream_multimodality_basic]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.generativeai.ContentMaker;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.PartMaker;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+
+public class Multimodal {
+  public static void main(String[] args) throws Exception {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "your-google-cloud-project-id";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-flash-001";
+
+    String output = nonStreamingMultimodal(projectId, location, modelName);
+    System.out.println(output);
+  }
+
+  // Ask a simple question and get the response.
+  public static String nonStreamingMultimodal(String projectId, String location, String modelName)
+      throws Exception {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
+
+      String videoUri = "gs://cloud-samples-data/video/animals.mp4";
+      String imgUri = "gs://cloud-samples-data/generative-ai/image/character.jpg";
+
+      // Get the response from the model.
+      GenerateContentResponse response = model.generateContent(
+          ContentMaker.fromMultiModalData(
+              PartMaker.fromMimeTypeAndData("video/mp4", videoUri),
+              PartMaker.fromMimeTypeAndData("image/jpeg", imgUri),
+              "Are this video and image correlated?"
+          ));
+
+      // Extract the generated text from the model's response.
+      String output = ResponseHandler.getText(response);
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_non_stream_multimodality_basic]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoInput.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoInput.java
index c3f049d06d5..8543b564a4e 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoInput.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoInput.java
@@ -31,7 +31,7 @@ public static void main(String[] args) throws IOException {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.0-pro-vision-001";
+    String modelName = "gemini-1.5-flash-001";
 
     multimodalVideoInput(projectId, location, modelName);
   }
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/QuestionAnswer.java b/vertexai/snippets/src/main/java/vertexai/gemini/QuestionAnswer.java
index 6a6f50da7f4..732e0f558a9 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/QuestionAnswer.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/QuestionAnswer.java
@@ -28,7 +28,7 @@ public static void main(String[] args) throws Exception {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.0-pro-vision-001";
+    String modelName = "gemini-1.5-flash-001";
 
     String output = simpleQuestion(projectId, location, modelName);
     System.out.println(output);
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/StreamingMultimodal.java b/vertexai/snippets/src/main/java/vertexai/gemini/StreamingMultimodal.java
new file mode 100644
index 00000000000..051e8a817d3
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/StreamingMultimodal.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_stream_multimodality_basic]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.generativeai.ContentMaker;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.PartMaker;
+
+public class StreamingMultimodal {
+  public static void main(String[] args) throws Exception {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "your-google-cloud-project-id";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-flash-001";
+
+    streamingMultimodal(projectId, location, modelName);
+  }
+
+  // Ask a simple question and get the response via streaming.
+  public static void streamingMultimodal(String projectId, String location, String modelName)
+      throws Exception {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
+
+      String videoUri = "gs://cloud-samples-data/video/animals.mp4";
+      String imgUri = "gs://cloud-samples-data/generative-ai/image/character.jpg";
+
+      // Stream the result.
+      model.generateContentStream(
+          ContentMaker.fromMultiModalData(
+              PartMaker.fromMimeTypeAndData("video/mp4", videoUri),
+              PartMaker.fromMimeTypeAndData("image/jpeg", imgUri),
+              "Are this video and image correlated?"
+          ))
+          .stream()
+          .forEach(System.out::println);
+    }
+  }
+}
+// [END generativeaionvertexai_stream_multimodality_basic]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/StreamingQuestionAnswer.java b/vertexai/snippets/src/main/java/vertexai/gemini/StreamingQuestionAnswer.java
index cccbe206e6e..dd09ea14593 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/StreamingQuestionAnswer.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/StreamingQuestionAnswer.java
@@ -16,6 +16,7 @@
 
 package vertexai.gemini;
 
+// [START generativeaionvertexai_stream_text_basic]
 import com.google.cloud.vertexai.VertexAI;
 import com.google.cloud.vertexai.generativeai.GenerativeModel;
 
@@ -25,7 +26,7 @@ public static void main(String[] args) throws Exception {
     // TODO(developer): Replace these variables before running the sample.
     String projectId = "your-google-cloud-project-id";
     String location = "us-central1";
-    String modelName = "gemini-1.0-pro-vision-001";
+    String modelName = "gemini-1.5-flash-001";
 
     streamingQuestion(projectId, location, modelName);
   }
@@ -47,3 +48,4 @@ public static void streamingQuestion(String projectId, String location, String m
     }
   }
 }
+// [END generativeaionvertexai_stream_text_basic]
\ No newline at end of file
diff --git a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
index 69254d634c8..6bb02b7248a 100644
--- a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
+++ b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
@@ -33,7 +33,6 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -47,6 +46,7 @@ public class SnippetsIT {
   private static final String GEMINI_PRO_VISION = "gemini-1.0-pro-vision-001";
   private static final String GEMINI_PRO = "gemini-1.0-pro-002";
   private static final String GEMINI_FLASH = "gemini-1.5-flash-001";
+  private static final String DATASTORE_ID = "grounding-test-datastore_1716831150046";
   private static final int MAX_ATTEMPT_COUNT = 3;
   private static final int INITIAL_BACKOFF_MILLIS = 120000; // 2 minutes
 
@@ -215,13 +215,16 @@ public void testSafetySettings() throws Exception {
 
   @Test
   public void testTokenCount() throws Exception {
-    String textPrompt = "Why is the sky blue?";
-
-    int tokenCount =
-        GetTokenCount.getTokenCount(PROJECT_ID, LOCATION, GEMINI_PRO_VISION, textPrompt);
+    int tokenCount = GetTokenCount.getTokenCount(PROJECT_ID, LOCATION, GEMINI_FLASH);
     assertThat(tokenCount).isEqualTo(6);
   }
 
+  @Test
+  public void testMediaTokenCount() throws Exception {
+    int tokenCount = GetTokenCount.getMediaTokenCount(PROJECT_ID, LOCATION, GEMINI_FLASH);
+    assertThat(tokenCount).isEqualTo(16822);
+  }
+
   @Test
   public void testFunctionCalling() throws Exception {
     String textPrompt = "What's the weather in Paris?";
@@ -277,4 +280,35 @@ public void testSystemInstruction() throws Exception {
     assertThat(output).ignoringCase().contains("bagels");
     assertThat(output).ignoringCase().contains("aime");
   }
+
+  @Test
+  public void testGroundingWithPublicData() throws Exception {
+    String output = GroundingWithData.groundWithPublicData(PROJECT_ID, LOCATION, GEMINI_FLASH);
+
+    assertThat(output).ignoringCase().contains("Rayleigh");
+  }
+
+  @Test
+  public void testGroundingWithPrivateData() throws Exception {
+    String output = GroundingWithData.groundWithPrivateData(PROJECT_ID, LOCATION, GEMINI_FLASH,
+        String.format(
+            "projects/%s/locations/global/collections/default_collection/dataStores/%s",
+            PROJECT_ID, DATASTORE_ID)
+        );
+
+    assertThat(output).ignoringCase().contains("Rayleigh");
+  }
+
+  @Test
+  public void testMultimodalStreaming() throws Exception {
+    StreamingMultimodal.streamingMultimodal(PROJECT_ID, LOCATION, GEMINI_FLASH);
+    assertThat(bout.toString()).ignoringCase().contains("no");
+  }
+
+  @Test
+  public void testMultimodalNonStreaming() throws Exception {
+    String output = Multimodal.nonStreamingMultimodal(PROJECT_ID, LOCATION, GEMINI_FLASH);
+
+    assertThat(output).ignoringCase().contains("no");
+  }
 }

From 53f0a80d462940f1e47673da61b9b3f9fb357798 Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Tue, 4 Jun 2024 16:40:52 +0200
Subject: [PATCH 07/13] fix assertion in the grounding with private data test
 case

---
 vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
index 6bb02b7248a..981f23a760c 100644
--- a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
+++ b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
@@ -296,7 +296,7 @@ public void testGroundingWithPrivateData() throws Exception {
             PROJECT_ID, DATASTORE_ID)
         );
 
-    assertThat(output).ignoringCase().contains("Rayleigh");
+    assertThat(output).ignoringCase().contains("DMV");
   }
 
   @Test

From ab9d5c3d6d7d190bde9608ba3d979a07a5bc99a5 Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Tue, 4 Jun 2024 21:31:37 +0200
Subject: [PATCH 08/13] - split the token count and grounding samples to have
 one file per feature - disable a flaky test assertion

---
 .../vertexai/gemini/GetMediaTokenCount.java   | 60 +++++++++++++++++
 .../java/vertexai/gemini/GetTokenCount.java   | 26 -------
 ...ata.java => GroundingWithPrivateData.java} | 37 +++-------
 .../gemini/GroundingWithPublicData.java       | 67 +++++++++++++++++++
 .../test/java/vertexai/gemini/SnippetsIT.java | 10 +--
 5 files changed, 143 insertions(+), 57 deletions(-)
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/GetMediaTokenCount.java
 rename vertexai/snippets/src/main/java/vertexai/gemini/{GroundingWithData.java => GroundingWithPrivateData.java} (66%)
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GetMediaTokenCount.java b/vertexai/snippets/src/main/java/vertexai/gemini/GetMediaTokenCount.java
new file mode 100644
index 00000000000..f8245eb3bd1
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GetMediaTokenCount.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_token_count_advanced]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.Content;
+import com.google.cloud.vertexai.api.CountTokensResponse;
+import com.google.cloud.vertexai.generativeai.ContentMaker;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.PartMaker;
+import java.io.IOException;
+// [END generativeaionvertexai_gemini_token_count_advanced]
+
+public class GetMediaTokenCount {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "your-google-cloud-project-id";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-flash-001";
+
+    getMediaTokenCount(projectId, location, modelName);
+  }
+
+  // [START generativeaionvertexai_gemini_token_count_advanced]
+  public static int getMediaTokenCount(String projectId, String location, String modelName)
+      throws IOException {
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
+
+      Content content = ContentMaker.fromMultiModalData(
+          "Provide a description of the video.",
+          PartMaker.fromMimeTypeAndData(
+              "video/mp4", "gs://cloud-samples-data/generative-ai/video/pixel8.mp4")
+      );
+
+      CountTokensResponse response = model.countTokens(content);
+
+      int tokenCount = response.getTotalTokens();
+      System.out.println("Token count: " + tokenCount);
+
+      return tokenCount;
+    }
+  }
+  // [END generativeaionvertexai_gemini_token_count_advanced]
+}
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java b/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
index da5ee96100f..617a6d0e1be 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
@@ -19,12 +19,9 @@
 // [START aiplatform_gemini_token_count]
 // [START generativeaionvertexai_gemini_token_count_advanced]
 import com.google.cloud.vertexai.VertexAI;
-import com.google.cloud.vertexai.api.Content;
 import com.google.cloud.vertexai.api.CountTokensResponse;
 import com.google.cloud.vertexai.api.GenerateContentResponse;
-import com.google.cloud.vertexai.generativeai.ContentMaker;
 import com.google.cloud.vertexai.generativeai.GenerativeModel;
-import com.google.cloud.vertexai.generativeai.PartMaker;
 import java.io.IOException;
 // [END aiplatform_gemini_token_count]
 // [END generativeaionvertexai_gemini_token_count_advanced]
@@ -37,7 +34,6 @@ public static void main(String[] args) throws IOException {
     String modelName = "gemini-1.5-flash-001";
 
     getTokenCount(projectId, location, modelName);
-    getMediaTokenCount(projectId, location, modelName);
   }
 
   // [START aiplatform_gemini_token_count]
@@ -69,26 +65,4 @@ public static int getTokenCount(String projectId, String location, String modelN
     }
   }
   // [END aiplatform_gemini_token_count]
-
-  // [START generativeaionvertexai_gemini_token_count_advanced]
-  public static int getMediaTokenCount(String projectId, String location, String modelName)
-      throws IOException {
-    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
-      GenerativeModel model = new GenerativeModel(modelName, vertexAI);
-
-      Content content = ContentMaker.fromMultiModalData(
-          "Provide a description of the video.",
-          PartMaker.fromMimeTypeAndData(
-              "video/mp4", "gs://cloud-samples-data/generative-ai/video/pixel8.mp4")
-      );
-
-      CountTokensResponse response = model.countTokens(content);
-
-      int tokenCount = response.getTotalTokens();
-      System.out.println("Token count: " + tokenCount);
-
-      return tokenCount;
-    }
-  }
-  // [END generativeaionvertexai_gemini_token_count_advanced]
 }
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithData.java b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPrivateData.java
similarity index 66%
rename from vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithData.java
rename to vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPrivateData.java
index 27add469260..8b5598ae2dd 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithData.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPrivateData.java
@@ -17,10 +17,8 @@
 package vertexai.gemini;
 
 // [START generativeaionvertexai_grounding_private_data_basic]
-// [START generativeaionvertexai_grounding_public_data_basic]
 import com.google.cloud.vertexai.VertexAI;
 import com.google.cloud.vertexai.api.GenerateContentResponse;
-import com.google.cloud.vertexai.api.GoogleSearchRetrieval;
 import com.google.cloud.vertexai.api.GroundingMetadata;
 import com.google.cloud.vertexai.api.Retrieval;
 import com.google.cloud.vertexai.api.Tool;
@@ -30,33 +28,18 @@
 import java.io.IOException;
 import java.util.Collections;
 // [END generativeaionvertexai_grounding_private_data_basic]
-// [END generativeaionvertexai_grounding_public_data_basic]
 
-public class GroundingWithData {
+public class GroundingWithPrivateData {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "your-google-cloud-project-id";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-flash-001";
+    String datastore = String.format(
+        "projects/%s/locations/global/collections/default_collection/dataStores/%s",
+        projectId, "datastore_id");
 
-  public static String groundWithPublicData(String projectId, String location, String modelName)
-      throws IOException {
-    // [START generativeaionvertexai_grounding_public_data_basic]
-    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
-      Tool googleSearchTool = Tool.newBuilder()
-          .setGoogleSearchRetrieval(GoogleSearchRetrieval.newBuilder().setDisableAttribution(false))
-          .build();
-
-      GenerativeModel model = new GenerativeModel(modelName, vertexAI).withTools(
-          Collections.singletonList(googleSearchTool)
-      );
-
-      GenerateContentResponse response = model.generateContent("Why is the sky blue?");
-
-      GroundingMetadata groundingMetadata = response.getCandidates(0).getGroundingMetadata();
-      String answer = ResponseHandler.getText(response);
-
-      System.out.println("Answer: " + answer);
-      System.out.println("Grounding metadata: " + groundingMetadata);
-      // [END generativeaionvertexai_grounding_public_data_basic]
-
-      return answer;
-    }
+    groundWithPrivateData(projectId, location, modelName, datastore);
   }
 
   public static String groundWithPrivateData(String projectId, String location, String modelName,
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
new file mode 100644
index 00000000000..4ea999cc35c
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_grounding_public_data_basic]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.api.GoogleSearchRetrieval;
+import com.google.cloud.vertexai.api.GroundingMetadata;
+import com.google.cloud.vertexai.api.Tool;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+import java.io.IOException;
+import java.util.Collections;
+// [END generativeaionvertexai_grounding_public_data_basic]
+
+public class GroundingWithPublicData {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "your-google-cloud-project-id";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-flash-001";
+
+    groundWithPublicData(projectId, location, modelName);
+  }
+
+  public static String groundWithPublicData(String projectId, String location, String modelName)
+      throws IOException {
+    // [START generativeaionvertexai_grounding_public_data_basic]
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      Tool googleSearchTool = Tool.newBuilder()
+          .setGoogleSearchRetrieval(
+              GoogleSearchRetrieval.newBuilder().setDisableAttribution(false))
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI).withTools(
+          Collections.singletonList(googleSearchTool)
+      );
+
+      GenerateContentResponse response = model.generateContent("Why is the sky blue?");
+
+      GroundingMetadata groundingMetadata = response.getCandidates(0).getGroundingMetadata();
+      String answer = ResponseHandler.getText(response);
+
+      System.out.println("Answer: " + answer);
+      System.out.println("Grounding metadata: " + groundingMetadata);
+      // [END generativeaionvertexai_grounding_public_data_basic]
+
+      return answer;
+    }
+  }
+
+}
diff --git a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
index 981f23a760c..743346dd74c 100644
--- a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
+++ b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
@@ -175,7 +175,7 @@ public void testSimpleQuestionAnswer() throws Exception {
   @Test
   public void testQuickstart() throws IOException {
     String output = Quickstart.quickstart(PROJECT_ID, LOCATION, GEMINI_PRO_VISION);
-    assertThat(output).contains("Colosseum");
+    // assertThat(output).contains("Colosseum");
   }
 
   @Test
@@ -221,7 +221,7 @@ public void testTokenCount() throws Exception {
 
   @Test
   public void testMediaTokenCount() throws Exception {
-    int tokenCount = GetTokenCount.getMediaTokenCount(PROJECT_ID, LOCATION, GEMINI_FLASH);
+    int tokenCount = GetMediaTokenCount.getMediaTokenCount(PROJECT_ID, LOCATION, GEMINI_FLASH);
     assertThat(tokenCount).isEqualTo(16822);
   }
 
@@ -283,14 +283,16 @@ public void testSystemInstruction() throws Exception {
 
   @Test
   public void testGroundingWithPublicData() throws Exception {
-    String output = GroundingWithData.groundWithPublicData(PROJECT_ID, LOCATION, GEMINI_FLASH);
+    String output = GroundingWithPublicData.groundWithPublicData(
+        PROJECT_ID, LOCATION, GEMINI_FLASH);
 
     assertThat(output).ignoringCase().contains("Rayleigh");
   }
 
   @Test
   public void testGroundingWithPrivateData() throws Exception {
-    String output = GroundingWithData.groundWithPrivateData(PROJECT_ID, LOCATION, GEMINI_FLASH,
+    String output = GroundingWithPrivateData.groundWithPrivateData(
+        PROJECT_ID, LOCATION, GEMINI_FLASH,
         String.format(
             "projects/%s/locations/global/collections/default_collection/dataStores/%s",
             PROJECT_ID, DATASTORE_ID)

From f87ef775adae49ed974e64d787ec64772efa74a4 Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Wed, 5 Jun 2024 16:00:37 +0200
Subject: [PATCH 09/13] - added some extra comments for clarity - updated the
 region tag locations

---
 .../src/main/java/vertexai/gemini/FunctionCalling.java   | 4 +++-
 .../main/java/vertexai/gemini/GetMediaTokenCount.java    | 7 ++++---
 .../src/main/java/vertexai/gemini/GetTokenCount.java     | 9 ++++-----
 .../java/vertexai/gemini/GroundingWithPrivateData.java   | 8 +++++---
 .../java/vertexai/gemini/GroundingWithPublicData.java    | 8 ++++----
 .../src/main/java/vertexai/gemini/Multimodal.java        | 4 ++--
 .../main/java/vertexai/gemini/MultimodalVideoInput.java  | 4 ++--
 .../main/java/vertexai/gemini/StreamingMultimodal.java   | 4 ++--
 .../java/vertexai/gemini/StreamingQuestionAnswer.java    | 4 ++--
 9 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/FunctionCalling.java b/vertexai/snippets/src/main/java/vertexai/gemini/FunctionCalling.java
index 3d8f875515e..a0cb2f05bfc 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/FunctionCalling.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/FunctionCalling.java
@@ -45,10 +45,12 @@ public static void main(String[] args) throws IOException {
     whatsTheWeatherLike(projectId, location, modelName, promptText);
   }
 
+  // A request involving the interaction with an external tool
   public static String whatsTheWeatherLike(String projectId, String location,
                                            String modelName, String promptText)
       throws IOException {
-
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
 
       FunctionDeclaration functionDeclaration = FunctionDeclaration.newBuilder()
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GetMediaTokenCount.java b/vertexai/snippets/src/main/java/vertexai/gemini/GetMediaTokenCount.java
index f8245eb3bd1..5aa45b8af4f 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GetMediaTokenCount.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GetMediaTokenCount.java
@@ -24,7 +24,6 @@
 import com.google.cloud.vertexai.generativeai.GenerativeModel;
 import com.google.cloud.vertexai.generativeai.PartMaker;
 import java.io.IOException;
-// [END generativeaionvertexai_gemini_token_count_advanced]
 
 public class GetMediaTokenCount {
   public static void main(String[] args) throws IOException {
@@ -36,9 +35,11 @@ public static void main(String[] args) throws IOException {
     getMediaTokenCount(projectId, location, modelName);
   }
 
-  // [START generativeaionvertexai_gemini_token_count_advanced]
+  // Gets the number of tokens for the prompt with text and video and the model's response.
   public static int getMediaTokenCount(String projectId, String location, String modelName)
       throws IOException {
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       GenerativeModel model = new GenerativeModel(modelName, vertexAI);
 
@@ -56,5 +57,5 @@ public static int getMediaTokenCount(String projectId, String location, String m
       return tokenCount;
     }
   }
-  // [END generativeaionvertexai_gemini_token_count_advanced]
 }
+// [END generativeaionvertexai_gemini_token_count_advanced]
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java b/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
index 617a6d0e1be..c74eba38240 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GetTokenCount.java
@@ -17,14 +17,11 @@
 package vertexai.gemini;
 
 // [START aiplatform_gemini_token_count]
-// [START generativeaionvertexai_gemini_token_count_advanced]
 import com.google.cloud.vertexai.VertexAI;
 import com.google.cloud.vertexai.api.CountTokensResponse;
 import com.google.cloud.vertexai.api.GenerateContentResponse;
 import com.google.cloud.vertexai.generativeai.GenerativeModel;
 import java.io.IOException;
-// [END aiplatform_gemini_token_count]
-// [END generativeaionvertexai_gemini_token_count_advanced]
 
 public class GetTokenCount {
   public static void main(String[] args) throws IOException {
@@ -36,9 +33,11 @@ public static void main(String[] args) throws IOException {
     getTokenCount(projectId, location, modelName);
   }
 
-  // [START aiplatform_gemini_token_count]
+  // Gets the number of tokens for the prompt and the model's response.
   public static int getTokenCount(String projectId, String location, String modelName)
       throws IOException {
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       GenerativeModel model = new GenerativeModel(modelName, vertexAI);
 
@@ -64,5 +63,5 @@ public static int getTokenCount(String projectId, String location, String modelN
       return promptTokenCount;
     }
   }
-  // [END aiplatform_gemini_token_count]
 }
+// [END aiplatform_gemini_token_count]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPrivateData.java b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPrivateData.java
index 8b5598ae2dd..495da7d80b6 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPrivateData.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPrivateData.java
@@ -27,7 +27,6 @@
 import com.google.cloud.vertexai.generativeai.ResponseHandler;
 import java.io.IOException;
 import java.util.Collections;
-// [END generativeaionvertexai_grounding_private_data_basic]
 
 public class GroundingWithPrivateData {
   public static void main(String[] args) throws IOException {
@@ -42,10 +41,13 @@ public static void main(String[] args) throws IOException {
     groundWithPrivateData(projectId, location, modelName, datastore);
   }
 
+  // A request whose response will be "grounded"
+  // with information found in Vertex AI Search datastores.
   public static String groundWithPrivateData(String projectId, String location, String modelName,
                                              String datastoreId)
       throws IOException {
-    // [START generativeaionvertexai_grounding_private_data_basic]
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       Tool datastoreTool = Tool.newBuilder()
           .setRetrieval(
@@ -66,9 +68,9 @@ public static String groundWithPrivateData(String projectId, String location, St
 
       System.out.println("Answer: " + answer);
       System.out.println("Grounding metadata: " + groundingMetadata);
-      // [END generativeaionvertexai_grounding_private_data_basic]
 
       return answer;
     }
   }
 }
+// [END generativeaionvertexai_grounding_private_data_basic]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
index 4ea999cc35c..4e528948e70 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
@@ -26,7 +26,6 @@
 import com.google.cloud.vertexai.generativeai.ResponseHandler;
 import java.io.IOException;
 import java.util.Collections;
-// [END generativeaionvertexai_grounding_public_data_basic]
 
 public class GroundingWithPublicData {
   public static void main(String[] args) throws IOException {
@@ -38,9 +37,11 @@ public static void main(String[] args) throws IOException {
     groundWithPublicData(projectId, location, modelName);
   }
 
+  // A request whose response will be "grounded" with information found in Google Search.
   public static String groundWithPublicData(String projectId, String location, String modelName)
       throws IOException {
-    // [START generativeaionvertexai_grounding_public_data_basic]
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       Tool googleSearchTool = Tool.newBuilder()
           .setGoogleSearchRetrieval(
@@ -58,10 +59,9 @@ public static String groundWithPublicData(String projectId, String location, Str
 
       System.out.println("Answer: " + answer);
       System.out.println("Grounding metadata: " + groundingMetadata);
-      // [END generativeaionvertexai_grounding_public_data_basic]
 
       return answer;
     }
   }
-
 }
+// [END generativeaionvertexai_grounding_public_data_basic]
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/Multimodal.java b/vertexai/snippets/src/main/java/vertexai/gemini/Multimodal.java
index 816e371f80b..c81c18ca478 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/Multimodal.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/Multimodal.java
@@ -38,8 +38,8 @@ public static void main(String[] args) throws Exception {
   // Ask a simple question and get the response.
   public static String nonStreamingMultimodal(String projectId, String location, String modelName)
       throws Exception {
-    // Initialize client that will be used to send requests. This client only needs
-    // to be created once, and can be reused for multiple requests.
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       GenerativeModel model = new GenerativeModel(modelName, vertexAI);
 
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoInput.java b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoInput.java
index 8543b564a4e..0ddef59a828 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoInput.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/MultimodalVideoInput.java
@@ -39,8 +39,8 @@ public static void main(String[] args) throws IOException {
   // Analyzes the given video input.
   public static void multimodalVideoInput(String projectId, String location, String modelName)
       throws IOException {
-    // Initialize client that will be used to send requests. This client only needs
-    // to be created once, and can be reused for multiple requests.
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       String videoUri = "gs://cloud-samples-data/video/animals.mp4";
 
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/StreamingMultimodal.java b/vertexai/snippets/src/main/java/vertexai/gemini/StreamingMultimodal.java
index 051e8a817d3..756e66b3075 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/StreamingMultimodal.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/StreamingMultimodal.java
@@ -35,8 +35,8 @@ public static void main(String[] args) throws Exception {
   // Ask a simple question and get the response via streaming.
   public static void streamingMultimodal(String projectId, String location, String modelName)
       throws Exception {
-    // Initialize client that will be used to send requests. This client only needs
-    // to be created once, and can be reused for multiple requests.
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       GenerativeModel model = new GenerativeModel(modelName, vertexAI);
 
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/StreamingQuestionAnswer.java b/vertexai/snippets/src/main/java/vertexai/gemini/StreamingQuestionAnswer.java
index dd09ea14593..c5d69ea92d7 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/StreamingQuestionAnswer.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/StreamingQuestionAnswer.java
@@ -34,8 +34,8 @@ public static void main(String[] args) throws Exception {
   // Ask a simple question and get the response via streaming.
   public static void streamingQuestion(String projectId, String location, String modelName)
       throws Exception {
-    // Initialize client that will be used to send requests. This client only needs
-    // to be created once, and can be reused for multiple requests.
+    // Initialize client that will be used to send requests.
+    // This client only needs to be created once, and can be reused for multiple requests.
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       GenerativeModel model = new GenerativeModel(modelName, vertexAI);
 

From e9b4fa259f19a8b9cde400191c699d8aae0fc8d0 Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Wed, 5 Jun 2024 16:52:21 +0200
Subject: [PATCH 10/13] - added a couple more comments for clarity and
 reference

---
 .../src/main/java/vertexai/gemini/GroundingWithPublicData.java   | 1 +
 vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
index 4e528948e70..3ebe1518f74 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
@@ -45,6 +45,7 @@ public static String groundWithPublicData(String projectId, String location, Str
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       Tool googleSearchTool = Tool.newBuilder()
           .setGoogleSearchRetrieval(
+              // Enable using the result from this tool in detecting grounding
               GoogleSearchRetrieval.newBuilder().setDisableAttribution(false))
           .build();
 
diff --git a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
index 743346dd74c..9d120eaa207 100644
--- a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
+++ b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
@@ -175,6 +175,7 @@ public void testSimpleQuestionAnswer() throws Exception {
   @Test
   public void testQuickstart() throws IOException {
     String output = Quickstart.quickstart(PROJECT_ID, LOCATION, GEMINI_PRO_VISION);
+    // Disabled assertion, pending resolution of b/342637034
     // assertThat(output).contains("Colosseum");
   }
 

From 511c29bbc8d5c397cd817b38607912212ceb3a3d Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Wed, 31 Jul 2024 21:02:00 +0200
Subject: [PATCH 11/13] feat(vertexai): Add controlled generation (constrained
 decoding)

---
 vertexai/snippets/pom.xml                     |   2 +-
 .../gemini/ControlledGenerationMimeType.java  |  63 +++++++
 .../gemini/ControlledGenerationSchema.java    |  72 ++++++++
 .../gemini/ControlledGenerationSchema2.java   |  77 +++++++++
 .../gemini/ControlledGenerationSchema3.java   | 103 ++++++++++++
 .../gemini/ControlledGenerationSchema4.java   |  90 ++++++++++
 .../gemini/ControlledGenerationSchema6.java   |  82 +++++++++
 .../gemini/GroundingWithPublicData.java       |   3 +-
 .../test/java/vertexai/gemini/SnippetsIT.java | 159 ++++++++++++++++++
 9 files changed, 648 insertions(+), 3 deletions(-)
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationMimeType.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema2.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema3.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema4.java
 create mode 100644 vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema6.java

diff --git a/vertexai/snippets/pom.xml b/vertexai/snippets/pom.xml
index 61c2dd0dd66..b4a5764881b 100644
--- a/vertexai/snippets/pom.xml
+++ b/vertexai/snippets/pom.xml
@@ -46,7 +46,7 @@
                 <groupId>com.google.cloud</groupId>
                 <scope>import</scope>
                 <type>pom</type>
-                <version>26.39.0</version>
+                <version>26.43.0</version>
             </dependency>
         </dependencies>
     </dependencyManagement>
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationMimeType.java b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationMimeType.java
new file mode 100644
index 00000000000..ed028ce7478
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationMimeType.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_controlled_generation_response_mime_type]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.api.GenerationConfig;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+import java.io.IOException;
+
+public class ControlledGenerationMimeType {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "genai-java-demos";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-flash-001";
+
+    controlGenerationWithMimeType(projectId, location, modelName);
+  }
+
+  // Generate responses that are always valid JSON
+  public static String controlGenerationWithMimeType(
+      String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerationConfig generationConfig = GenerationConfig.newBuilder()
+          .setResponseMimeType("application/json")
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI)
+          .withGenerationConfig(generationConfig);
+
+      GenerateContentResponse response = model.generateContent(
+          "List a few popular cookie recipes using this JSON schema:\n"
+               + "Recipe = {\"recipe_name\": str}\n"
+               + "Return: list[Recipe]"
+      );
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_controlled_generation_response_mime_type]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema.java b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema.java
new file mode 100644
index 00000000000..efd96eecb54
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_controlled_generation_response_schema]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.api.GenerationConfig;
+import com.google.cloud.vertexai.api.Schema;
+import com.google.cloud.vertexai.api.Type;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+import java.io.IOException;
+import java.util.Collections;
+
+public class ControlledGenerationSchema {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "genai-java-demos";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-pro-001";
+
+    controlGenerationWithJsonSchema(projectId, location, modelName);
+  }
+
+  // Generate responses that are always valid JSON and comply with a JSON schema
+  public static String controlGenerationWithJsonSchema(
+      String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerationConfig generationConfig = GenerationConfig.newBuilder()
+          .setResponseMimeType("application/json")
+          .setResponseSchema(Schema.newBuilder()
+              .setType(Type.ARRAY)
+              .setItems(Schema.newBuilder()
+                  .setType(Type.OBJECT)
+                  .putProperties("recipe_name", Schema.newBuilder().setType(Type.STRING).build())
+                  .addAllRequired(Collections.singletonList("recipe_name"))
+                  .build())
+              .build())
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI)
+          .withGenerationConfig(generationConfig);
+
+      GenerateContentResponse response = model.generateContent(
+          "List a few popular cookie recipes."
+      );
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_controlled_generation_response_schema]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema2.java b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema2.java
new file mode 100644
index 00000000000..9167a60abfd
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema2.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_controlled_generation_response_schema_2]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.api.GenerationConfig;
+import com.google.cloud.vertexai.api.Schema;
+import com.google.cloud.vertexai.api.Type;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+import java.io.IOException;
+import java.util.Arrays;
+
+public class ControlledGenerationSchema2 {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "genai-java-demos";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-pro-001";
+
+    controlGenerationWithJsonSchema2(projectId, location, modelName);
+  }
+
+  // Generate responses that are always valid JSON and comply with a JSON schema
+  public static String controlGenerationWithJsonSchema2(
+      String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerationConfig generationConfig = GenerationConfig.newBuilder()
+          .setResponseMimeType("application/json")
+          .setResponseSchema(Schema.newBuilder()
+              .setType(Type.ARRAY)
+              .setItems(Schema.newBuilder()
+                  .setType(Type.OBJECT)
+                  .putProperties("rating", Schema.newBuilder().setType(Type.INTEGER).build())
+                  .putProperties("flavor", Schema.newBuilder().setType(Type.STRING).build())
+                  .addAllRequired(Arrays.asList("rating", "flavor"))
+                  .build())
+              .build())
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI)
+          .withGenerationConfig(generationConfig);
+
+      GenerateContentResponse response = model.generateContent(
+          "Reviews from our social media:\n"
+              + "\"Absolutely loved it! Best ice cream I've ever had.\" "
+              + "Rating: 4, Flavor: Strawberry Cheesecake\n"
+              + "\"Quite good, but a bit too sweet for my taste.\" "
+              + "Rating: 1, Flavor: Mango Tango"
+      );
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_controlled_generation_response_schema_2]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema3.java b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema3.java
new file mode 100644
index 00000000000..bffe8ffedc9
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema3.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_controlled_generation_response_schema_3]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.api.GenerationConfig;
+import com.google.cloud.vertexai.api.Schema;
+import com.google.cloud.vertexai.api.Type;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+import java.io.IOException;
+import java.util.Arrays;
+
+public class ControlledGenerationSchema3 {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "genai-java-demos";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-pro-001";
+
+    controlGenerationWithJsonSchema3(projectId, location, modelName);
+  }
+
+  // Generate responses that are always valid JSON and comply with a JSON schema
+  public static String controlGenerationWithJsonSchema3(
+      String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerationConfig generationConfig = GenerationConfig.newBuilder()
+          .setResponseMimeType("application/json")
+          .setResponseSchema(Schema.newBuilder()
+              .setType(Type.OBJECT)
+              .putProperties("forecast", Schema.newBuilder()
+                      .setType(Type.ARRAY)
+                      .setItems(Schema.newBuilder()
+                          .setType(Type.OBJECT)
+                          .putProperties("Day", Schema.newBuilder()
+                              .setType(Type.STRING)
+                              .build())
+                          .putProperties("Forecast", Schema.newBuilder()
+                              .setType(Type.STRING)
+                              .build())
+                          .putProperties("Humidity", Schema.newBuilder()
+                              .setType(Type.STRING)
+                              .build())
+                          .putProperties("Temperature", Schema.newBuilder()
+                              .setType(Type.INTEGER)
+                              .build())
+                          .putProperties("Wind Speed", Schema.newBuilder()
+                              .setType(Type.INTEGER)
+                              .build())
+                          .addAllRequired(Arrays.asList("Day", "Temperature", "Forecast"))
+                          .build())
+                      .build())
+              )
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI)
+          .withGenerationConfig(generationConfig);
+
+      GenerateContentResponse response = model.generateContent(
+          "The week ahead brings a mix of weather conditions.\n"
+              + "Sunday is expected to be sunny with a temperature of 77°F and a humidity level "
+              + "of 50%. Winds will be light at around 10 km/h.\n"
+              + "Monday will see partly cloudy skies with a slightly cooler temperature of 72°F "
+              + "and humidity increasing to 55%. Winds will pick up slightly to around 15 km/h.\n"
+              + "Tuesday brings rain showers, with temperatures dropping to 64°F and humidity"
+              + "rising to 70%. Expect stronger winds at 20 km/h.\n"
+              + "Wednesday may see thunderstorms, with a temperature of 68°F and high humidity "
+              + "of 75%. Winds will be gusty at 25 km/h.\n"
+              + "Thursday will be cloudy with a temperature of 66°F and moderate humidity at 60%. "
+              + "Winds will ease slightly to 18 km/h.\n"
+              + "Friday returns to partly cloudy conditions, with a temperature of 73°F and lower "
+              + "humidity at 45%. Winds will be light at 12 km/h.\n"
+              + "Finally, Saturday rounds off the week with sunny skies, a temperature of 80°F, "
+              + "and a humidity level of 40%. Winds will be gentle at 8 km/h."
+      );
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_controlled_generation_response_schema_3]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema4.java b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema4.java
new file mode 100644
index 00000000000..ab59d9d93b0
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema4.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_controlled_generation_response_schema_4]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.api.GenerationConfig;
+import com.google.cloud.vertexai.api.Schema;
+import com.google.cloud.vertexai.api.Type;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+import java.io.IOException;
+import java.util.Arrays;
+
+public class ControlledGenerationSchema4 {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "genai-java-demos";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-pro-001";
+
+    controlGenerationWithJsonSchema4(projectId, location, modelName);
+  }
+
+  // Generate responses that are always valid JSON and comply with a JSON schema
+  public static String controlGenerationWithJsonSchema4(
+      String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      Schema itemSchema = Schema.newBuilder()
+          .setType(Type.OBJECT)
+          .putProperties("to_discard", Schema.newBuilder().setType(Type.INTEGER).build())
+          .putProperties("subcategory", Schema.newBuilder().setType(Type.STRING).build())
+          .putProperties("safe_handling", Schema.newBuilder().setType(Type.INTEGER).build())
+          .putProperties("item_category", Schema.newBuilder()
+              .setType(Type.STRING)
+              .addAllEnum(Arrays.asList(
+                  "clothing", "winter apparel", "specialized apparel", "furniture",
+                  "decor", "tableware", "cookware", "toys"))
+              .build())
+          .putProperties("for_resale", Schema.newBuilder().setType(Type.INTEGER).build())
+          .putProperties("condition", Schema.newBuilder()
+              .setType(Type.STRING)
+              .addAllEnum(Arrays.asList(
+                  "new in package", "like new", "gently used", "used", "damanged", "soiled"))
+              .build())
+          .build();
+
+      GenerationConfig generationConfig = GenerationConfig.newBuilder()
+          .setResponseMimeType("application/json")
+          .setResponseSchema(Schema.newBuilder()
+              .setType(Type.ARRAY)
+              .setItems(itemSchema)
+              .build())
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI)
+          .withGenerationConfig(generationConfig);
+
+      GenerateContentResponse response = model.generateContent(
+          "Item description:\n"
+              + "The item is a long winter coat that has many tears all around the seams "
+              + "and is falling apart.\n"
+              + "It has large questionable stains on it."
+      );
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_controlled_generation_response_schema_4]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema6.java b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema6.java
new file mode 100644
index 00000000000..f90c37cb797
--- /dev/null
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema6.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vertexai.gemini;
+
+// [START generativeaionvertexai_gemini_controlled_generation_response_schema_6]
+import com.google.cloud.vertexai.VertexAI;
+import com.google.cloud.vertexai.api.GenerateContentResponse;
+import com.google.cloud.vertexai.api.GenerationConfig;
+import com.google.cloud.vertexai.api.Schema;
+import com.google.cloud.vertexai.api.Type;
+import com.google.cloud.vertexai.generativeai.ContentMaker;
+import com.google.cloud.vertexai.generativeai.GenerativeModel;
+import com.google.cloud.vertexai.generativeai.PartMaker;
+import com.google.cloud.vertexai.generativeai.ResponseHandler;
+import java.io.IOException;
+
+public class ControlledGenerationSchema6 {
+  public static void main(String[] args) throws IOException {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "genai-java-demos";
+    String location = "us-central1";
+    String modelName = "gemini-1.5-pro-001";
+
+    controlGenerationWithJsonSchema6(projectId, location, modelName);
+  }
+
+  // Generate responses that are always valid JSON and comply with a JSON schema
+  public static String controlGenerationWithJsonSchema6(
+      String projectId, String location, String modelName)
+      throws IOException {
+    // Initialize client that will be used to send requests. This client only needs
+    // to be created once, and can be reused for multiple requests.
+    try (VertexAI vertexAI = new VertexAI(projectId, location)) {
+      GenerationConfig generationConfig = GenerationConfig.newBuilder()
+          .setResponseMimeType("application/json")
+          .setResponseSchema(Schema.newBuilder()
+              .setType(Type.ARRAY)
+              .setItems(Schema.newBuilder()
+                  .setType(Type.OBJECT)
+                  .putProperties("object", Schema.newBuilder().setType(Type.STRING).build())
+                  .build())
+              .build())
+          .build();
+
+      GenerativeModel model = new GenerativeModel(modelName, vertexAI)
+          .withGenerationConfig(generationConfig);
+
+      // These images in Cloud Storage are viewable at
+      // https://storage.googleapis.com/cloud-samples-data/generative-ai/image/office-desk.jpeg
+      // https://storage.googleapis.com/cloud-samples-data/generative-ai/image/gardening-tools.jpeg
+
+      GenerateContentResponse response = model.generateContent(
+          ContentMaker.fromMultiModalData(
+              PartMaker.fromMimeTypeAndData("image/jpeg",
+                  "gs://cloud-samples-data/generative-ai/image/office-desk.jpeg"),
+              PartMaker.fromMimeTypeAndData("image/jpeg",
+                  "gs://cloud-samples-data/generative-ai/image/gardening-tools.jpeg"),
+              "Generate a list of objects in the images."
+          )
+      );
+
+      String output = ResponseHandler.getText(response);
+      System.out.println(output);
+      return output;
+    }
+  }
+}
+// [END generativeaionvertexai_gemini_controlled_generation_response_schema_6]
\ No newline at end of file
diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
index 3ebe1518f74..f9042e838eb 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/GroundingWithPublicData.java
@@ -45,8 +45,7 @@ public static String groundWithPublicData(String projectId, String location, Str
     try (VertexAI vertexAI = new VertexAI(projectId, location)) {
       Tool googleSearchTool = Tool.newBuilder()
           .setGoogleSearchRetrieval(
-              // Enable using the result from this tool in detecting grounding
-              GoogleSearchRetrieval.newBuilder().setDisableAttribution(false))
+              GoogleSearchRetrieval.newBuilder())
           .build();
 
       GenerativeModel model = new GenerativeModel(modelName, vertexAI).withTools(
diff --git a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
index 6a3abc39e44..80166f1d027 100644
--- a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
+++ b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
@@ -22,13 +22,17 @@
 import static com.google.common.truth.Truth.assertWithMessage;
 
 import com.google.cloud.testing.junit4.MultipleAttemptsRule;
+import com.google.gson.Gson;
+import com.google.gson.annotations.SerializedName;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.util.Arrays;
 import java.util.Base64;
+import java.util.stream.Collectors;
 import javax.net.ssl.HttpsURLConnection;
 import org.junit.After;
 import org.junit.Before;
@@ -44,6 +48,7 @@ public class SnippetsIT {
   private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT");
   private static final String LOCATION = "us-central1";
   private static final String GEMINI_FLASH = "gemini-1.5-flash-001";
+  private static final String GEMINI_PRO = "gemini-1.5-pro-001";
   private static final String DATASTORE_ID = "grounding-test-datastore_1716831150046";
   private static final int MAX_ATTEMPT_COUNT = 3;
   private static final int INITIAL_BACKOFF_MILLIS = 120000; // 2 minutes
@@ -312,4 +317,158 @@ public void testMultimodalNonStreaming() throws Exception {
 
     assertThat(output).ignoringCase().contains("no");
   }
+
+  private class Recipe {
+    @SerializedName("recipe_name")
+    public String recipeName;
+  }
+
+  @Test
+  public void testControlledGenerationWithMimeType() throws Exception {
+    String output = ControlledGenerationMimeType
+        .controlGenerationWithMimeType(PROJECT_ID, LOCATION, GEMINI_FLASH);
+    Recipe[] recipes = new Gson().fromJson(output, Recipe[].class);
+
+    assertThat(recipes).isNotEmpty();
+    assertThat(recipes[0].recipeName).isNotEmpty();
+  }
+
+  @Test
+  public void testControlledGenerationWithJsonSchema() throws Exception {
+    String output = ControlledGenerationSchema
+        .controlGenerationWithJsonSchema(PROJECT_ID, LOCATION, GEMINI_PRO);
+    Recipe[] recipes = new Gson().fromJson(output, Recipe[].class);
+
+    assertThat(recipes).isNotEmpty();
+    assertThat(recipes[0].recipeName).isNotEmpty();
+  }
+
+  private class Review {
+    public int rating;
+    public String flavor;
+  }
+
+  @Test
+  public void testControlledGenerationWithJsonSchema2() throws Exception {
+    String output = ControlledGenerationSchema2
+        .controlGenerationWithJsonSchema2(PROJECT_ID, LOCATION, GEMINI_PRO);
+    Review[] recipes = new Gson().fromJson(output, Review[].class);
+
+    assertThat(recipes).hasLength(2);
+    assertThat(recipes[0].flavor).isNotEmpty();
+    assertThat(recipes[0].rating).isEqualTo(4);
+    assertThat(recipes[1].flavor).isNotEmpty();
+    assertThat(recipes[1].rating).isEqualTo(1);
+  }
+
+  private class WeatherForecast {
+    public DayForecast[] forecast;
+  }
+
+  private class DayForecast {
+    @SerializedName("Day")
+    public String day;
+    @SerializedName("Forecast")
+    public String forecast;
+    @SerializedName("Humidity")
+    public String humidity;
+    @SerializedName("Temperature")
+    public int temperature;
+    @SerializedName("Wind Speed")
+    public int windSpeed;
+  }
+
+  @Test
+  public void testControlledGenerationWithJsonSchema3() throws Exception {
+    String output = ControlledGenerationSchema3
+        .controlGenerationWithJsonSchema3(PROJECT_ID, LOCATION, GEMINI_PRO);
+    WeatherForecast weatherForecast = new Gson().fromJson(output, WeatherForecast.class);
+
+    assertThat(weatherForecast.forecast).hasLength(7);
+
+    assertThat(weatherForecast.forecast[0].day).ignoringCase().isEqualTo("Sunday");
+    assertThat(weatherForecast.forecast[0].forecast).ignoringCase().isEqualTo("Sunny");
+    assertThat(weatherForecast.forecast[0].temperature).isEqualTo(77);
+    assertThat(weatherForecast.forecast[0].humidity).ignoringCase().isEqualTo("50%");
+    assertThat(weatherForecast.forecast[0].windSpeed).isEqualTo(10);
+
+    assertThat(weatherForecast.forecast[1].day).ignoringCase().isEqualTo("Monday");
+    assertThat(weatherForecast.forecast[1].forecast).ignoringCase().isEqualTo("Partly Cloudy");
+    assertThat(weatherForecast.forecast[1].temperature).isEqualTo(72);
+    assertThat(weatherForecast.forecast[1].humidity).ignoringCase().isEqualTo("55%");
+    assertThat(weatherForecast.forecast[1].windSpeed).isEqualTo(15);
+
+    assertThat(weatherForecast.forecast[2].day).ignoringCase().isEqualTo("Tuesday");
+    assertThat(weatherForecast.forecast[2].forecast).ignoringCase().contains("Rain");
+    assertThat(weatherForecast.forecast[2].temperature).isEqualTo(64);
+    assertThat(weatherForecast.forecast[2].humidity).ignoringCase().isEqualTo("70%");
+    assertThat(weatherForecast.forecast[2].windSpeed).isEqualTo(20);
+
+    assertThat(weatherForecast.forecast[3].day).ignoringCase().isEqualTo("Wednesday");
+    assertThat(weatherForecast.forecast[3].forecast).ignoringCase().contains("Thunder");
+    assertThat(weatherForecast.forecast[3].temperature).isEqualTo(68);
+    assertThat(weatherForecast.forecast[3].humidity).ignoringCase().isEqualTo("75%");
+    assertThat(weatherForecast.forecast[3].windSpeed).isEqualTo(25);
+
+    assertThat(weatherForecast.forecast[4].day).ignoringCase().isEqualTo("Thursday");
+    assertThat(weatherForecast.forecast[4].forecast).ignoringCase().isEqualTo("Cloudy");
+    assertThat(weatherForecast.forecast[4].temperature).isEqualTo(66);
+    assertThat(weatherForecast.forecast[4].humidity).ignoringCase().isEqualTo("60%");
+    assertThat(weatherForecast.forecast[4].windSpeed).isEqualTo(18);
+
+    assertThat(weatherForecast.forecast[5].day).ignoringCase().isEqualTo("Friday");
+    assertThat(weatherForecast.forecast[5].forecast).ignoringCase().isEqualTo("Partly Cloudy");
+    assertThat(weatherForecast.forecast[5].temperature).isEqualTo(73);
+    assertThat(weatherForecast.forecast[5].humidity).ignoringCase().isEqualTo("45%");
+    assertThat(weatherForecast.forecast[5].windSpeed).isEqualTo(12);
+
+    assertThat(weatherForecast.forecast[6].day).ignoringCase().isEqualTo("Saturday");
+    assertThat(weatherForecast.forecast[6].forecast).ignoringCase().isEqualTo("Sunny");
+    assertThat(weatherForecast.forecast[6].temperature).isEqualTo(80);
+    assertThat(weatherForecast.forecast[6].humidity).ignoringCase().isEqualTo("40%");
+    assertThat(weatherForecast.forecast[6].windSpeed).isEqualTo(8);
+  }
+
+  private class Item {
+    @SerializedName("to_discard")
+    public Integer toDiscard;
+    public String subcategory;
+    @SerializedName("safe_handling")
+    public Integer safeHandling;
+    @SerializedName("item_category")
+    public String itemCategory;
+    @SerializedName("for_resale")
+    public Integer forResale;
+    public String condition;
+  }
+
+  @Test
+  public void testControlledGenerationWithJsonSchema4() throws Exception {
+    String output = ControlledGenerationSchema4
+        .controlGenerationWithJsonSchema4(PROJECT_ID, LOCATION, GEMINI_PRO);
+    Item[] items = new Gson().fromJson(output, Item[].class);
+
+    assertThat(items).isNotEmpty();
+  }
+
+  private class Obj {
+    public String object;
+  }
+
+  @Test
+  public void testControlledGenerationWithJsonSchema6() throws Exception {
+    String output = ControlledGenerationSchema6
+        .controlGenerationWithJsonSchema6(PROJECT_ID, LOCATION, GEMINI_PRO);
+
+    Obj[] objects = new Gson().fromJson(output, Obj[].class);
+    String recognizedObjects = Arrays.stream(objects)
+        .map(obj -> obj.object.toLowerCase())
+        .collect(Collectors.joining(" "));
+
+    assertThat(recognizedObjects).isNotEmpty();
+    assertThat(recognizedObjects).contains("globe");
+    assertThat(recognizedObjects).contains("keyboard");
+    assertThat(recognizedObjects).contains("passport");
+    assertThat(recognizedObjects).contains("pot");
+  }
 }

From 638f46bba4042c0b9384530830a4c8aa930556e2 Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Thu, 1 Aug 2024 18:22:50 +0200
Subject: [PATCH 12/13] feat(vertexai): Add controlled generation (constrained
 decoding) (small typo)

---
 .../main/java/vertexai/gemini/ControlledGenerationSchema4.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema4.java b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema4.java
index ab59d9d93b0..e9e957c01a0 100644
--- a/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema4.java
+++ b/vertexai/snippets/src/main/java/vertexai/gemini/ControlledGenerationSchema4.java
@@ -59,7 +59,7 @@ public static String controlGenerationWithJsonSchema4(
           .putProperties("condition", Schema.newBuilder()
               .setType(Type.STRING)
               .addAllEnum(Arrays.asList(
-                  "new in package", "like new", "gently used", "used", "damanged", "soiled"))
+                  "new in package", "like new", "gently used", "used", "damaged", "soiled"))
               .build())
           .build();
 

From e8a20791df3cd8a99c56f5e2c8b7787694ccfc9f Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Mon, 31 Mar 2025 21:36:40 +0200
Subject: [PATCH 13/13] feat(vertexai): Upgrade most snippets to use Gemini 2.0
 Flash

---
 .../test/java/vertexai/gemini/SnippetsIT.java | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
index d39e140fe03..6a77706bd96 100644
--- a/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
+++ b/vertexai/snippets/src/test/java/vertexai/gemini/SnippetsIT.java
@@ -47,8 +47,8 @@ public class SnippetsIT {
 
   private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT");
   private static final String LOCATION = "us-central1";
-  private static final String GEMINI_FLASH = "gemini-1.5-flash-001";
-  private static final String GEMINI_PRO = "gemini-1.5-pro-001";
+  private static final String GEMINI_FLASH = "gemini-2.0-flash-001";
+  private static final String GEMINI_FLASH_1_5 = "gemini-1.5-flash-001";
   private static final String DATASTORE_ID = "grounding-test-datastore_1716831150046";
   private static final int MAX_ATTEMPT_COUNT = 3;
   private static final int INITIAL_BACKOFF_MILLIS = 120000;
@@ -183,7 +183,7 @@ public void testSimpleQuestionAnswer() throws Exception {
   @Test
   public void testQuickstart() throws IOException {
     String output = Quickstart.quickstart(PROJECT_ID, LOCATION, GEMINI_FLASH);
-    assertThat(output).contains("scones");
+    assertThat(output).contains("cookie");
   }
 
   @Test
@@ -229,7 +229,7 @@ public void testTokenCount() throws Exception {
   @Test
   public void testMediaTokenCount() throws Exception {
     int tokenCount = GetMediaTokenCount.getMediaTokenCount(PROJECT_ID, LOCATION, GEMINI_FLASH);
-    assertThat(tokenCount).isEqualTo(16822);
+    assertThat(tokenCount).isEqualTo(16252);
   }
 
   @Test
@@ -314,7 +314,7 @@ public void testSystemInstruction() throws Exception {
   @Test
   public void testGroundingWithPublicData() throws Exception {
     String output =
-        GroundingWithPublicData.groundWithPublicData(PROJECT_ID, LOCATION, GEMINI_FLASH);
+        GroundingWithPublicData.groundWithPublicData(PROJECT_ID, LOCATION, GEMINI_FLASH_1_5);
 
     assertThat(output).ignoringCase().contains("Rayleigh");
   }
@@ -364,7 +364,7 @@ public void testControlledGenerationWithMimeType() throws Exception {
   @Test
   public void testControlledGenerationWithJsonSchema() throws Exception {
     String output = ControlledGenerationSchema
-        .controlGenerationWithJsonSchema(PROJECT_ID, LOCATION, GEMINI_PRO);
+        .controlGenerationWithJsonSchema(PROJECT_ID, LOCATION, GEMINI_FLASH);
     Recipe[] recipes = new Gson().fromJson(output, Recipe[].class);
 
     assertThat(recipes).isNotEmpty();
@@ -379,7 +379,7 @@ private class Review {
   @Test
   public void testControlledGenerationWithJsonSchema2() throws Exception {
     String output = ControlledGenerationSchema2
-        .controlGenerationWithJsonSchema2(PROJECT_ID, LOCATION, GEMINI_PRO);
+        .controlGenerationWithJsonSchema2(PROJECT_ID, LOCATION, GEMINI_FLASH);
     Review[] recipes = new Gson().fromJson(output, Review[].class);
 
     assertThat(recipes).hasLength(2);
@@ -409,7 +409,7 @@ private class DayForecast {
   @Test
   public void testControlledGenerationWithJsonSchema3() throws Exception {
     String output = ControlledGenerationSchema3
-        .controlGenerationWithJsonSchema3(PROJECT_ID, LOCATION, GEMINI_PRO);
+        .controlGenerationWithJsonSchema3(PROJECT_ID, LOCATION, GEMINI_FLASH);
     WeatherForecast weatherForecast = new Gson().fromJson(output, WeatherForecast.class);
 
     assertThat(weatherForecast.forecast).hasLength(7);
@@ -473,7 +473,7 @@ private class Item {
   @Test
   public void testControlledGenerationWithJsonSchema4() throws Exception {
     String output = ControlledGenerationSchema4
-        .controlGenerationWithJsonSchema4(PROJECT_ID, LOCATION, GEMINI_PRO);
+        .controlGenerationWithJsonSchema4(PROJECT_ID, LOCATION, GEMINI_FLASH);
     Item[] items = new Gson().fromJson(output, Item[].class);
 
     assertThat(items).isNotEmpty();
@@ -486,7 +486,7 @@ private class Obj {
   @Test
   public void testControlledGenerationWithJsonSchema6() throws Exception {
     String output = ControlledGenerationSchema6
-        .controlGenerationWithJsonSchema6(PROJECT_ID, LOCATION, GEMINI_PRO);
+        .controlGenerationWithJsonSchema6(PROJECT_ID, LOCATION, GEMINI_FLASH);
 
     Obj[] objects = new Gson().fromJson(output, Obj[].class);
     String recognizedObjects = Arrays.stream(objects)
@@ -503,7 +503,7 @@ public void testControlledGenerationWithJsonSchema6() throws Exception {
   @Test
   public void testGeminiTranslate() throws Exception {
     String output = GeminiTranslate.geminiTranslate(
-        PROJECT_ID, LOCATION, GEMINI_PRO, TEXT_TO_TRANSLATE, TARGET_LANGUAGE_CODE);
+        PROJECT_ID, LOCATION, GEMINI_FLASH, TEXT_TO_TRANSLATE, TARGET_LANGUAGE_CODE);
 
     assertThat(output).ignoringCase().contains("Bonjour");
     assertThat(output).ignoringCase().contains("aujourd'hui");