diff --git a/litellm/llms/vertex_ai.py b/litellm/llms/vertex_ai.py index 18c06d4d67..6752b3cb30 100644 --- a/litellm/llms/vertex_ai.py +++ b/litellm/llms/vertex_ai.py @@ -225,6 +225,17 @@ def _gemini_vision_convert_messages(messages: list): part_mime = "video/mp4" google_clooud_part = Part.from_uri(img, mime_type=part_mime) processed_images.append(google_clooud_part) + elif "base64" in img: + # Case 4: Images with base64 encoding + import base64 + + # base 64 is passed as data:image/jpeg;base64, + img_without_base_64 = img.split(",")[1] + decoded_img = base64.b64decode(img_without_base_64) + processed_image = Part.from_data( + data=decoded_img, mime_type="image/jpeg" + ) + processed_images.append(processed_image) return prompt, processed_images except Exception as e: raise e diff --git a/litellm/tests/cached_logo.jpg b/litellm/tests/cached_logo.jpg new file mode 100644 index 0000000000..ddf8b9e820 Binary files /dev/null and b/litellm/tests/cached_logo.jpg differ diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index f5df00c8d9..35d66907c3 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -336,6 +336,52 @@ def test_gemini_pro_vision(): # test_gemini_pro_vision() +def encode_image(image_path): + import base64 + + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + + +@pytest.mark.skip( + reason="we already test gemini-pro-vision, this is just another way to pass images" +) +def test_gemini_pro_vision_base64(): + try: + load_vertex_ai_credentials() + litellm.set_verbose = True + litellm.num_retries = 3 + image_path = "cached_logo.jpg" + # Getting the base64 string + base64_image = encode_image(image_path) + resp = litellm.completion( + model="vertex_ai/gemini-pro-vision", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Whats in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64," + base64_image + }, + }, + ], + } + ], + ) + print(resp) + + prompt_tokens = resp.usage.prompt_tokens + + except Exception as e: + if "500 Internal error encountered.'" in str(e): + pass + else: + pytest.fail(f"An exception occurred - {str(e)}") + + def test_gemini_pro_function_calling(): load_vertex_ai_credentials() tools = [