Merge pull request #567 from Portkey-AI/fix/google-gemini-image-url-p…

…arsing

fix: update gemini image_url parsing logic to support other mime type…

src/providers/google/chatComplete.ts

@@ -7,6 +7,7 @@ import {
   ToolCall,
   ToolChoice,
 } from '../../types/requestBody';
+import { getMimeType } from '../google-vertex-ai/utils';
 import {
   ChatCompletionResponse,
   ErrorResponse,
@@ -170,12 +171,37 @@ export const GoogleChatCompleteConfig: ProviderConfig = {
                 });
               }
               if (c.type === 'image_url') {
-                parts.push({
-                  inlineData: {
-                    mimeType: 'image/jpeg',
-                    data: c.image_url?.url,
-                  },
-                });
+                const { url } = c.image_url || {};
+                if (!url) return;
+
+                if (url.startsWith('data:')) {
+                  const [mimeTypeWithPrefix, base64Image] =
+                    url.split(';base64,');
+                  const mimeType = mimeTypeWithPrefix.split(':')[1];
+
+                  parts.push({
+                    inlineData: {
+                      mimeType: mimeType,
+                      data: base64Image,
+                    },
+                  });
+                } else if (url.startsWith('gs://')) {
+                  parts.push({
+                    fileData: {
+                      mimeType: getMimeType(url),
+                      fileUri: url,
+                    },
+                  });
+                } else {
+                  // NOTE: This block is kept to maintain backward compatibility
+                  // Earlier we were assuming that all images will be base64 with image/jpeg mimeType
+                  parts.push({
+                    inlineData: {
+                      mimeType: 'image/jpeg',
+                      data: c.image_url?.url,
+                    },
+                  });
+                }
               }
             });
           } else if (typeof message.content === 'string') {
@@ -186,19 +212,17 @@ export const GoogleChatCompleteConfig: ProviderConfig = {
 
           // @NOTE: This takes care of the "Please ensure that multiturn requests alternate between user and model."
           // error that occurs when we have multiple user messages in a row.
-          const shouldAppendEmptyModeChat =
-            lastRole === 'user' &&
-            role === 'user' &&
-            !params.model?.includes('vision');
+          const shouldCombineMessages =
+            lastRole === role && !params.model?.includes('vision');
 
-          if (shouldAppendEmptyModeChat) {
-            messages.push({ role: 'model', parts: [{ text: '' }] });
+          if (shouldCombineMessages) {
+            messages[messages.length - 1].parts.push(...parts);
+          } else {
+            messages.push({ role, parts });
           }
 
-          messages.push({ role, parts });
           lastRole = role;
         });
-
         return messages;
       },
     },
@@ -389,12 +413,12 @@ export const GoogleChatCompleteResponseTransform: (
       choices:
         response.candidates?.map((generation) => {
           let message: Message = { role: 'assistant', content: '' };
-          if (generation.content.parts[0]?.text) {
+          if (generation.content?.parts[0]?.text) {
             message = {
               role: 'assistant',
               content: generation.content.parts[0]?.text,
             };
-          } else if (generation.content.parts[0]?.functionCall) {
+          } else if (generation.content?.parts[0]?.functionCall) {
             message = {
               role: 'assistant',
               tool_calls: generation.content.parts.map((part) => {