diff --git a/src/providers/google/chatComplete.ts b/src/providers/google/chatComplete.ts index 14287a1f6..156436229 100644 --- a/src/providers/google/chatComplete.ts +++ b/src/providers/google/chatComplete.ts @@ -7,6 +7,7 @@ import { ToolCall, ToolChoice, } from '../../types/requestBody'; +import { getMimeType } from '../google-vertex-ai/utils'; import { ChatCompletionResponse, ErrorResponse, @@ -170,12 +171,37 @@ export const GoogleChatCompleteConfig: ProviderConfig = { }); } if (c.type === 'image_url') { - parts.push({ - inlineData: { - mimeType: 'image/jpeg', - data: c.image_url?.url, - }, - }); + const { url } = c.image_url || {}; + if (!url) return; + + if (url.startsWith('data:')) { + const [mimeTypeWithPrefix, base64Image] = + url.split(';base64,'); + const mimeType = mimeTypeWithPrefix.split(':')[1]; + + parts.push({ + inlineData: { + mimeType: mimeType, + data: base64Image, + }, + }); + } else if (url.startsWith('gs://')) { + parts.push({ + fileData: { + mimeType: getMimeType(url), + fileUri: url, + }, + }); + } else { + // NOTE: This block is kept to maintain backward compatibility + // Earlier we were assuming that all images will be base64 with image/jpeg mimeType + parts.push({ + inlineData: { + mimeType: 'image/jpeg', + data: c.image_url?.url, + }, + }); + } } }); } else if (typeof message.content === 'string') { @@ -186,19 +212,17 @@ export const GoogleChatCompleteConfig: ProviderConfig = { // @NOTE: This takes care of the "Please ensure that multiturn requests alternate between user and model." // error that occurs when we have multiple user messages in a row. - const shouldAppendEmptyModeChat = - lastRole === 'user' && - role === 'user' && - !params.model?.includes('vision'); + const shouldCombineMessages = + lastRole === role && !params.model?.includes('vision'); - if (shouldAppendEmptyModeChat) { - messages.push({ role: 'model', parts: [{ text: '' }] }); + if (shouldCombineMessages) { + messages[messages.length - 1].parts.push(...parts); + } else { + messages.push({ role, parts }); } - messages.push({ role, parts }); lastRole = role; }); - return messages; }, }, @@ -389,12 +413,12 @@ export const GoogleChatCompleteResponseTransform: ( choices: response.candidates?.map((generation) => { let message: Message = { role: 'assistant', content: '' }; - if (generation.content.parts[0]?.text) { + if (generation.content?.parts[0]?.text) { message = { role: 'assistant', content: generation.content.parts[0]?.text, }; - } else if (generation.content.parts[0]?.functionCall) { + } else if (generation.content?.parts[0]?.functionCall) { message = { role: 'assistant', tool_calls: generation.content.parts.map((part) => {