From 4efbb4744bf8e8b725bdad1a9729c49222ad8f6e Mon Sep 17 00:00:00 2001 From: raspawar Date: Mon, 2 Dec 2024 21:51:31 +0530 Subject: [PATCH 1/3] add default model and test cases --- .../examples/multi_modal/nvidia_multi_modal.ipynb | 13 +++++++++++-- .../llama_index/multi_modal_llms/nvidia/base.py | 14 +++++++++++++- .../tests/test_multi_modal_nvidia.py | 13 +++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/docs/examples/multi_modal/nvidia_multi_modal.ipynb b/docs/docs/examples/multi_modal/nvidia_multi_modal.ipynb index 5753e690097f0..17c1434f31214 100644 --- a/docs/docs/examples/multi_modal/nvidia_multi_modal.ipynb +++ b/docs/docs/examples/multi_modal/nvidia_multi_modal.ipynb @@ -74,6 +74,16 @@ "llm = NVIDIAMultiModal()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## To get available models use available_models property\n", + "llm.available_models" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -482,8 +492,7 @@ " [\n", " ChatMessage(\n", " role=\"user\",\n", - " content=f\"\"\"\"\"\",\n", + " content=f\"\"\"Describe the Image: \"\"\",\n", " )\n", " ]\n", ")" diff --git a/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/llama_index/multi_modal_llms/nvidia/base.py b/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/llama_index/multi_modal_llms/nvidia/base.py index e14dfa616f510..bd8eabe05f889 100644 --- a/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/llama_index/multi_modal_llms/nvidia/base.py +++ b/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/llama_index/multi_modal_llms/nvidia/base.py @@ -30,6 +30,7 @@ from llama_index.multi_modal_llms.nvidia.utils import ( BASE_URL, + DEFAULT_MODEL, KNOWN_URLS, NVIDIA_MULTI_MODAL_MODELS, generate_nvidia_multi_modal_chat_message, @@ -38,6 +39,7 @@ ) import aiohttp import json +import warnings from llama_index.core.bridge.pydantic import BaseModel @@ -154,7 +156,7 @@ class NVIDIAMultiModal(MultiModalLLM): def __init__( self, - model: str = "microsoft/phi-3-vision-128k-instruct", + model: str = None, temperature: float = DEFAULT_TEMPERATURE, max_tokens: Optional[int] = 300, nvidia_api_key: Optional[str] = None, @@ -177,6 +179,16 @@ def __init__( "An API key is required for the hosted NIM. This will become an error in 0.2.0." ) + ## set default model + if not model: + model = DEFAULT_MODEL + warnings.warn( + f"Default model is set as: {model}. \n" + "Set model using model parameter. \n" + "To get available models use available_models property.", + UserWarning, + ) + super().__init__( model=model, temperature=temperature, diff --git a/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/tests/test_multi_modal_nvidia.py b/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/tests/test_multi_modal_nvidia.py index afe97b2d210c3..e4387884990bc 100644 --- a/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/tests/test_multi_modal_nvidia.py +++ b/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/tests/test_multi_modal_nvidia.py @@ -2,6 +2,7 @@ from llama_index.multi_modal_llms.nvidia import NVIDIAMultiModal from llama_index.multi_modal_llms.nvidia.utils import ( NVIDIA_MULTI_MODAL_MODELS, + DEFAULT_MODEL, ) import base64 import os @@ -434,3 +435,15 @@ async def test_vlm_chat_async_stream(vlm_model: str) -> None: async for token in await llm.astream_chat(messages): assert isinstance(token, ChatResponse) assert isinstance(token.delta, str) + + +def test_default_known() -> None: + """ + Test that a model in the model table will be accepted. + """ + # check if default model is getting set + with pytest.warns(UserWarning) as record: + x = NVIDIAMultiModal() + assert x.model == DEFAULT_MODEL + assert len(record) == 1 + assert f"Default model is set as: {DEFAULT_MODEL}" in str(record[0].message) From f0812fd062a2ef160080055a7f5fe66c2f015443 Mon Sep 17 00:00:00 2001 From: raspawar Date: Mon, 2 Dec 2024 22:11:01 +0530 Subject: [PATCH 2/3] version bump --- .../llama-index-multi-modal-llms-nvidia/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/pyproject.toml b/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/pyproject.toml index 7ddbfba1c7a4b..019199e1620fa 100644 --- a/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/pyproject.toml +++ b/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/pyproject.toml @@ -30,7 +30,7 @@ license = "MIT" name = "llama-index-multi-modal-llms-nvidia" packages = [{include = "llama_index/"}] readme = "README.md" -version = "0.3.0" +version = "0.3.1" [tool.poetry.dependencies] python = ">=3.9,<4.0" From c0ac5e26b6597374a1c57bb488b667166505a25a Mon Sep 17 00:00:00 2001 From: raspawar Date: Mon, 2 Dec 2024 22:37:13 +0530 Subject: [PATCH 3/3] update default model --- .../llama_index/multi_modal_llms/nvidia/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/llama_index/multi_modal_llms/nvidia/utils.py b/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/llama_index/multi_modal_llms/nvidia/utils.py index fa4d00bcd2a65..7401238bc9512 100644 --- a/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/llama_index/multi_modal_llms/nvidia/utils.py +++ b/llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-nvidia/llama_index/multi_modal_llms/nvidia/utils.py @@ -8,7 +8,7 @@ import urllib from llama_index.core.base.llms.types import ChatMessage -DEFAULT_MODEL = "microsoft/phi-3-vision-128k-instruct" +DEFAULT_MODEL = "nvidia/neva-22b" BASE_URL = "https://ai.api.nvidia.com/v1/" KNOWN_URLS = [