adjust image score (#169)

aigc-apps · Aug 28, 2024 · 2b488c1 · 2b488c1
1 parent 79c0a3f
commit 2b488c1
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/src/pai_rag/integrations/query_engine/multi_modal_query_engine.py b/src/pai_rag/integrations/query_engine/multi_modal_query_engine.py
@@ -27,6 +27,8 @@
 )
 
 IMAGE_MAX_PIECES = 5
+TEXT_IMAGE_WEIGHT = 0.5
+
 
 if TYPE_CHECKING:
     from llama_index.core.indices.multi_modal import MultiModalVectorIndexRetriever
@@ -74,6 +76,8 @@ def _get_image_and_text_nodes(
                     )
     image_nodes.sort(key=lambda x: x.score, reverse=True)
     text_image_nodes.sort(key=lambda x: x.score, reverse=True)
+    for text_image_node in text_image_nodes:
+        text_image_node.score *= TEXT_IMAGE_WEIGHT
     image_nodes.extend(text_image_nodes)
     image_nodes = image_nodes[:IMAGE_MAX_PIECES]
     return image_nodes, text_nodes

diff --git a/src/pai_rag/utils/prompt_template.py b/src/pai_rag/utils/prompt_template.py
@@ -125,7 +125,7 @@
     "---------------------\n\n"
     "{context_str}\n"
     "---------------------\n\n"
-    "请根据给定的材料回答给出的问题，如果材料中没有找到答案，就说没有找到相关的信息，不要编造答案。\n\n"
+    "请根据给定的材料回答给出的问题，回答中需要有文字描述和图片。如果材料中没有找到答案，就说没有找到相关的信息，不要编造答案。\n\n"
     "如果上面有图片对你生成答案有帮助，请从图片列表中找到对应的图片链接并用markdown格式给出，如![](image_url)。\n\n"
     "---------------------\n\n"
     "问题: {query_str}\n"