fix: hardcoded stop tokens to patch Groq API's new 4 stop token limit…

… for `/completions` (#1288)
letta-ai · Apr 23, 2024 · 4e67403 · 4e67403
1 parent 7d328fe
commit 4e67403
Showing 1 changed file with 20 additions and 0 deletions.
diff --git a/memgpt/local_llm/groq/api.py b/memgpt/local_llm/groq/api.py
@@ -26,6 +26,26 @@ def get_groq_completion(endpoint: str, auth_type: str, auth_key: str, model: str
             # "top_p",
             # "stream",
             # "stop",
+            # Groq only allows 4 stop tokens
+            "stop": [
+                "\nUSER",
+                "\nASSISTANT",
+                "\nFUNCTION",
+                # "\nFUNCTION RETURN",
+                # "<|im_start|>",
+                # "<|im_end|>",
+                # "<|im_sep|>",
+                # # airoboros specific
+                # "\n### ",
+                # # '\n' +
+                # # '</s>',
+                # # '<|',
+                # "\n#",
+                # # "\n\n\n",
+                # # prevent chaining function calls / multi json objects / run-on generations
+                # # NOTE: this requires the ability to patch the extra '}}' back into the prompt
+                "  }\n}\n",
+            ]
         }
     )