From bdd3aa4457fe57ac7e1aab91cda1ede696798709 Mon Sep 17 00:00:00 2001
From: Charles Packer <packercharles@gmail.com>
Date: Mon, 22 Apr 2024 20:20:17 -0700
Subject: [PATCH] fix: hardcoded stop tokens to patch Groq API's new 4 stop
 token limit for `/completions` (#1288)

---
 memgpt/local_llm/groq/api.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/memgpt/local_llm/groq/api.py b/memgpt/local_llm/groq/api.py
index 289f047f..9c8df23f 100644
--- a/memgpt/local_llm/groq/api.py
+++ b/memgpt/local_llm/groq/api.py
@@ -26,6 +26,26 @@ def get_groq_completion(endpoint: str, auth_type: str, auth_key: str, model: str
             # "top_p",
             # "stream",
             # "stop",
+            # Groq only allows 4 stop tokens
+            "stop": [
+                "\nUSER",
+                "\nASSISTANT",
+                "\nFUNCTION",
+                # "\nFUNCTION RETURN",
+                # "<|im_start|>",
+                # "<|im_end|>",
+                # "<|im_sep|>",
+                # # airoboros specific
+                # "\n### ",
+                # # '\n' +
+                # # '</s>',
+                # # '<|',
+                # "\n#",
+                # # "\n\n\n",
+                # # prevent chaining function calls / multi json objects / run-on generations
+                # # NOTE: this requires the ability to patch the extra '}}' back into the prompt
+                "  }\n}\n",
+            ]
         }
     )