From 7dc98969a825853dbff8a5b5ad5dea5b30fbac95 Mon Sep 17 00:00:00 2001 From: Matthew Zhou Date: Fri, 14 Feb 2025 11:59:32 -0800 Subject: [PATCH] feat: Fix VLLM tool_choice (#1001) --- letta/llm_api/llm_api_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letta/llm_api/llm_api_tools.py b/letta/llm_api/llm_api_tools.py index 65bdc1f1..e4bc63f6 100644 --- a/letta/llm_api/llm_api_tools.py +++ b/letta/llm_api/llm_api_tools.py @@ -151,7 +151,8 @@ def create( if function_call is None and functions is not None and len(functions) > 0: # force function calling for reliability, see https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice # TODO(matt) move into LLMConfig - if llm_config.model_endpoint == "https://inference.memgpt.ai": + # TODO: This vllm checking is very brittle and is a patch at most + if llm_config.model_endpoint == "https://inference.memgpt.ai" or (llm_config.handle and "vllm" in llm_config.handle): function_call = "auto" # TODO change to "required" once proxy supports it else: function_call = "required"