From a9cd78cc3bc634575a14ce5bfb2fbb60b36231c9 Mon Sep 17 00:00:00 2001 From: cthomas Date: Mon, 28 Apr 2025 16:37:29 -0700 Subject: [PATCH] fix: put inner thoughts in kwargs for anthropic (#1911) --- letta/llm_api/llm_api_tools.py | 3 +++ letta/schemas/llm_config.py | 3 +++ letta/schemas/providers.py | 3 ++- letta/server/server.py | 2 ++ .../configs/llm_model_configs/claude-3-7-sonnet-extended.json | 2 +- 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/letta/llm_api/llm_api_tools.py b/letta/llm_api/llm_api_tools.py index ffc88ef6..be1b9d82 100644 --- a/letta/llm_api/llm_api_tools.py +++ b/letta/llm_api/llm_api_tools.py @@ -327,6 +327,9 @@ def create( if not use_tool_naming: raise NotImplementedError("Only tool calling supported on Anthropic API requests") + if llm_config.enable_reasoner: + llm_config.put_inner_thoughts_in_kwargs = False + # Force tool calling tool_call = None if functions is None: diff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.py index f94e5f35..3dc2c92e 100644 --- a/letta/schemas/llm_config.py +++ b/letta/schemas/llm_config.py @@ -111,6 +111,9 @@ class LLMConfig(BaseModel): if is_openai_reasoning_model(model): values["put_inner_thoughts_in_kwargs"] = False + if values.get("enable_reasoner") and values.get("model_endpoint_type") == "anthropic": + values["put_inner_thoughts_in_kwargs"] = False + return values @model_validator(mode="after") diff --git a/letta/schemas/providers.py b/letta/schemas/providers.py index 3a20aea7..f5198d7f 100644 --- a/letta/schemas/providers.py +++ b/letta/schemas/providers.py @@ -744,7 +744,8 @@ class AnthropicProvider(Provider): # reliable for tool calling (no chance of a non-tool call step) # Since tool_choice_type 'any' doesn't work with in-content COT # NOTE For Haiku, it can be flaky if we don't enable this by default - inner_thoughts_in_kwargs = True if "haiku" in model["id"] else False + # inner_thoughts_in_kwargs = True if "haiku" in model["id"] else False + inner_thoughts_in_kwargs = True # we no longer support thinking tags configs.append( LLMConfig( diff --git a/letta/server/server.py b/letta/server/server.py index 681ae6c9..e532d467 100644 --- a/letta/server/server.py +++ b/letta/server/server.py @@ -1211,6 +1211,8 @@ class SyncServer(Server): llm_config.max_reasoning_tokens = max_reasoning_tokens if enable_reasoner is not None: llm_config.enable_reasoner = enable_reasoner + if enable_reasoner and llm_config.model_endpoint_type == "anthropic": + llm_config.put_inner_thoughts_in_kwargs = False return llm_config diff --git a/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json b/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json index a7abf66a..f38ffb41 100644 --- a/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json +++ b/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json @@ -4,7 +4,7 @@ "model_endpoint": "https://api.anthropic.com/v1", "model_wrapper": null, "context_window": 200000, - "put_inner_thoughts_in_kwargs": false, + "put_inner_thoughts_in_kwargs": true, "enable_reasoner": true, "max_reasoning_tokens": 1024 }