feat: updated backend to not allow minimal for codex [LET-5883] (#5760)

* updated backend

* add function in openai_client

* remove values before error

* remove test

---------

Co-authored-by: Ari Webb <ari@letta.com>
This commit is contained in:
Ari Webb
2025-10-27 18:31:58 -07:00
committed by Caren Thomas
parent e3b0398aee
commit f3a40a41f5
3 changed files with 57 additions and 3 deletions

View File

@@ -64,6 +64,14 @@ def is_openai_reasoning_model(model: str) -> bool:
return is_reasoning
def does_not_support_minimal_reasoning(model: str) -> bool:
"""Check if the model does not support minimal reasoning effort.
Currently, models that contain codex don't support minimal reasoning.
"""
return "codex" in model.lower()
def is_openai_5_model(model: str) -> bool:
"""Utility function to check if the model is a '5' model"""
return model.startswith("gpt-5")

View File

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Literal, Optional
from pydantic import BaseModel, ConfigDict, Field, model_validator
from letta.constants import LETTA_MODEL_ENDPOINT
from letta.errors import LettaInvalidArgumentError
from letta.log import get_logger
from letta.schemas.enums import AgentType, ProviderCategory
@@ -163,6 +164,24 @@ class LLMConfig(BaseModel):
return values
@model_validator(mode="before")
@classmethod
def validate_codex_reasoning_effort(cls, values):
"""
Validate that gpt-5-codex models do not use 'minimal' reasoning effort.
Codex models require at least 'low' reasoning effort.
"""
from letta.llm_api.openai_client import does_not_support_minimal_reasoning
model = values.get("model")
reasoning_effort = values.get("reasoning_effort")
if model and does_not_support_minimal_reasoning(model) and reasoning_effort == "minimal":
raise LettaInvalidArgumentError(
f"Model '{model}' does not support 'minimal' reasoning effort. Please use 'low', 'medium', or 'high' instead."
)
return values
@classmethod
def default_config(cls, model_name: str):
"""
@@ -277,6 +296,8 @@ class LLMConfig(BaseModel):
- Google Gemini (2.5 family): force disabled until native reasoning supported
- All others: disabled (no simulated reasoning via kwargs)
"""
from letta.llm_api.openai_client import does_not_support_minimal_reasoning
# V1 agent policy: do not allow simulated reasoning for non-native models
if agent_type is not None and agent_type == AgentType.letta_v1_agent:
# OpenAI native reasoning models: always on
@@ -284,7 +305,8 @@ class LLMConfig(BaseModel):
config.put_inner_thoughts_in_kwargs = False
config.enable_reasoner = True
if config.reasoning_effort is None:
if config.model.startswith("gpt-5"):
# Codex models cannot use "minimal" reasoning effort
if config.model.startswith("gpt-5") and not does_not_support_minimal_reasoning(config.model):
config.reasoning_effort = "minimal"
else:
config.reasoning_effort = "medium"
@@ -324,7 +346,8 @@ class LLMConfig(BaseModel):
config.enable_reasoner = True
if config.reasoning_effort is None:
# GPT-5 models default to minimal, others to medium
if config.model.startswith("gpt-5"):
# Codex models cannot use "minimal" reasoning effort
if config.model.startswith("gpt-5") and not does_not_support_minimal_reasoning(config.model):
config.reasoning_effort = "minimal"
else:
config.reasoning_effort = "medium"
@@ -357,7 +380,8 @@ class LLMConfig(BaseModel):
config.put_inner_thoughts_in_kwargs = False
if config.reasoning_effort is None:
# GPT-5 models default to minimal, others to medium
if config.model.startswith("gpt-5"):
# Codex models cannot use "minimal" reasoning effort
if config.model.startswith("gpt-5") and not does_not_support_minimal_reasoning(config.model):
config.reasoning_effort = "minimal"
else:
config.reasoning_effort = "medium"

View File

@@ -382,3 +382,25 @@ def test_reasoning_toggle_by_provider(
assert new_config.put_inner_thoughts_in_kwargs == expected_put_inner_thoughts_in_kwargs
assert new_config.reasoning_effort == expected_reasoning_effort
assert new_config.max_reasoning_tokens == expected_max_reasoning_tokens
def test_codex_default_reasoning_effort():
"""Test that gpt-5-codex defaults to 'medium' reasoning effort, not 'minimal'."""
# Test with apply_reasoning_setting_to_config for v2 agent
config = LLMConfig(
model="gpt-5-codex",
model_endpoint_type="openai",
context_window=272000,
)
# For v2 agent with reasoning=True
new_config = LLMConfig.apply_reasoning_setting_to_config(config, reasoning=True, agent_type=AgentType.memgpt_v2_agent)
assert new_config.reasoning_effort == "medium", "gpt-5-codex should default to 'medium', not 'minimal'"
# For v2 agent with reasoning=False (still can't disable for reasoning models)
new_config = LLMConfig.apply_reasoning_setting_to_config(config, reasoning=False, agent_type=AgentType.memgpt_v2_agent)
assert new_config.reasoning_effort == "medium", "gpt-5-codex should default to 'medium', not 'minimal'"
# For v1 agent with reasoning=True
new_config = LLMConfig.apply_reasoning_setting_to_config(config, reasoning=True, agent_type=AgentType.letta_v1_agent)
assert new_config.reasoning_effort == "medium", "gpt-5-codex should default to 'medium', not 'minimal'"