feat: updated backend to not allow minimal for codex [LET-5883] (#5760)
* updated backend * add function in openai_client * remove values before error * remove test --------- Co-authored-by: Ari Webb <ari@letta.com>
This commit is contained in:
@@ -64,6 +64,14 @@ def is_openai_reasoning_model(model: str) -> bool:
|
||||
return is_reasoning
|
||||
|
||||
|
||||
def does_not_support_minimal_reasoning(model: str) -> bool:
|
||||
"""Check if the model does not support minimal reasoning effort.
|
||||
|
||||
Currently, models that contain codex don't support minimal reasoning.
|
||||
"""
|
||||
return "codex" in model.lower()
|
||||
|
||||
|
||||
def is_openai_5_model(model: str) -> bool:
|
||||
"""Utility function to check if the model is a '5' model"""
|
||||
return model.startswith("gpt-5")
|
||||
|
||||
@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Literal, Optional
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from letta.constants import LETTA_MODEL_ENDPOINT
|
||||
from letta.errors import LettaInvalidArgumentError
|
||||
from letta.log import get_logger
|
||||
from letta.schemas.enums import AgentType, ProviderCategory
|
||||
|
||||
@@ -163,6 +164,24 @@ class LLMConfig(BaseModel):
|
||||
|
||||
return values
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def validate_codex_reasoning_effort(cls, values):
|
||||
"""
|
||||
Validate that gpt-5-codex models do not use 'minimal' reasoning effort.
|
||||
Codex models require at least 'low' reasoning effort.
|
||||
"""
|
||||
from letta.llm_api.openai_client import does_not_support_minimal_reasoning
|
||||
|
||||
model = values.get("model")
|
||||
reasoning_effort = values.get("reasoning_effort")
|
||||
|
||||
if model and does_not_support_minimal_reasoning(model) and reasoning_effort == "minimal":
|
||||
raise LettaInvalidArgumentError(
|
||||
f"Model '{model}' does not support 'minimal' reasoning effort. Please use 'low', 'medium', or 'high' instead."
|
||||
)
|
||||
return values
|
||||
|
||||
@classmethod
|
||||
def default_config(cls, model_name: str):
|
||||
"""
|
||||
@@ -277,6 +296,8 @@ class LLMConfig(BaseModel):
|
||||
- Google Gemini (2.5 family): force disabled until native reasoning supported
|
||||
- All others: disabled (no simulated reasoning via kwargs)
|
||||
"""
|
||||
from letta.llm_api.openai_client import does_not_support_minimal_reasoning
|
||||
|
||||
# V1 agent policy: do not allow simulated reasoning for non-native models
|
||||
if agent_type is not None and agent_type == AgentType.letta_v1_agent:
|
||||
# OpenAI native reasoning models: always on
|
||||
@@ -284,7 +305,8 @@ class LLMConfig(BaseModel):
|
||||
config.put_inner_thoughts_in_kwargs = False
|
||||
config.enable_reasoner = True
|
||||
if config.reasoning_effort is None:
|
||||
if config.model.startswith("gpt-5"):
|
||||
# Codex models cannot use "minimal" reasoning effort
|
||||
if config.model.startswith("gpt-5") and not does_not_support_minimal_reasoning(config.model):
|
||||
config.reasoning_effort = "minimal"
|
||||
else:
|
||||
config.reasoning_effort = "medium"
|
||||
@@ -324,7 +346,8 @@ class LLMConfig(BaseModel):
|
||||
config.enable_reasoner = True
|
||||
if config.reasoning_effort is None:
|
||||
# GPT-5 models default to minimal, others to medium
|
||||
if config.model.startswith("gpt-5"):
|
||||
# Codex models cannot use "minimal" reasoning effort
|
||||
if config.model.startswith("gpt-5") and not does_not_support_minimal_reasoning(config.model):
|
||||
config.reasoning_effort = "minimal"
|
||||
else:
|
||||
config.reasoning_effort = "medium"
|
||||
@@ -357,7 +380,8 @@ class LLMConfig(BaseModel):
|
||||
config.put_inner_thoughts_in_kwargs = False
|
||||
if config.reasoning_effort is None:
|
||||
# GPT-5 models default to minimal, others to medium
|
||||
if config.model.startswith("gpt-5"):
|
||||
# Codex models cannot use "minimal" reasoning effort
|
||||
if config.model.startswith("gpt-5") and not does_not_support_minimal_reasoning(config.model):
|
||||
config.reasoning_effort = "minimal"
|
||||
else:
|
||||
config.reasoning_effort = "medium"
|
||||
|
||||
@@ -382,3 +382,25 @@ def test_reasoning_toggle_by_provider(
|
||||
assert new_config.put_inner_thoughts_in_kwargs == expected_put_inner_thoughts_in_kwargs
|
||||
assert new_config.reasoning_effort == expected_reasoning_effort
|
||||
assert new_config.max_reasoning_tokens == expected_max_reasoning_tokens
|
||||
|
||||
|
||||
def test_codex_default_reasoning_effort():
|
||||
"""Test that gpt-5-codex defaults to 'medium' reasoning effort, not 'minimal'."""
|
||||
# Test with apply_reasoning_setting_to_config for v2 agent
|
||||
config = LLMConfig(
|
||||
model="gpt-5-codex",
|
||||
model_endpoint_type="openai",
|
||||
context_window=272000,
|
||||
)
|
||||
|
||||
# For v2 agent with reasoning=True
|
||||
new_config = LLMConfig.apply_reasoning_setting_to_config(config, reasoning=True, agent_type=AgentType.memgpt_v2_agent)
|
||||
assert new_config.reasoning_effort == "medium", "gpt-5-codex should default to 'medium', not 'minimal'"
|
||||
|
||||
# For v2 agent with reasoning=False (still can't disable for reasoning models)
|
||||
new_config = LLMConfig.apply_reasoning_setting_to_config(config, reasoning=False, agent_type=AgentType.memgpt_v2_agent)
|
||||
assert new_config.reasoning_effort == "medium", "gpt-5-codex should default to 'medium', not 'minimal'"
|
||||
|
||||
# For v1 agent with reasoning=True
|
||||
new_config = LLMConfig.apply_reasoning_setting_to_config(config, reasoning=True, agent_type=AgentType.letta_v1_agent)
|
||||
assert new_config.reasoning_effort == "medium", "gpt-5-codex should default to 'medium', not 'minimal'"
|
||||
|
||||
Reference in New Issue
Block a user