From 3e6a223d17dc83d584487f313bd7ae4393cbef70 Mon Sep 17 00:00:00 2001 From: cthomas Date: Wed, 27 Aug 2025 17:07:47 -0700 Subject: [PATCH] feat: add invalid_llm_response stop reason [LET-4083] (#4269) * feat: add invalid_llm_response stop reason * add sqllite support * simply skip for sqllite * fix imports * fix isort --- ...convert_stop_reason_from_enum_to_string.py | 39 +++++++++++++++++++ letta/agents/letta_agent.py | 36 +++++++++++++---- letta/orm/step.py | 3 +- letta/schemas/letta_stop_reason.py | 8 +++- 4 files changed, 75 insertions(+), 11 deletions(-) create mode 100644 alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py diff --git a/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py b/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py new file mode 100644 index 00000000..e3302993 --- /dev/null +++ b/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py @@ -0,0 +1,39 @@ +"""convert_stop_reason_from_enum_to_string + +Revision ID: 887a4367b560 +Revises: d5103ee17ed5 +Create Date: 2025-08-27 16:34:45.605580 + +""" + +from typing import Sequence, Union + +from alembic import op +from letta.settings import settings + +# revision identifiers, used by Alembic. +revision: str = "887a4367b560" +down_revision: Union[str, None] = "d5103ee17ed5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Skip this migration for SQLite it doesn't enforce column types strictly, + # so the existing enum values will continue to work as strings. + if not settings.letta_pg_uri_no_default: + return + + op.execute( + """ + ALTER TABLE steps + ALTER COLUMN stop_reason TYPE VARCHAR + USING stop_reason::VARCHAR + """ + ) + + +def downgrade() -> None: + # This is a one-way migration as we can't easily recreate the enum type + # If needed, you would need to create the enum type and cast back + pass diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.py index ea2472e0..6adf8597 100644 --- a/letta/agents/letta_agent.py +++ b/letta/agents/letta_agent.py @@ -285,7 +285,11 @@ class LettaAgent(BaseAgent): step_progression = StepProgression.RESPONSE_RECEIVED log_event("agent.stream_no_tokens.llm_response.received") # [3^] - response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config) + try: + response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config) + except ValueError as e: + stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value) + raise e # update usage usage.step_count += 1 @@ -395,8 +399,12 @@ class LettaAgent(BaseAgent): stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value) elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule): self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason) - elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call): - raise ValueError(f"Invalid Stop Reason: {stop_reason}") + elif stop_reason.stop_reason not in ( + StopReasonType.no_tool_call, + StopReasonType.invalid_tool_call, + StopReasonType.invalid_llm_response, + ): + self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason) # Send error stop reason to client and re-raise yield f"data: {stop_reason.model_dump_json()}\n\n", 500 @@ -582,7 +590,11 @@ class LettaAgent(BaseAgent): step_progression = StepProgression.RESPONSE_RECEIVED log_event("agent.step.llm_response.received") # [3^] - response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config) + try: + response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config) + except ValueError as e: + stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value) + raise e usage.step_count += 1 usage.completion_tokens += response.usage.completion_tokens @@ -683,8 +695,12 @@ class LettaAgent(BaseAgent): stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value) elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule): self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason) - elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call): - raise ValueError(f"Invalid Stop Reason: {stop_reason}") + elif stop_reason.stop_reason not in ( + StopReasonType.no_tool_call, + StopReasonType.invalid_tool_call, + StopReasonType.invalid_llm_response, + ): + self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason) raise # Update step if it needs to be updated @@ -1076,8 +1092,12 @@ class LettaAgent(BaseAgent): stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value) elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule): self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason) - elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call): - raise ValueError(f"Invalid Stop Reason: {stop_reason}") + elif stop_reason.stop_reason not in ( + StopReasonType.no_tool_call, + StopReasonType.invalid_tool_call, + StopReasonType.invalid_llm_response, + ): + self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason) # Send error stop reason to client and re-raise with expected response code yield f"data: {stop_reason.model_dump_json()}\n\n", 500 diff --git a/letta/orm/step.py b/letta/orm/step.py index a70e831f..49e90c42 100644 --- a/letta/orm/step.py +++ b/letta/orm/step.py @@ -7,7 +7,6 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship from letta.orm.mixins import ProjectMixin from letta.orm.sqlalchemy_base import SqlalchemyBase from letta.schemas.enums import StepStatus -from letta.schemas.letta_stop_reason import StopReasonType from letta.schemas.step import Step as PydanticStep if TYPE_CHECKING: @@ -51,7 +50,7 @@ class Step(SqlalchemyBase, ProjectMixin): prompt_tokens: Mapped[int] = mapped_column(default=0, doc="Number of tokens in the prompt") total_tokens: Mapped[int] = mapped_column(default=0, doc="Total number of tokens processed by the agent") completion_tokens_details: Mapped[Optional[Dict]] = mapped_column(JSON, nullable=True, doc="metadata for the agent.") - stop_reason: Mapped[Optional[StopReasonType]] = mapped_column(None, nullable=True, doc="The stop reason associated with this step.") + stop_reason: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="The stop reason associated with this step.") tags: Mapped[Optional[List]] = mapped_column(JSON, doc="Metadata tags.") tid: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="Transaction ID that processed the step.") trace_id: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="The trace id of the agent step.") diff --git a/letta/schemas/letta_stop_reason.py b/letta/schemas/letta_stop_reason.py index 8a0955f7..60365ef4 100644 --- a/letta/schemas/letta_stop_reason.py +++ b/letta/schemas/letta_stop_reason.py @@ -9,6 +9,7 @@ from letta.schemas.enums import JobStatus class StopReasonType(str, Enum): end_turn = "end_turn" error = "error" + invalid_llm_response = "invalid_llm_response" invalid_tool_call = "invalid_tool_call" max_steps = "max_steps" no_tool_call = "no_tool_call" @@ -23,7 +24,12 @@ class StopReasonType(str, Enum): StopReasonType.tool_rule, ): return JobStatus.completed - elif self in (StopReasonType.error, StopReasonType.invalid_tool_call, StopReasonType.no_tool_call): + elif self in ( + StopReasonType.error, + StopReasonType.invalid_tool_call, + StopReasonType.no_tool_call, + StopReasonType.invalid_llm_response, + ): return JobStatus.failed elif self == StopReasonType.cancelled: return JobStatus.cancelled