feat: add invalid_llm_response stop reason [LET-4083] (#4269)

* feat: add invalid_llm_response stop reason

* add sqllite support

* simply skip for sqllite

* fix imports

* fix isort

alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py

@@ -0,0 +1,39 @@
+"""convert_stop_reason_from_enum_to_string
+
+Revision ID: 887a4367b560
+Revises: d5103ee17ed5
+Create Date: 2025-08-27 16:34:45.605580
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+from letta.settings import settings
+
+# revision identifiers, used by Alembic.
+revision: str = "887a4367b560"
+down_revision: Union[str, None] = "d5103ee17ed5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Skip this migration for SQLite it doesn't enforce column types strictly,
+    # so the existing enum values will continue to work as strings.
+    if not settings.letta_pg_uri_no_default:
+        return
+
+    op.execute(
+        """
+        ALTER TABLE steps
+        ALTER COLUMN stop_reason TYPE VARCHAR
+        USING stop_reason::VARCHAR
+        """
+    )
+
+
+def downgrade() -> None:
+    # This is a one-way migration as we can't easily recreate the enum type
+    # If needed, you would need to create the enum type and cast back
+    pass

letta/agents/letta_agent.py

@@ -285,7 +285,11 @@ class LettaAgent(BaseAgent):
                 step_progression = StepProgression.RESPONSE_RECEIVED
                 log_event("agent.stream_no_tokens.llm_response.received")  # [3^]
 
-                response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)
+                try:
+                    response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)
+                except ValueError as e:
+                    stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)
+                    raise e
 
                 # update usage
                 usage.step_count += 1
@@ -395,8 +399,12 @@ class LettaAgent(BaseAgent):
                     stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)
                 elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):
                     self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)
-                elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):
-                    raise ValueError(f"Invalid Stop Reason: {stop_reason}")
+                elif stop_reason.stop_reason not in (
+                    StopReasonType.no_tool_call,
+                    StopReasonType.invalid_tool_call,
+                    StopReasonType.invalid_llm_response,
+                ):
+                    self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)
 
                 # Send error stop reason to client and re-raise
                 yield f"data: {stop_reason.model_dump_json()}\n\n", 500
@@ -582,7 +590,11 @@ class LettaAgent(BaseAgent):
                 step_progression = StepProgression.RESPONSE_RECEIVED
                 log_event("agent.step.llm_response.received")  # [3^]
 
-                response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)
+                try:
+                    response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)
+                except ValueError as e:
+                    stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)
+                    raise e
 
                 usage.step_count += 1
                 usage.completion_tokens += response.usage.completion_tokens
@@ -683,8 +695,12 @@ class LettaAgent(BaseAgent):
                     stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)
                 elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):
                     self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)
-                elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):
-                    raise ValueError(f"Invalid Stop Reason: {stop_reason}")
+                elif stop_reason.stop_reason not in (
+                    StopReasonType.no_tool_call,
+                    StopReasonType.invalid_tool_call,
+                    StopReasonType.invalid_llm_response,
+                ):
+                    self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)
                 raise
 
                 # Update step if it needs to be updated
@@ -1076,8 +1092,12 @@ class LettaAgent(BaseAgent):
                     stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)
                 elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):
                     self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)
-                elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):
-                    raise ValueError(f"Invalid Stop Reason: {stop_reason}")
+                elif stop_reason.stop_reason not in (
+                    StopReasonType.no_tool_call,
+                    StopReasonType.invalid_tool_call,
+                    StopReasonType.invalid_llm_response,
+                ):
+                    self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)
 
                 # Send error stop reason to client and re-raise with expected response code
                 yield f"data: {stop_reason.model_dump_json()}\n\n", 500

letta/orm/step.py

@@ -7,7 +7,6 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
 from letta.orm.mixins import ProjectMixin
 from letta.orm.sqlalchemy_base import SqlalchemyBase
 from letta.schemas.enums import StepStatus
-from letta.schemas.letta_stop_reason import StopReasonType
 from letta.schemas.step import Step as PydanticStep
 
 if TYPE_CHECKING:
@@ -51,7 +50,7 @@ class Step(SqlalchemyBase, ProjectMixin):
     prompt_tokens: Mapped[int] = mapped_column(default=0, doc="Number of tokens in the prompt")
     total_tokens: Mapped[int] = mapped_column(default=0, doc="Total number of tokens processed by the agent")
     completion_tokens_details: Mapped[Optional[Dict]] = mapped_column(JSON, nullable=True, doc="metadata for the agent.")
-    stop_reason: Mapped[Optional[StopReasonType]] = mapped_column(None, nullable=True, doc="The stop reason associated with this step.")
+    stop_reason: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="The stop reason associated with this step.")
     tags: Mapped[Optional[List]] = mapped_column(JSON, doc="Metadata tags.")
     tid: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="Transaction ID that processed the step.")
     trace_id: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="The trace id of the agent step.")

letta/schemas/letta_stop_reason.py

@@ -9,6 +9,7 @@ from letta.schemas.enums import JobStatus
 class StopReasonType(str, Enum):
     end_turn = "end_turn"
     error = "error"
+    invalid_llm_response = "invalid_llm_response"
     invalid_tool_call = "invalid_tool_call"
     max_steps = "max_steps"
     no_tool_call = "no_tool_call"
@@ -23,7 +24,12 @@ class StopReasonType(str, Enum):
             StopReasonType.tool_rule,
         ):
             return JobStatus.completed
-        elif self in (StopReasonType.error, StopReasonType.invalid_tool_call, StopReasonType.no_tool_call):
+        elif self in (
+            StopReasonType.error,
+            StopReasonType.invalid_tool_call,
+            StopReasonType.no_tool_call,
+            StopReasonType.invalid_llm_response,
+        ):
             return JobStatus.failed
         elif self == StopReasonType.cancelled:
             return JobStatus.cancelled