From 881831501a304cd80fb2644bc64c18ad5a5794f0 Mon Sep 17 00:00:00 2001 From: Christina Tong Date: Wed, 5 Nov 2025 13:58:51 -0800 Subject: [PATCH] feat: filter list agents by stop reason [LET-5928] (#5779) * feat: add last_stop_reason to AgentState [LET-5911] * feat: filter list agents by stop reason [LET-5928] * undo agent loop changes, use update_run_by_id_async * add run manager test * add integration tests * remove comment * fix duplicate * fix docs --- fern/openapi.json | 18 +++++ letta/server/rest_api/routers/v1/agents.py | 2 + letta/services/agent_manager.py | 5 +- .../services/helpers/agent_manager_helper.py | 9 ++- tests/managers/test_agent_manager.py | 66 +++++++++++++++++++ .../integration_test_human_in_the_loop.py | 47 +++++++++++++ 6 files changed, 145 insertions(+), 2 deletions(-) diff --git a/fern/openapi.json b/fern/openapi.json index 39682b97..c119f6f2 100644 --- a/fern/openapi.json +++ b/fern/openapi.json @@ -4091,6 +4091,24 @@ }, "description": "Field to sort by. Options: 'created_at' (default), 'last_run_completion'", "deprecated": true + }, + { + "name": "last_stop_reason", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/StopReasonType" + }, + { + "type": "null" + } + ], + "description": "Filter agents by their last stop reason.", + "title": "Last Stop Reason" + }, + "description": "Filter agents by their last stop reason." } ], "responses": { diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.py index 558c9422..f081aa11 100644 --- a/letta/server/rest_api/routers/v1/agents.py +++ b/letta/server/rest_api/routers/v1/agents.py @@ -124,6 +124,7 @@ async def list_agents( include_in_schema=False, description="If set to True, include agents marked as hidden in the results.", ), + last_stop_reason: Optional[StopReasonType] = Query(None, description="Filter agents by their last stop reason."), ): """ Get a list of all agents. @@ -158,6 +159,7 @@ async def list_agents( ascending=final_ascending, sort_by=final_sort_by, show_hidden_agents=show_hidden_agents, + last_stop_reason=last_stop_reason, ) diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.py index 7f40a628..ab616e6b 100644 --- a/letta/services/agent_manager.py +++ b/letta/services/agent_manager.py @@ -62,6 +62,7 @@ from letta.schemas.embedding_config import EmbeddingConfig from letta.schemas.enums import AgentType, PrimitiveType, ProviderType, TagMatchMode, ToolType, VectorDBProvider from letta.schemas.file import FileMetadata as PydanticFileMetadata from letta.schemas.group import Group as PydanticGroup, ManagerType +from letta.schemas.letta_stop_reason import StopReasonType from letta.schemas.llm_config import LLMConfig from letta.schemas.memory import ContextWindowOverview, Memory from letta.schemas.message import Message, Message as PydanticMessage, MessageCreate, MessageUpdate @@ -873,6 +874,7 @@ class AgentManager: ascending: bool = True, sort_by: Optional[str] = "created_at", show_hidden_agents: Optional[bool] = None, + last_stop_reason: Optional[StopReasonType] = None, ) -> List[PydanticAgentState]: """ Retrieves agents with optimized filtering and optional field selection. @@ -895,6 +897,7 @@ class AgentManager: ascending (bool): Sort agents in ascending order. sort_by (Optional[str]): Sort agents by this field. show_hidden_agents (bool): If True, include agents marked as hidden in the results. + last_stop_reason (Optional[str]): Filter by the agent's last stop reason (e.g., 'requires_approval', 'error'). Returns: List[PydanticAgentState]: The filtered list of matching agents. @@ -904,7 +907,7 @@ class AgentManager: query = AgentModel.apply_access_predicate(query, actor, ["read"], AccessType.ORGANIZATION) # Apply filters - query = _apply_filters(query, name, query_text, project_id, template_id, base_template_id) + query = _apply_filters(query, name, query_text, project_id, template_id, base_template_id, last_stop_reason) query = _apply_identity_filters(query, identity_id, identifier_keys) query = _apply_tag_filter(query, tags, match_all_tags) query = _apply_relationship_filters(query, include_relationships, include) diff --git a/letta/services/helpers/agent_manager_helper.py b/letta/services/helpers/agent_manager_helper.py index 30e429e8..5416c715 100644 --- a/letta/services/helpers/agent_manager_helper.py +++ b/letta/services/helpers/agent_manager_helper.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import List, Literal, Optional, Set from letta.log import get_logger +from letta.schemas.letta_stop_reason import StopReasonType logger = get_logger(__name__) @@ -738,12 +739,14 @@ def _apply_filters( project_id: Optional[str], template_id: Optional[str], base_template_id: Optional[str], + last_stop_reason: Optional[StopReasonType] = None, ): """ Apply basic filtering criteria to the agent query. This helper function adds WHERE clauses based on provided parameters such as - exact name, partial name match (using ILIKE), project ID, template ID, and base template ID. + exact name, partial name match (using ILIKE), project ID, template ID, base template ID, + and last stop reason. Args: query: The SQLAlchemy query object to be modified. @@ -752,6 +755,7 @@ def _apply_filters( project_id (Optional[str]): Filter for agents belonging to a specific project. template_id (Optional[str]): Filter for agents using a specific template. base_template_id (Optional[str]): Filter for agents using a specific base template. + last_stop_reason (Optional[StopReasonType]): Filter for agents by their last stop reason (e.g., 'requires_approval', 'error'). Returns: The modified query with the applied filters. @@ -776,6 +780,9 @@ def _apply_filters( # Filter agents by base template ID. if base_template_id: query = query.where(AgentModel.base_template_id == base_template_id) + # Filter agents by last stop reason. + if last_stop_reason: + query = query.where(AgentModel.last_stop_reason == last_stop_reason) return query diff --git a/tests/managers/test_agent_manager.py b/tests/managers/test_agent_manager.py index 5493f6cf..bcd482dd 100644 --- a/tests/managers/test_agent_manager.py +++ b/tests/managers/test_agent_manager.py @@ -835,6 +835,72 @@ async def test_list_agents_descending(server: SyncServer, default_user): assert names.index("agent_newest") < names.index("agent_oldest") +@pytest.mark.asyncio +async def test_list_agents_by_last_stop_reason(server: SyncServer, default_user): + # Create agent with requires_approval stop reason + agent1 = await server.agent_manager.create_agent_async( + agent_create=CreateAgent( + name="agent_requires_approval", + llm_config=LLMConfig.default_config("gpt-4o-mini"), + embedding_config=EmbeddingConfig.default_config(provider="openai"), + memory_blocks=[], + include_base_tools=False, + ), + actor=default_user, + ) + await server.agent_manager.update_agent_async( + agent_id=agent1.id, + agent_update=UpdateAgent(last_stop_reason=StopReasonType.requires_approval), + actor=default_user, + ) + + # Create agent with error stop reason + agent2 = await server.agent_manager.create_agent_async( + agent_create=CreateAgent( + name="agent_error", + llm_config=LLMConfig.default_config("gpt-4o-mini"), + embedding_config=EmbeddingConfig.default_config(provider="openai"), + memory_blocks=[], + include_base_tools=False, + ), + actor=default_user, + ) + await server.agent_manager.update_agent_async( + agent_id=agent2.id, + agent_update=UpdateAgent(last_stop_reason=StopReasonType.error), + actor=default_user, + ) + + # Create agent with no stop reason + agent3 = await server.agent_manager.create_agent_async( + agent_create=CreateAgent( + name="agent_no_stop_reason", + llm_config=LLMConfig.default_config("gpt-4o-mini"), + embedding_config=EmbeddingConfig.default_config(provider="openai"), + memory_blocks=[], + include_base_tools=False, + ), + actor=default_user, + ) + + # Filter by requires_approval + approval_agents = await server.agent_manager.list_agents_async( + actor=default_user, last_stop_reason=StopReasonType.requires_approval.value + ) + approval_names = {agent.name for agent in approval_agents} + assert approval_names == {"agent_requires_approval"} + + # Filter by error + error_agents = await server.agent_manager.list_agents_async(actor=default_user, last_stop_reason=StopReasonType.error.value) + error_names = {agent.name for agent in error_agents} + assert error_names == {"agent_error"} + + # No filter - should return all agents + all_agents = await server.agent_manager.list_agents_async(actor=default_user) + all_names = {agent.name for agent in all_agents} + assert {"agent_requires_approval", "agent_error", "agent_no_stop_reason"}.issubset(all_names) + + @pytest.mark.asyncio async def test_list_agents_ordering_and_pagination(server: SyncServer, default_user): names = ["alpha_agent", "beta_agent", "gamma_agent"] diff --git a/tests/sdk_v1/integration/integration_test_human_in_the_loop.py b/tests/sdk_v1/integration/integration_test_human_in_the_loop.py index 6b2c6a08..0c546d7d 100644 --- a/tests/sdk_v1/integration/integration_test_human_in_the_loop.py +++ b/tests/sdk_v1/integration/integration_test_human_in_the_loop.py @@ -768,6 +768,53 @@ def test_deny_and_follow_up( assert messages[3].message_type == "usage_statistics" +def test_agent_records_last_stop_reason_after_approval_flow( + client: Letta, + agent: AgentState, +) -> None: + """ + Test that the agent's last_stop_reason is properly updated after a human-in-the-loop flow. + This verifies the integration between run completion and agent state updates. + """ + # Get initial agent state + initial_agent = client.agents.retrieve(agent_id=agent.id) + initial_stop_reason = initial_agent.last_stop_reason + + # Trigger approval request + response = client.agents.messages.send( + agent_id=agent.id, + messages=USER_MESSAGE_TEST_APPROVAL, + ) + + # Verify we got an approval request + messages = response.messages + assert messages is not None + assert len(messages) == 3 + assert messages[2].message_type == "approval_request_message" + + # Check agent after approval request (run should be paused with requires_approval) + agent_after_request = client.agents.retrieve(agent_id=agent.id) + assert agent_after_request.last_stop_reason == "requires_approval" + + # Approve the tool call + approve_tool_call(client, agent.id, response.messages[2].tool_call.tool_call_id) + + # Check agent after approval (run should complete with end_turn or similar) + agent_after_approval = client.agents.retrieve(agent_id=agent.id) + assert agent_after_approval.last_stop_reason is not None + assert agent_after_approval.last_stop_reason != initial_stop_reason + + # Send follow-up message to complete the flow + response2 = client.agents.messages.send( + agent_id=agent.id, + messages=USER_MESSAGE_FOLLOW_UP, + ) + + # Verify final agent state has the most recent stop reason + final_agent = client.agents.retrieve(agent_id=agent.id) + assert final_agent.last_stop_reason is not None + + # -------------------------------- # Client-Side Execution Test Cases # --------------------------------