From 44574ec2645aee12c2e3b0382b22f6d415b5b509 Mon Sep 17 00:00:00 2001 From: Christina Tong Date: Tue, 21 Oct 2025 14:54:55 -0700 Subject: [PATCH] feat: add internal runs route with template_family filtering [LET-5416] (#5543) * feat: add tool_used field to run_metrics [LET-5419] * change to tool name * use tool ids over names * feat: add internal runs route with template_family filtering * remove import * add auto generated * restrict internal runs * add test, address comments * add docs and auto generated fields * remove unused template mixins * update openapi * add generated --- fern/openapi-overrides.yml | 3 + fern/openapi.json | 283 +++++++++++++++++- letta/schemas/run.py | 4 + letta/server/rest_api/routers/v1/__init__.py | 2 + .../rest_api/routers/v1/internal_runs.py | 98 ++++++ letta/services/run_manager.py | 5 + tests/managers/test_run_manager.py | 13 + 7 files changed, 399 insertions(+), 9 deletions(-) create mode 100644 letta/server/rest_api/routers/v1/internal_runs.py diff --git a/fern/openapi-overrides.yml b/fern/openapi-overrides.yml index 202b4453..ba32c53a 100644 --- a/fern/openapi-overrides.yml +++ b/fern/openapi-overrides.yml @@ -1147,6 +1147,9 @@ paths: /v1/_internal_templates/blocks/batch: post: x-fern-ignore: true + /v1/_internal_runs/: + get: + x-fern-ignore: true /v1/projects: get: x-fern-sdk-group-name: diff --git a/fern/openapi.json b/fern/openapi.json index f8f4234c..85635997 100644 --- a/fern/openapi.json +++ b/fern/openapi.json @@ -3899,14 +3899,8 @@ "required": true, "schema": { "type": "string", - "minLength": 42, - "maxLength": 42, - "pattern": "^agent-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", - "description": "The ID of the agent in the format 'agent-'", - "examples": ["agent-123e4567-e89b-42d3-8456-426614174000"], "title": "Agent Id" - }, - "description": "The ID of the agent in the format 'agent-'" + } }, { "name": "max_steps", @@ -8880,6 +8874,265 @@ } } }, + "/v1/_internal_runs/": { + "get": { + "tags": ["_internal_runs"], + "summary": "List Runs", + "description": "List all runs.", + "operationId": "list_runs", + "parameters": [ + { + "name": "agent_id", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The unique identifier of the agent associated with the run.", + "title": "Agent Id" + }, + "description": "The unique identifier of the agent associated with the run." + }, + { + "name": "agent_ids", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ], + "description": "The unique identifiers of the agents associated with the run. Deprecated in favor of agent_id field.", + "deprecated": true, + "title": "Agent Ids" + }, + "description": "The unique identifiers of the agents associated with the run. Deprecated in favor of agent_id field.", + "deprecated": true + }, + { + "name": "statuses", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ], + "description": "Filter runs by status. Can specify multiple statuses.", + "title": "Statuses" + }, + "description": "Filter runs by status. Can specify multiple statuses." + }, + { + "name": "background", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "If True, filters for runs that were created in background mode.", + "title": "Background" + }, + "description": "If True, filters for runs that were created in background mode." + }, + { + "name": "stop_reason", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/StopReasonType" + }, + { + "type": "null" + } + ], + "description": "Filter runs by stop reason.", + "title": "Stop Reason" + }, + "description": "Filter runs by stop reason." + }, + { + "name": "template_family", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Filter runs by template family (base_template_id).", + "title": "Template Family" + }, + "description": "Filter runs by template family (base_template_id)." + }, + { + "name": "before", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Run ID cursor for pagination. Returns runs that come before this run ID in the specified sort order", + "title": "Before" + }, + "description": "Run ID cursor for pagination. Returns runs that come before this run ID in the specified sort order" + }, + { + "name": "after", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Run ID cursor for pagination. Returns runs that come after this run ID in the specified sort order", + "title": "After" + }, + "description": "Run ID cursor for pagination. Returns runs that come after this run ID in the specified sort order" + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Maximum number of runs to return", + "default": 100, + "title": "Limit" + }, + "description": "Maximum number of runs to return" + }, + { + "name": "order", + "in": "query", + "required": false, + "schema": { + "enum": ["asc", "desc"], + "type": "string", + "description": "Sort order for runs by creation time. 'asc' for oldest first, 'desc' for newest first", + "default": "desc", + "title": "Order" + }, + "description": "Sort order for runs by creation time. 'asc' for oldest first, 'desc' for newest first" + }, + { + "name": "order_by", + "in": "query", + "required": false, + "schema": { + "const": "created_at", + "type": "string", + "description": "Field to sort by", + "default": "created_at", + "title": "Order By" + }, + "description": "Field to sort by" + }, + { + "name": "active", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "description": "Filter for active runs.", + "default": false, + "title": "Active" + }, + "description": "Filter for active runs." + }, + { + "name": "ascending", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "description": "Whether to sort agents oldest to newest (True) or newest to oldest (False, default). Deprecated in favor of order field.", + "deprecated": true, + "default": false, + "title": "Ascending" + }, + "description": "Whether to sort agents oldest to newest (True) or newest to oldest (False, default). Deprecated in favor of order field.", + "deprecated": true + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Run" + }, + "title": "Response List Runs" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v1/_internal_templates/groups": { "post": { "tags": ["_internal_templates"], @@ -10838,7 +11091,7 @@ "tags": ["runs"], "summary": "List Runs", "description": "List all runs.", - "operationId": "list_runs", + "operationId": "list_runs1", "parameters": [ { "name": "agent_id", @@ -34151,6 +34404,18 @@ "title": "Agent Id", "description": "The unique identifier of the agent associated with the run." }, + "base_template_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Base Template Id", + "description": "The base template ID that the run belongs to." + }, "background": { "anyOf": [ { @@ -34276,7 +34541,7 @@ "type": "object", "required": ["agent_id"], "title": "Run", - "description": "Representation of a run - a conversation or processing session for an agent.\nRuns track when agents process messages and maintain the relationship between agents, steps, and messages.\n\nParameters:\n id (str): The unique identifier of the run (prefixed with 'run-').\n status (JobStatus): The current status of the run.\n created_at (datetime): The timestamp when the run was created.\n completed_at (datetime): The timestamp when the run was completed.\n agent_id (str): The unique identifier of the agent associated with the run.\n stop_reason (StopReasonType): The reason why the run was stopped.\n background (bool): Whether the run was created in background mode.\n metadata (dict): Additional metadata for the run.\n request_config (LettaRequestConfig): The request configuration for the run." + "description": "Representation of a run - a conversation or processing session for an agent.\nRuns track when agents process messages and maintain the relationship between agents, steps, and messages.\n\nParameters:\n id (str): The unique identifier of the run (prefixed with 'run-').\n status (JobStatus): The current status of the run.\n created_at (datetime): The timestamp when the run was created.\n completed_at (datetime): The timestamp when the run was completed.\n agent_id (str): The unique identifier of the agent associated with the run.\n base_template_id (str): The base template ID that the run belongs to.\n stop_reason (StopReasonType): The reason why the run was stopped.\n background (bool): Whether the run was created in background mode.\n metadata (dict): Additional metadata for the run.\n request_config (LettaRequestConfig): The request configuration for the run." }, "RunMetrics": { "properties": { diff --git a/letta/schemas/run.py b/letta/schemas/run.py index c029281f..8b3bf6fb 100644 --- a/letta/schemas/run.py +++ b/letta/schemas/run.py @@ -25,6 +25,7 @@ class Run(RunBase): created_at (datetime): The timestamp when the run was created. completed_at (datetime): The timestamp when the run was completed. agent_id (str): The unique identifier of the agent associated with the run. + base_template_id (str): The base template ID that the run belongs to. stop_reason (StopReasonType): The reason why the run was stopped. background (bool): Whether the run was created in background mode. metadata (dict): Additional metadata for the run. @@ -41,6 +42,9 @@ class Run(RunBase): # Agent relationship agent_id: str = Field(..., description="The unique identifier of the agent associated with the run.") + # Template fields + base_template_id: Optional[str] = Field(None, description="The base template ID that the run belongs to.") + # Run configuration background: Optional[bool] = Field(None, description="Whether the run was created in background mode.") metadata: Optional[dict] = Field(None, validation_alias="metadata_", description="Additional metadata for the run.") diff --git a/letta/server/rest_api/routers/v1/__init__.py b/letta/server/rest_api/routers/v1/__init__.py index 6935faf8..8568485c 100644 --- a/letta/server/rest_api/routers/v1/__init__.py +++ b/letta/server/rest_api/routers/v1/__init__.py @@ -7,6 +7,7 @@ from letta.server.rest_api.routers.v1.folders import router as folders_router from letta.server.rest_api.routers.v1.groups import router as groups_router from letta.server.rest_api.routers.v1.health import router as health_router from letta.server.rest_api.routers.v1.identities import router as identities_router +from letta.server.rest_api.routers.v1.internal_runs import router as internal_runs_router from letta.server.rest_api.routers.v1.internal_templates import router as internal_templates_router from letta.server.rest_api.routers.v1.jobs import router as jobs_router from letta.server.rest_api.routers.v1.llms import router as llm_router @@ -30,6 +31,7 @@ ROUTERS = [ chat_completions_router, groups_router, identities_router, + internal_runs_router, internal_templates_router, llm_router, blocks_router, diff --git a/letta/server/rest_api/routers/v1/internal_runs.py b/letta/server/rest_api/routers/v1/internal_runs.py new file mode 100644 index 00000000..545698c4 --- /dev/null +++ b/letta/server/rest_api/routers/v1/internal_runs.py @@ -0,0 +1,98 @@ +from typing import List, Literal, Optional + +from fastapi import APIRouter, Depends, Query + +from letta.schemas.enums import RunStatus +from letta.schemas.letta_stop_reason import StopReasonType +from letta.schemas.run import Run +from letta.server.rest_api.dependencies import HeaderParams, get_headers, get_letta_server +from letta.server.server import SyncServer +from letta.services.run_manager import RunManager + +router = APIRouter(prefix="/_internal_runs", tags=["_internal_runs"]) + + +def convert_statuses_to_enum(statuses: Optional[List[str]]) -> Optional[List[RunStatus]]: + """Convert a list of status strings to RunStatus enum values. + + Args: + statuses: List of status strings or None + + Returns: + List of RunStatus enum values or None if input is None + """ + if statuses is None: + return None + return [RunStatus(status) for status in statuses] + + +@router.get("/", response_model=List[Run], operation_id="list_runs") +async def list_runs( + server: "SyncServer" = Depends(get_letta_server), + agent_id: Optional[str] = Query(None, description="The unique identifier of the agent associated with the run."), + agent_ids: Optional[List[str]] = Query( + None, + description="The unique identifiers of the agents associated with the run. Deprecated in favor of agent_id field.", + deprecated=True, + ), + statuses: Optional[List[str]] = Query(None, description="Filter runs by status. Can specify multiple statuses."), + background: Optional[bool] = Query(None, description="If True, filters for runs that were created in background mode."), + stop_reason: Optional[StopReasonType] = Query(None, description="Filter runs by stop reason."), + template_family: Optional[str] = Query(None, description="Filter runs by template family (base_template_id)."), + before: Optional[str] = Query( + None, description="Run ID cursor for pagination. Returns runs that come before this run ID in the specified sort order" + ), + after: Optional[str] = Query( + None, description="Run ID cursor for pagination. Returns runs that come after this run ID in the specified sort order" + ), + limit: Optional[int] = Query(100, description="Maximum number of runs to return"), + order: Literal["asc", "desc"] = Query( + "desc", description="Sort order for runs by creation time. 'asc' for oldest first, 'desc' for newest first" + ), + order_by: Literal["created_at"] = Query("created_at", description="Field to sort by"), + active: bool = Query(False, description="Filter for active runs."), + ascending: bool = Query( + False, + description="Whether to sort agents oldest to newest (True) or newest to oldest (False, default). Deprecated in favor of order field.", + deprecated=True, + ), + headers: HeaderParams = Depends(get_headers), +): + """ + List all runs. + """ + actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id) + runs_manager = server.run_manager + + # Handle backwards compatibility: if statuses not provided but active=True, filter by active statuses + if statuses is None and active: + statuses = [RunStatus.created, RunStatus.running] + + if agent_id: + # NOTE: we are deprecating agent_ids so this will the primary path soon + agent_ids = [agent_id] + + # Handle backward compatibility: if ascending is explicitly set, use it; otherwise use order + if ascending is not False: + # ascending was explicitly set to True + sort_ascending = ascending + else: + # Use the new order parameter + sort_ascending = order == "asc" + + # Convert string statuses to RunStatus enum + parsed_statuses = convert_statuses_to_enum(statuses) + + runs = await runs_manager.list_runs( + actor=actor, + agent_ids=agent_ids, + statuses=parsed_statuses, + limit=limit, + before=before, + after=after, + ascending=sort_ascending, + stop_reason=stop_reason, + background=background, + template_family=template_family, + ) + return runs diff --git a/letta/services/run_manager.py b/letta/services/run_manager.py index c390462f..fa4671eb 100644 --- a/letta/services/run_manager.py +++ b/letta/services/run_manager.py @@ -108,6 +108,7 @@ class RunManager: ascending: bool = False, stop_reason: Optional[str] = None, background: Optional[bool] = None, + template_family: Optional[str] = None, ) -> List[PydanticRun]: """List runs with filtering options.""" async with db_registry.async_session() as session: @@ -133,6 +134,10 @@ class RunManager: if background is not None: query = query.filter(RunModel.background == background) + # Filter by template_family (base_template_id) + if template_family: + query = query.filter(RunModel.base_template_id == template_family) + # Apply pagination from letta.services.helpers.run_manager_helper import _apply_pagination_async diff --git a/tests/managers/test_run_manager.py b/tests/managers/test_run_manager.py index f64f5ee1..9428233f 100644 --- a/tests/managers/test_run_manager.py +++ b/tests/managers/test_run_manager.py @@ -382,6 +382,19 @@ async def test_list_runs_by_stop_reason(server: SyncServer, sarah_agent, default assert runs[0].id == run.id +@pytest.mark.asyncio +async def test_list_runs_by_base_template_id(server: SyncServer, sarah_agent, default_user): + """Test listing runs by template family.""" + run_data = PydanticRun( + agent_id=sarah_agent.id, + base_template_id="test-template-family", + ) + + await server.run_manager.create_run(pydantic_run=run_data, actor=default_user) + runs = await server.run_manager.list_runs(actor=default_user, template_family="test-template-family") + assert len(runs) == 1 + + async def test_e2e_run_callback(monkeypatch, server: SyncServer, default_user, sarah_agent): """Test that run callbacks are properly dispatched when a run is completed.""" captured = {}