feat: add internal runs route with template_family filtering [LET-5416] (#5543)
* feat: add tool_used field to run_metrics [LET-5419] * change to tool name * use tool ids over names * feat: add internal runs route with template_family filtering * remove import * add auto generated * restrict internal runs * add test, address comments * add docs and auto generated fields * remove unused template mixins * update openapi * add generated
This commit is contained in:
Christina Tong
committed by
Caren Thomas
Caren Thomas
parent
e3f794dac5
commit
44574ec264
@@ -1147,6 +1147,9 @@ paths:
|
||||
/v1/_internal_templates/blocks/batch:
|
||||
post:
|
||||
x-fern-ignore: true
|
||||
/v1/_internal_runs/:
|
||||
get:
|
||||
x-fern-ignore: true
|
||||
/v1/projects:
|
||||
get:
|
||||
x-fern-sdk-group-name:
|
||||
|
||||
@@ -3899,14 +3899,8 @@
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 42,
|
||||
"maxLength": 42,
|
||||
"pattern": "^agent-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
|
||||
"description": "The ID of the agent in the format 'agent-<uuid4>'",
|
||||
"examples": ["agent-123e4567-e89b-42d3-8456-426614174000"],
|
||||
"title": "Agent Id"
|
||||
},
|
||||
"description": "The ID of the agent in the format 'agent-<uuid4>'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "max_steps",
|
||||
@@ -8880,6 +8874,265 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/_internal_runs/": {
|
||||
"get": {
|
||||
"tags": ["_internal_runs"],
|
||||
"summary": "List Runs",
|
||||
"description": "List all runs.",
|
||||
"operationId": "list_runs",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "agent_id",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "The unique identifier of the agent associated with the run.",
|
||||
"title": "Agent Id"
|
||||
},
|
||||
"description": "The unique identifier of the agent associated with the run."
|
||||
},
|
||||
{
|
||||
"name": "agent_ids",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "The unique identifiers of the agents associated with the run. Deprecated in favor of agent_id field.",
|
||||
"deprecated": true,
|
||||
"title": "Agent Ids"
|
||||
},
|
||||
"description": "The unique identifiers of the agents associated with the run. Deprecated in favor of agent_id field.",
|
||||
"deprecated": true
|
||||
},
|
||||
{
|
||||
"name": "statuses",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Filter runs by status. Can specify multiple statuses.",
|
||||
"title": "Statuses"
|
||||
},
|
||||
"description": "Filter runs by status. Can specify multiple statuses."
|
||||
},
|
||||
{
|
||||
"name": "background",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "If True, filters for runs that were created in background mode.",
|
||||
"title": "Background"
|
||||
},
|
||||
"description": "If True, filters for runs that were created in background mode."
|
||||
},
|
||||
{
|
||||
"name": "stop_reason",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/StopReasonType"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Filter runs by stop reason.",
|
||||
"title": "Stop Reason"
|
||||
},
|
||||
"description": "Filter runs by stop reason."
|
||||
},
|
||||
{
|
||||
"name": "template_family",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Filter runs by template family (base_template_id).",
|
||||
"title": "Template Family"
|
||||
},
|
||||
"description": "Filter runs by template family (base_template_id)."
|
||||
},
|
||||
{
|
||||
"name": "before",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Run ID cursor for pagination. Returns runs that come before this run ID in the specified sort order",
|
||||
"title": "Before"
|
||||
},
|
||||
"description": "Run ID cursor for pagination. Returns runs that come before this run ID in the specified sort order"
|
||||
},
|
||||
{
|
||||
"name": "after",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Run ID cursor for pagination. Returns runs that come after this run ID in the specified sort order",
|
||||
"title": "After"
|
||||
},
|
||||
"description": "Run ID cursor for pagination. Returns runs that come after this run ID in the specified sort order"
|
||||
},
|
||||
{
|
||||
"name": "limit",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Maximum number of runs to return",
|
||||
"default": 100,
|
||||
"title": "Limit"
|
||||
},
|
||||
"description": "Maximum number of runs to return"
|
||||
},
|
||||
{
|
||||
"name": "order",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": ["asc", "desc"],
|
||||
"type": "string",
|
||||
"description": "Sort order for runs by creation time. 'asc' for oldest first, 'desc' for newest first",
|
||||
"default": "desc",
|
||||
"title": "Order"
|
||||
},
|
||||
"description": "Sort order for runs by creation time. 'asc' for oldest first, 'desc' for newest first"
|
||||
},
|
||||
{
|
||||
"name": "order_by",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"const": "created_at",
|
||||
"type": "string",
|
||||
"description": "Field to sort by",
|
||||
"default": "created_at",
|
||||
"title": "Order By"
|
||||
},
|
||||
"description": "Field to sort by"
|
||||
},
|
||||
{
|
||||
"name": "active",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "boolean",
|
||||
"description": "Filter for active runs.",
|
||||
"default": false,
|
||||
"title": "Active"
|
||||
},
|
||||
"description": "Filter for active runs."
|
||||
},
|
||||
{
|
||||
"name": "ascending",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "boolean",
|
||||
"description": "Whether to sort agents oldest to newest (True) or newest to oldest (False, default). Deprecated in favor of order field.",
|
||||
"deprecated": true,
|
||||
"default": false,
|
||||
"title": "Ascending"
|
||||
},
|
||||
"description": "Whether to sort agents oldest to newest (True) or newest to oldest (False, default). Deprecated in favor of order field.",
|
||||
"deprecated": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Successful Response",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/Run"
|
||||
},
|
||||
"title": "Response List Runs"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"422": {
|
||||
"description": "Validation Error",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/_internal_templates/groups": {
|
||||
"post": {
|
||||
"tags": ["_internal_templates"],
|
||||
@@ -10838,7 +11091,7 @@
|
||||
"tags": ["runs"],
|
||||
"summary": "List Runs",
|
||||
"description": "List all runs.",
|
||||
"operationId": "list_runs",
|
||||
"operationId": "list_runs1",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "agent_id",
|
||||
@@ -34151,6 +34404,18 @@
|
||||
"title": "Agent Id",
|
||||
"description": "The unique identifier of the agent associated with the run."
|
||||
},
|
||||
"base_template_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Base Template Id",
|
||||
"description": "The base template ID that the run belongs to."
|
||||
},
|
||||
"background": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -34276,7 +34541,7 @@
|
||||
"type": "object",
|
||||
"required": ["agent_id"],
|
||||
"title": "Run",
|
||||
"description": "Representation of a run - a conversation or processing session for an agent.\nRuns track when agents process messages and maintain the relationship between agents, steps, and messages.\n\nParameters:\n id (str): The unique identifier of the run (prefixed with 'run-').\n status (JobStatus): The current status of the run.\n created_at (datetime): The timestamp when the run was created.\n completed_at (datetime): The timestamp when the run was completed.\n agent_id (str): The unique identifier of the agent associated with the run.\n stop_reason (StopReasonType): The reason why the run was stopped.\n background (bool): Whether the run was created in background mode.\n metadata (dict): Additional metadata for the run.\n request_config (LettaRequestConfig): The request configuration for the run."
|
||||
"description": "Representation of a run - a conversation or processing session for an agent.\nRuns track when agents process messages and maintain the relationship between agents, steps, and messages.\n\nParameters:\n id (str): The unique identifier of the run (prefixed with 'run-').\n status (JobStatus): The current status of the run.\n created_at (datetime): The timestamp when the run was created.\n completed_at (datetime): The timestamp when the run was completed.\n agent_id (str): The unique identifier of the agent associated with the run.\n base_template_id (str): The base template ID that the run belongs to.\n stop_reason (StopReasonType): The reason why the run was stopped.\n background (bool): Whether the run was created in background mode.\n metadata (dict): Additional metadata for the run.\n request_config (LettaRequestConfig): The request configuration for the run."
|
||||
},
|
||||
"RunMetrics": {
|
||||
"properties": {
|
||||
|
||||
@@ -25,6 +25,7 @@ class Run(RunBase):
|
||||
created_at (datetime): The timestamp when the run was created.
|
||||
completed_at (datetime): The timestamp when the run was completed.
|
||||
agent_id (str): The unique identifier of the agent associated with the run.
|
||||
base_template_id (str): The base template ID that the run belongs to.
|
||||
stop_reason (StopReasonType): The reason why the run was stopped.
|
||||
background (bool): Whether the run was created in background mode.
|
||||
metadata (dict): Additional metadata for the run.
|
||||
@@ -41,6 +42,9 @@ class Run(RunBase):
|
||||
# Agent relationship
|
||||
agent_id: str = Field(..., description="The unique identifier of the agent associated with the run.")
|
||||
|
||||
# Template fields
|
||||
base_template_id: Optional[str] = Field(None, description="The base template ID that the run belongs to.")
|
||||
|
||||
# Run configuration
|
||||
background: Optional[bool] = Field(None, description="Whether the run was created in background mode.")
|
||||
metadata: Optional[dict] = Field(None, validation_alias="metadata_", description="Additional metadata for the run.")
|
||||
|
||||
@@ -7,6 +7,7 @@ from letta.server.rest_api.routers.v1.folders import router as folders_router
|
||||
from letta.server.rest_api.routers.v1.groups import router as groups_router
|
||||
from letta.server.rest_api.routers.v1.health import router as health_router
|
||||
from letta.server.rest_api.routers.v1.identities import router as identities_router
|
||||
from letta.server.rest_api.routers.v1.internal_runs import router as internal_runs_router
|
||||
from letta.server.rest_api.routers.v1.internal_templates import router as internal_templates_router
|
||||
from letta.server.rest_api.routers.v1.jobs import router as jobs_router
|
||||
from letta.server.rest_api.routers.v1.llms import router as llm_router
|
||||
@@ -30,6 +31,7 @@ ROUTERS = [
|
||||
chat_completions_router,
|
||||
groups_router,
|
||||
identities_router,
|
||||
internal_runs_router,
|
||||
internal_templates_router,
|
||||
llm_router,
|
||||
blocks_router,
|
||||
|
||||
98
letta/server/rest_api/routers/v1/internal_runs.py
Normal file
98
letta/server/rest_api/routers/v1/internal_runs.py
Normal file
@@ -0,0 +1,98 @@
|
||||
from typing import List, Literal, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from letta.schemas.enums import RunStatus
|
||||
from letta.schemas.letta_stop_reason import StopReasonType
|
||||
from letta.schemas.run import Run
|
||||
from letta.server.rest_api.dependencies import HeaderParams, get_headers, get_letta_server
|
||||
from letta.server.server import SyncServer
|
||||
from letta.services.run_manager import RunManager
|
||||
|
||||
router = APIRouter(prefix="/_internal_runs", tags=["_internal_runs"])
|
||||
|
||||
|
||||
def convert_statuses_to_enum(statuses: Optional[List[str]]) -> Optional[List[RunStatus]]:
|
||||
"""Convert a list of status strings to RunStatus enum values.
|
||||
|
||||
Args:
|
||||
statuses: List of status strings or None
|
||||
|
||||
Returns:
|
||||
List of RunStatus enum values or None if input is None
|
||||
"""
|
||||
if statuses is None:
|
||||
return None
|
||||
return [RunStatus(status) for status in statuses]
|
||||
|
||||
|
||||
@router.get("/", response_model=List[Run], operation_id="list_runs")
|
||||
async def list_runs(
|
||||
server: "SyncServer" = Depends(get_letta_server),
|
||||
agent_id: Optional[str] = Query(None, description="The unique identifier of the agent associated with the run."),
|
||||
agent_ids: Optional[List[str]] = Query(
|
||||
None,
|
||||
description="The unique identifiers of the agents associated with the run. Deprecated in favor of agent_id field.",
|
||||
deprecated=True,
|
||||
),
|
||||
statuses: Optional[List[str]] = Query(None, description="Filter runs by status. Can specify multiple statuses."),
|
||||
background: Optional[bool] = Query(None, description="If True, filters for runs that were created in background mode."),
|
||||
stop_reason: Optional[StopReasonType] = Query(None, description="Filter runs by stop reason."),
|
||||
template_family: Optional[str] = Query(None, description="Filter runs by template family (base_template_id)."),
|
||||
before: Optional[str] = Query(
|
||||
None, description="Run ID cursor for pagination. Returns runs that come before this run ID in the specified sort order"
|
||||
),
|
||||
after: Optional[str] = Query(
|
||||
None, description="Run ID cursor for pagination. Returns runs that come after this run ID in the specified sort order"
|
||||
),
|
||||
limit: Optional[int] = Query(100, description="Maximum number of runs to return"),
|
||||
order: Literal["asc", "desc"] = Query(
|
||||
"desc", description="Sort order for runs by creation time. 'asc' for oldest first, 'desc' for newest first"
|
||||
),
|
||||
order_by: Literal["created_at"] = Query("created_at", description="Field to sort by"),
|
||||
active: bool = Query(False, description="Filter for active runs."),
|
||||
ascending: bool = Query(
|
||||
False,
|
||||
description="Whether to sort agents oldest to newest (True) or newest to oldest (False, default). Deprecated in favor of order field.",
|
||||
deprecated=True,
|
||||
),
|
||||
headers: HeaderParams = Depends(get_headers),
|
||||
):
|
||||
"""
|
||||
List all runs.
|
||||
"""
|
||||
actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)
|
||||
runs_manager = server.run_manager
|
||||
|
||||
# Handle backwards compatibility: if statuses not provided but active=True, filter by active statuses
|
||||
if statuses is None and active:
|
||||
statuses = [RunStatus.created, RunStatus.running]
|
||||
|
||||
if agent_id:
|
||||
# NOTE: we are deprecating agent_ids so this will the primary path soon
|
||||
agent_ids = [agent_id]
|
||||
|
||||
# Handle backward compatibility: if ascending is explicitly set, use it; otherwise use order
|
||||
if ascending is not False:
|
||||
# ascending was explicitly set to True
|
||||
sort_ascending = ascending
|
||||
else:
|
||||
# Use the new order parameter
|
||||
sort_ascending = order == "asc"
|
||||
|
||||
# Convert string statuses to RunStatus enum
|
||||
parsed_statuses = convert_statuses_to_enum(statuses)
|
||||
|
||||
runs = await runs_manager.list_runs(
|
||||
actor=actor,
|
||||
agent_ids=agent_ids,
|
||||
statuses=parsed_statuses,
|
||||
limit=limit,
|
||||
before=before,
|
||||
after=after,
|
||||
ascending=sort_ascending,
|
||||
stop_reason=stop_reason,
|
||||
background=background,
|
||||
template_family=template_family,
|
||||
)
|
||||
return runs
|
||||
@@ -108,6 +108,7 @@ class RunManager:
|
||||
ascending: bool = False,
|
||||
stop_reason: Optional[str] = None,
|
||||
background: Optional[bool] = None,
|
||||
template_family: Optional[str] = None,
|
||||
) -> List[PydanticRun]:
|
||||
"""List runs with filtering options."""
|
||||
async with db_registry.async_session() as session:
|
||||
@@ -133,6 +134,10 @@ class RunManager:
|
||||
if background is not None:
|
||||
query = query.filter(RunModel.background == background)
|
||||
|
||||
# Filter by template_family (base_template_id)
|
||||
if template_family:
|
||||
query = query.filter(RunModel.base_template_id == template_family)
|
||||
|
||||
# Apply pagination
|
||||
from letta.services.helpers.run_manager_helper import _apply_pagination_async
|
||||
|
||||
|
||||
@@ -382,6 +382,19 @@ async def test_list_runs_by_stop_reason(server: SyncServer, sarah_agent, default
|
||||
assert runs[0].id == run.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_runs_by_base_template_id(server: SyncServer, sarah_agent, default_user):
|
||||
"""Test listing runs by template family."""
|
||||
run_data = PydanticRun(
|
||||
agent_id=sarah_agent.id,
|
||||
base_template_id="test-template-family",
|
||||
)
|
||||
|
||||
await server.run_manager.create_run(pydantic_run=run_data, actor=default_user)
|
||||
runs = await server.run_manager.list_runs(actor=default_user, template_family="test-template-family")
|
||||
assert len(runs) == 1
|
||||
|
||||
|
||||
async def test_e2e_run_callback(monkeypatch, server: SyncServer, default_user, sarah_agent):
|
||||
"""Test that run callbacks are properly dispatched when a run is completed."""
|
||||
captured = {}
|
||||
|
||||
Reference in New Issue
Block a user