diff --git a/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py b/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py new file mode 100644 index 00000000..0a948ab0 --- /dev/null +++ b/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py @@ -0,0 +1,31 @@ +"""Add tools_used field to run_metrics table + +Revision ID: 6756d04c3ddb +Revises: e67961ed7c32 +Create Date: 2025-10-17 14:52:53.601368 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "6756d04c3ddb" +down_revision: Union[str, None] = "e67961ed7c32" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("run_metrics", sa.Column("tools_used", sa.JSON(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("run_metrics", "tools_used") + # ### end Alembic commands ### diff --git a/fern/openapi.json b/fern/openapi.json index 912bd408..0e2258ed 100644 --- a/fern/openapi.json +++ b/fern/openapi.json @@ -33932,6 +33932,21 @@ "title": "Num Steps", "description": "The number of steps in the run." }, + "tools_used": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools Used", + "description": "List of tool IDs that were used in this run." + }, "template_id": { "anyOf": [ { diff --git a/letta/orm/run_metrics.py b/letta/orm/run_metrics.py index 71e9aa84..22c5d8e7 100644 --- a/letta/orm/run_metrics.py +++ b/letta/orm/run_metrics.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, List, Optional -from sqlalchemy import BigInteger, ForeignKey, Integer, String +from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, String from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, Session, mapped_column, relationship @@ -43,6 +43,11 @@ class RunMetrics(SqlalchemyBase, ProjectMixin, AgentMixin, OrganizationMixin, Te nullable=True, doc="The number of steps in the run", ) + tools_used: Mapped[Optional[List[str]]] = mapped_column( + JSON, + nullable=True, + doc="List of tool IDs that were used in this run", + ) run: Mapped[Optional["Run"]] = relationship("Run", foreign_keys=[id]) agent: Mapped[Optional["Agent"]] = relationship("Agent") diff --git a/letta/schemas/run_metrics.py b/letta/schemas/run_metrics.py index 40971ab6..458a1cad 100644 --- a/letta/schemas/run_metrics.py +++ b/letta/schemas/run_metrics.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional from pydantic import Field @@ -17,5 +17,6 @@ class RunMetrics(RunMetricsBase): run_start_ns: Optional[int] = Field(None, description="The timestamp of the start of the run in nanoseconds.") run_ns: Optional[int] = Field(None, description="Total time for the run in nanoseconds.") num_steps: Optional[int] = Field(None, description="The number of steps in the run.") + tools_used: Optional[List[str]] = Field(None, description="List of tool IDs that were used in this run.") template_id: Optional[str] = Field(None, description="The template ID that the run belongs to (cloud only).") base_template_id: Optional[str] = Field(None, description="The base template ID that the run belongs to (cloud only).") diff --git a/letta/services/run_manager.py b/letta/services/run_manager.py index 75871d8e..c390462f 100644 --- a/letta/services/run_manager.py +++ b/letta/services/run_manager.py @@ -203,6 +203,22 @@ class RunManager: # update run metrics table num_steps = len(await self.step_manager.list_steps_async(run_id=run_id, actor=actor)) + + # Collect tools used from run messages + tools_used = set() + messages = await self.message_manager.list_messages(actor=actor, run_id=run_id) + for message in messages: + if message.tool_calls: + for tool_call in message.tool_calls: + if hasattr(tool_call, "function") and hasattr(tool_call.function, "name"): + # Get tool ID from tool name + from letta.services.tool_manager import ToolManager + + tool_manager = ToolManager() + tool_id = await tool_manager.get_tool_id_by_name_async(tool_call.function.name, actor) + if tool_id: + tools_used.add(tool_id) + async with db_registry.async_session() as session: metrics = await RunMetricsModel.read_async(db_session=session, identifier=run_id, actor=actor) # Calculate runtime if run is completing @@ -217,6 +233,7 @@ class RunManager: current_ns = int(time.time() * 1e9) metrics.run_ns = current_ns - metrics.run_start_ns metrics.num_steps = num_steps + metrics.tools_used = list(tools_used) if tools_used else None await metrics.update_async(db_session=session, actor=actor, no_commit=True, no_refresh=True) await session.commit()