feat: add tool_used field to run_metrics [LET-5419] (#5540)

* feat: add tool_used field to run_metrics [LET-5419]

* change to tool name

* use tool ids over names

* add generated files

* chore: update OpenAPI schema to include tools_used field in RunMetrics

* update alembic file

alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py

@@ -0,0 +1,31 @@
+"""Add tools_used field to run_metrics table
+
+Revision ID: 6756d04c3ddb
+Revises: e67961ed7c32
+Create Date: 2025-10-17 14:52:53.601368
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "6756d04c3ddb"
+down_revision: Union[str, None] = "e67961ed7c32"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("run_metrics", sa.Column("tools_used", sa.JSON(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("run_metrics", "tools_used")
+    # ### end Alembic commands ###

fern/openapi.json

@@ -33932,6 +33932,21 @@
             "title": "Num Steps",
             "description": "The number of steps in the run."
           },
+          "tools_used": {
+            "anyOf": [
+              {
+                "items": {
+                  "type": "string"
+                },
+                "type": "array"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Tools Used",
+            "description": "List of tool IDs that were used in this run."
+          },
           "template_id": {
             "anyOf": [
               {

letta/orm/run_metrics.py

@@ -1,7 +1,7 @@
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, List, Optional
 
-from sqlalchemy import BigInteger, ForeignKey, Integer, String
+from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, String
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Mapped, Session, mapped_column, relationship
 
@@ -43,6 +43,11 @@ class RunMetrics(SqlalchemyBase, ProjectMixin, AgentMixin, OrganizationMixin, Te
         nullable=True,
         doc="The number of steps in the run",
     )
+    tools_used: Mapped[Optional[List[str]]] = mapped_column(
+        JSON,
+        nullable=True,
+        doc="List of tool IDs that were used in this run",
+    )
     run: Mapped[Optional["Run"]] = relationship("Run", foreign_keys=[id])
     agent: Mapped[Optional["Agent"]] = relationship("Agent")
 

letta/schemas/run_metrics.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import List, Optional
 
 from pydantic import Field
 
@@ -17,5 +17,6 @@ class RunMetrics(RunMetricsBase):
     run_start_ns: Optional[int] = Field(None, description="The timestamp of the start of the run in nanoseconds.")
     run_ns: Optional[int] = Field(None, description="Total time for the run in nanoseconds.")
     num_steps: Optional[int] = Field(None, description="The number of steps in the run.")
+    tools_used: Optional[List[str]] = Field(None, description="List of tool IDs that were used in this run.")
     template_id: Optional[str] = Field(None, description="The template ID that the run belongs to (cloud only).")
     base_template_id: Optional[str] = Field(None, description="The base template ID that the run belongs to (cloud only).")

letta/services/run_manager.py

@@ -203,6 +203,22 @@ class RunManager:
 
         # update run metrics table
         num_steps = len(await self.step_manager.list_steps_async(run_id=run_id, actor=actor))
+
+        # Collect tools used from run messages
+        tools_used = set()
+        messages = await self.message_manager.list_messages(actor=actor, run_id=run_id)
+        for message in messages:
+            if message.tool_calls:
+                for tool_call in message.tool_calls:
+                    if hasattr(tool_call, "function") and hasattr(tool_call.function, "name"):
+                        # Get tool ID from tool name
+                        from letta.services.tool_manager import ToolManager
+
+                        tool_manager = ToolManager()
+                        tool_id = await tool_manager.get_tool_id_by_name_async(tool_call.function.name, actor)
+                        if tool_id:
+                            tools_used.add(tool_id)
+
         async with db_registry.async_session() as session:
             metrics = await RunMetricsModel.read_async(db_session=session, identifier=run_id, actor=actor)
             # Calculate runtime if run is completing
@@ -217,6 +233,7 @@ class RunManager:
                     current_ns = int(time.time() * 1e9)
                     metrics.run_ns = current_ns - metrics.run_start_ns
             metrics.num_steps = num_steps
+            metrics.tools_used = list(tools_used) if tools_used else None
             await metrics.update_async(db_session=session, actor=actor, no_commit=True, no_refresh=True)
             await session.commit()