feat: Add testing for dry run agent loop (#3265)

letta/server/rest_api/routers/v1/agents.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import traceback
 from datetime import datetime, timezone
-from typing import Annotated, Any, List, Optional
+from typing import Annotated, Any, Dict, List, Optional, Union
 
 from fastapi import APIRouter, Body, Depends, File, Header, HTTPException, Query, Request, UploadFile, status
 from fastapi.responses import JSONResponse
@@ -1162,12 +1162,12 @@ async def list_agent_groups(
 
 @router.post(
     "/{agent_id}/messages/preview-raw-payload",
-    response_model=dict,
+    response_model=Dict[str, Any],
     operation_id="preview_raw_payload",
 )
 async def preview_raw_payload(
     agent_id: str,
-    request: LettaRequest = Body(...),
+    request: Union[LettaRequest, LettaStreamingRequest] = Body(...),
     server: SyncServer = Depends(get_letta_server),
     actor_id: str | None = Header(None, alias="user_id"),
 ):
@@ -1214,6 +1214,7 @@ async def preview_raw_payload(
             include_return_message_types=request.include_return_message_types,
             dry_run=True,
         )
+
     else:
         raise HTTPException(
             status_code=status.HTTP_403_FORBIDDEN,

pyproject.toml

@@ -72,7 +72,7 @@ llama-index = "^0.12.2"
 llama-index-embeddings-openai = "^0.3.1"
 e2b-code-interpreter = {version = "^1.0.3", optional = true}
 anthropic = "^0.49.0"
-letta_client = "^0.1.183"
+letta_client = "^0.1.196"
 openai = "^1.60.0"
 opentelemetry-api = "1.30.0"
 opentelemetry-sdk = "1.30.0"

tests/test_sdk_client.py

@@ -9,7 +9,7 @@ import pytest
 from dotenv import load_dotenv
 from letta_client import CreateBlock
 from letta_client import Letta as LettaSDKClient
-from letta_client import MessageCreate
+from letta_client import MessageCreate, TextContent
 from letta_client.client import BaseTool
 from letta_client.core import ApiError
 from letta_client.types import AgentState, ToolReturnMessage
@@ -967,3 +967,67 @@ def test_create_tool_from_function_with_docstring(e2b_sandbox_mode, client: Lett
     assert "roll_dice" in tool_names
 
     client.tools.delete(tool.id)
+
+
+def test_preview_payload(client: LettaSDKClient, agent):
+    payload = client.agents.messages.preview_raw_payload(
+        agent_id=agent.id,
+        messages=[
+            MessageCreate(
+                role="user",
+                content=[
+                    TextContent(
+                        text="text",
+                    )
+                ],
+            )
+        ],
+    )
+
+    assert isinstance(payload, dict)
+    assert "model" in payload
+    assert "messages" in payload
+    assert "tools" in payload
+    assert "frequency_penalty" in payload
+    assert "max_completion_tokens" in payload
+    assert "temperature" in payload
+    assert "user" in payload
+    assert "parallel_tool_calls" in payload
+    assert "tool_choice" in payload
+
+    assert payload["model"] == "gpt-4o-mini"
+
+    assert isinstance(payload["messages"], list)
+    assert len(payload["messages"]) >= 3
+
+    system_message = payload["messages"][0]
+    assert system_message["role"] == "system"
+    assert "base_instructions" in system_message["content"]
+    assert "memory_blocks" in system_message["content"]
+    assert "tool_usage_rules" in system_message["content"]
+    assert "Letta" in system_message["content"]
+
+    assert isinstance(payload["tools"], list)
+    assert len(payload["tools"]) > 0
+
+    tool_names = [tool["function"]["name"] for tool in payload["tools"]]
+    expected_tools = ["send_message", "conversation_search", "core_memory_replace", "core_memory_append"]
+    for tool_name in expected_tools:
+        assert tool_name in tool_names, f"Expected tool {tool_name} not found in tools"
+
+    for tool in payload["tools"]:
+        assert tool["type"] == "function"
+        assert "function" in tool
+        assert "name" in tool["function"]
+        assert "description" in tool["function"]
+        assert "parameters" in tool["function"]
+        assert tool["function"]["strict"] is True
+
+    assert payload["frequency_penalty"] == 1.0
+    assert payload["max_completion_tokens"] == 4096
+    assert payload["temperature"] == 0.7
+    assert payload["parallel_tool_calls"] is False
+    assert payload["tool_choice"] == "required"
+    assert payload["user"].startswith("user-")
+
+    print(payload)