fix: multi_agent flaky test (#9314)
* fix(core): handle PermissionDeniedError in provider API key validation Fixed OpenAI PermissionDeniedError being raised as unknown error when validating provider API keys. The check_api_key methods in OpenAI-based providers (OpenAI, OpenRouter, Azure, Together) now properly catch and re-raise PermissionDeniedError as LLMPermissionDeniedError. 🐛 Generated with [Letta Code](https://letta.com) Co-Authored-By: Letta <noreply@letta.com> * fix(core): handle Unicode surrogates in OpenAI requests Sanitize invalid UTF-16 surrogates before sending requests to OpenAI API. Fixes UnicodeEncodeError when message content contains unpaired surrogates from corrupted emoji data or malformed Unicode sequences. 🐾 Generated with [Letta Code](https://letta.com) Co-Authored-By: Letta <noreply@letta.com> * try to fix * revert random stuff * revert some stuff --------- Co-authored-by: Letta <noreply@letta.com>
This commit is contained in:
@@ -135,18 +135,50 @@ def roll_dice_tool(client: Letta):
|
||||
def test_send_message_to_agent(client: Letta, agent_obj: AgentState, other_agent_obj: AgentState):
|
||||
secret_word = "banana"
|
||||
|
||||
# Encourage the agent to send a message to the other agent_obj with the secret string
|
||||
# Send a message to the agent asking it to use the tool
|
||||
response = client.agents.messages.create(
|
||||
agent_id=agent_obj.id,
|
||||
messages=[
|
||||
MessageCreateParam(
|
||||
role="user",
|
||||
content=f"Use your tool to send a message to another agent with id {other_agent_obj.id} to share the secret word: {secret_word}!",
|
||||
content=f"IMPORTANT: You MUST use the send_message_to_agent_and_wait_for_reply tool RIGHT NOW to send a message to agent {other_agent_obj.id}. Include the exact secret word '{secret_word}' in your message. Call the tool immediately.",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
# Get messages from the other agent
|
||||
# FIRST: Verify the sender agent actually called the tool
|
||||
# This catches LLM non-determinism early with a clear error message
|
||||
found_tool_call = False
|
||||
tool_return_message = None
|
||||
target_snippet = f"'agent_id': '{other_agent_obj.id}', 'response': ["
|
||||
|
||||
for m in response.messages:
|
||||
if isinstance(m, ToolReturnMessage):
|
||||
if target_snippet in m.tool_return:
|
||||
found_tool_call = True
|
||||
tool_return_message = m
|
||||
break
|
||||
|
||||
if not found_tool_call:
|
||||
# Print debug info to help diagnose the issue
|
||||
print("\n=== DEBUG: Sender agent did not call the tool ===")
|
||||
print(f"Response messages from agent {agent_obj.id}:")
|
||||
for i, m in enumerate(response.messages):
|
||||
print(f"\nMessage {i} ({type(m).__name__}):")
|
||||
if isinstance(m, ToolReturnMessage):
|
||||
print(f" Tool return: {m.tool_return}")
|
||||
elif hasattr(m, "content"):
|
||||
print(f" Content: {m.content}")
|
||||
else:
|
||||
print(f" {m}")
|
||||
raise AssertionError(
|
||||
f"Sender agent {agent_obj.id} did not call send_message_to_agent_and_wait_for_reply tool. "
|
||||
f"This is likely LLM non-determinism. Check debug output above for what the agent did instead."
|
||||
)
|
||||
|
||||
print(f"\n✓ Tool call verified: {tool_return_message.tool_return[:200]}...")
|
||||
|
||||
# SECOND: Now verify the message arrived at the receiver
|
||||
messages_page = client.agents.messages.list(agent_id=other_agent_obj.id)
|
||||
messages = messages_page.items
|
||||
|
||||
@@ -161,46 +193,6 @@ def test_send_message_to_agent(client: Letta, agent_obj: AgentState, other_agent
|
||||
|
||||
assert found_secret, f"Secret word '{secret_word}' not found in system messages of agent {other_agent_obj.id}"
|
||||
|
||||
# Search the sender agent for the response from another agent
|
||||
in_context_messages_page = client.agents.messages.list(agent_id=agent_obj.id)
|
||||
in_context_messages = in_context_messages_page.items
|
||||
found = False
|
||||
target_snippet = f"'agent_id': '{other_agent_obj.id}', 'response': ["
|
||||
|
||||
for m in in_context_messages:
|
||||
# Check ToolReturnMessage for the response
|
||||
if isinstance(m, ToolReturnMessage):
|
||||
if target_snippet in m.tool_return:
|
||||
found = True
|
||||
break
|
||||
# Handle different message content structures
|
||||
elif hasattr(m, "content"):
|
||||
if isinstance(m.content, list) and len(m.content) > 0:
|
||||
content_text = m.content[0].text if hasattr(m.content[0], "text") else str(m.content[0])
|
||||
else:
|
||||
content_text = str(m.content)
|
||||
|
||||
if target_snippet in content_text:
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
# Print debug info
|
||||
joined = "\n".join(
|
||||
[
|
||||
str(
|
||||
m.content[0].text
|
||||
if hasattr(m, "content") and isinstance(m.content, list) and len(m.content) > 0 and hasattr(m.content[0], "text")
|
||||
else m.content
|
||||
if hasattr(m, "content")
|
||||
else f"<{type(m).__name__}>"
|
||||
)
|
||||
for m in in_context_messages[1:]
|
||||
]
|
||||
)
|
||||
print(f"In context messages of the sender agent (without system):\n\n{joined}")
|
||||
raise Exception(f"Was not able to find an instance of the target snippet: {target_snippet}")
|
||||
|
||||
# Test that the agent can still receive messages fine
|
||||
response = client.agents.messages.create(
|
||||
agent_id=agent_obj.id,
|
||||
|
||||
Reference in New Issue
Block a user