From b0c40b6b1d03f44315465a4e2c1bee7b3bdd8bc5 Mon Sep 17 00:00:00 2001 From: Kian Jones <11655409+kianjones9@users.noreply.github.com> Date: Thu, 5 Feb 2026 18:46:15 -0800 Subject: [PATCH] fix: multi_agent flaky test (#9314) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(core): handle PermissionDeniedError in provider API key validation Fixed OpenAI PermissionDeniedError being raised as unknown error when validating provider API keys. The check_api_key methods in OpenAI-based providers (OpenAI, OpenRouter, Azure, Together) now properly catch and re-raise PermissionDeniedError as LLMPermissionDeniedError. šŸ› Generated with [Letta Code](https://letta.com) Co-Authored-By: Letta * fix(core): handle Unicode surrogates in OpenAI requests Sanitize invalid UTF-16 surrogates before sending requests to OpenAI API. Fixes UnicodeEncodeError when message content contains unpaired surrogates from corrupted emoji data or malformed Unicode sequences. šŸ¾ Generated with [Letta Code](https://letta.com) Co-Authored-By: Letta * try to fix * revert random stuff * revert some stuff --------- Co-authored-by: Letta --- tests/integration_test_multi_agent.py | 78 ++++++++++++--------------- 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/tests/integration_test_multi_agent.py b/tests/integration_test_multi_agent.py index f4aec50f..92fa120c 100644 --- a/tests/integration_test_multi_agent.py +++ b/tests/integration_test_multi_agent.py @@ -135,18 +135,50 @@ def roll_dice_tool(client: Letta): def test_send_message_to_agent(client: Letta, agent_obj: AgentState, other_agent_obj: AgentState): secret_word = "banana" - # Encourage the agent to send a message to the other agent_obj with the secret string + # Send a message to the agent asking it to use the tool response = client.agents.messages.create( agent_id=agent_obj.id, messages=[ MessageCreateParam( role="user", - content=f"Use your tool to send a message to another agent with id {other_agent_obj.id} to share the secret word: {secret_word}!", + content=f"IMPORTANT: You MUST use the send_message_to_agent_and_wait_for_reply tool RIGHT NOW to send a message to agent {other_agent_obj.id}. Include the exact secret word '{secret_word}' in your message. Call the tool immediately.", ) ], ) - # Get messages from the other agent + # FIRST: Verify the sender agent actually called the tool + # This catches LLM non-determinism early with a clear error message + found_tool_call = False + tool_return_message = None + target_snippet = f"'agent_id': '{other_agent_obj.id}', 'response': [" + + for m in response.messages: + if isinstance(m, ToolReturnMessage): + if target_snippet in m.tool_return: + found_tool_call = True + tool_return_message = m + break + + if not found_tool_call: + # Print debug info to help diagnose the issue + print("\n=== DEBUG: Sender agent did not call the tool ===") + print(f"Response messages from agent {agent_obj.id}:") + for i, m in enumerate(response.messages): + print(f"\nMessage {i} ({type(m).__name__}):") + if isinstance(m, ToolReturnMessage): + print(f" Tool return: {m.tool_return}") + elif hasattr(m, "content"): + print(f" Content: {m.content}") + else: + print(f" {m}") + raise AssertionError( + f"Sender agent {agent_obj.id} did not call send_message_to_agent_and_wait_for_reply tool. " + f"This is likely LLM non-determinism. Check debug output above for what the agent did instead." + ) + + print(f"\nāœ“ Tool call verified: {tool_return_message.tool_return[:200]}...") + + # SECOND: Now verify the message arrived at the receiver messages_page = client.agents.messages.list(agent_id=other_agent_obj.id) messages = messages_page.items @@ -161,46 +193,6 @@ def test_send_message_to_agent(client: Letta, agent_obj: AgentState, other_agent assert found_secret, f"Secret word '{secret_word}' not found in system messages of agent {other_agent_obj.id}" - # Search the sender agent for the response from another agent - in_context_messages_page = client.agents.messages.list(agent_id=agent_obj.id) - in_context_messages = in_context_messages_page.items - found = False - target_snippet = f"'agent_id': '{other_agent_obj.id}', 'response': [" - - for m in in_context_messages: - # Check ToolReturnMessage for the response - if isinstance(m, ToolReturnMessage): - if target_snippet in m.tool_return: - found = True - break - # Handle different message content structures - elif hasattr(m, "content"): - if isinstance(m.content, list) and len(m.content) > 0: - content_text = m.content[0].text if hasattr(m.content[0], "text") else str(m.content[0]) - else: - content_text = str(m.content) - - if target_snippet in content_text: - found = True - break - - if not found: - # Print debug info - joined = "\n".join( - [ - str( - m.content[0].text - if hasattr(m, "content") and isinstance(m.content, list) and len(m.content) > 0 and hasattr(m.content[0], "text") - else m.content - if hasattr(m, "content") - else f"<{type(m).__name__}>" - ) - for m in in_context_messages[1:] - ] - ) - print(f"In context messages of the sender agent (without system):\n\n{joined}") - raise Exception(f"Was not able to find an instance of the target snippet: {target_snippet}") - # Test that the agent can still receive messages fine response = client.agents.messages.create( agent_id=agent_obj.id,