From e03fb59d111d0fd4ffbe9f9222bcf11ea187b33b Mon Sep 17 00:00:00 2001 From: Kian Jones Date: Fri, 20 Jun 2025 16:42:25 -0700 Subject: [PATCH] final touches --- .../generate_model_sweep_markdown.py | 4 +- .../scripts/model-sweep/supported-models.mdx | 4930 ++++++++--------- 2 files changed, 2467 insertions(+), 2467 deletions(-) diff --git a/.github/scripts/model-sweep/generate_model_sweep_markdown.py b/.github/scripts/model-sweep/generate_model_sweep_markdown.py index 9d965b57..2b63ec13 100644 --- a/.github/scripts/model-sweep/generate_model_sweep_markdown.py +++ b/.github/scripts/model-sweep/generate_model_sweep_markdown.py @@ -145,8 +145,8 @@ def generate_test_details(model_info, feature_mapping): test_file_path = os.path.join(script_dir, 'model_sweep.py') test_line_numbers = get_test_function_line_numbers(test_file_path) - # Use the specific GitHub URL for the branch - base_github_url = "https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py" + # Use the main branch GitHub URL + base_github_url = "https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py" for feature, tests in model_info['categorized_tests'].items(): if not tests: diff --git a/.github/scripts/model-sweep/supported-models.mdx b/.github/scripts/model-sweep/supported-models.mdx index 7d43ed2a..94d8e25a 100644 --- a/.github/scripts/model-sweep/supported-models.mdx +++ b/.github/scripts/model-sweep/supported-models.mdx @@ -1,6 +1,6 @@ --- title: Support Models -generated: 2025-06-20T16:32:55.429847 +generated: 2025-06-20T16:40:44.072054 --- # Supported Models @@ -219,252 +219,252 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### claude-3-5-sonnet-20241022 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### claude-3-7-sonnet-20250219 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### claude-sonnet-4-20250514 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### claude-opus-4-20250514 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### claude-3-5-sonnet-20240620 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### claude-3-haiku-20240307 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### claude-3-opus-20240229 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### claude-3-sonnet-20240229 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) @@ -475,812 +475,812 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4.1-2025-04-14 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4.1-nano-2025-04-14 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4o ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4o-2024-05-13 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4-turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4.1-mini ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4.5-preview ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4.5-preview-2025-02-27 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4o-2024-08-06 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4-0613 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4-1106-preview ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4-turbo-2024-04-09 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4.1-mini-2025-04-14 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4.1-nano ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4o-2024-11-20 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4-turbo-preview ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4-0125-preview ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4o-mini ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4o-mini-2024-07-18 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gpt-4 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### o1 ### Basic -- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### o1-2024-12-17 ### Basic -- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### o3 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### o3-2025-04-16 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### o3-mini ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### o3-mini-2025-01-31 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### o3-pro ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### o3-pro-2025-06-10 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) @@ -1291,980 +1291,980 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.5-pro-002 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.5-pro-latest ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-flash-preview-04-17-thinking ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-pro-preview-03-25 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-pro-preview-05-06 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-flash-preview-05-20 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-thinking-exp ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-thinking-exp-1219 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-thinking-exp-01-21 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-flash-preview-04-17 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-pro-preview-06-05 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.0-pro-vision-latest ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.5-flash ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.5-flash-002 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.5-flash-8b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.5-flash-8b-001 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.5-flash-8b-latest ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-1.5-flash-latest ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-001 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-exp ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-exp-image-generation ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-lite ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-lite-001 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-lite-preview ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-lite-preview-02-05 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-flash-preview-image-generation ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-pro-exp ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.0-pro-exp-02-05 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-flash-preview-tts ### Basic -- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-pro-exp-03-25 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-2.5-pro-preview-tts ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-exp-1206 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemini-pro-vision ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) @@ -2275,28 +2275,28 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) @@ -2307,1540 +2307,1540 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### arcee-ai/virtuoso-large ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### Qwen/QwQ-32B ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### Qwen/Qwen2.5-7B-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### Qwen/Qwen2.5-Coder-32B-Instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### arcee-ai/coder-large ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### arcee_ai/arcee-spotlight ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-3.2-3B-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-3.3-70B-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### arcee-ai/virtuoso-medium-v2 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### Qwen/Qwen3-235B-A22B-fp8-tput ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-ai/DeepSeek-V3 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-4-Scout-17B-16E-Instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### mistralai/Mixtral-8x7B-Instruct-v0.1 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### arcee-ai/caller ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### mistralai/Mistral-Small-24B-Instruct-2501 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### Qwen/Qwen2-72B-Instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### Qwen/Qwen2-VL-72B-Instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### Qwen/Qwen2.5-VL-72B-Instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### arcee-ai/arcee-blitz ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### arcee-ai/maestro-reasoning ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-ai/DeepSeek-R1 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-ai/DeepSeek-R1-Distill-Llama-70B ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-ai/DeepSeek-V3-p-dp ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### google/gemma-2-27b-it ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### lgai/exaone-3-5-32b-instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### lgai/exaone-deep-32b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### marin-community/marin-8b-instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-3-70b-chat-hf ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-3-8b-chat-hf ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Llama-Vision-Free ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Meta-Llama-3-70B-Instruct-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/Meta-Llama-3-8B-Instruct-Lite ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### mistralai/Mistral-7B-Instruct-v0.1 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### mistralai/Mistral-7B-Instruct-v0.2 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### mistralai/Mistral-7B-Instruct-v0.3 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### perplexity-ai/r1-1776 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### scb10x/scb10x-llama3-1-typhoon2-70b-instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### scb10x/scb10x-typhoon-2-1-gemma3-12b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### togethercomputer/MoA-1 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### togethercomputer/MoA-1-Turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### togethercomputer/Refuel-Llm-V2 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### togethercomputer/Refuel-Llm-V2-Small ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) @@ -3851,56 +3851,56 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-reasoner ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) @@ -3911,643 +3911,643 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### compound-beta ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### compound-beta-mini ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### deepseek-r1-distill-llama-70b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### distil-whisper-large-v3-en ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### gemma2-9b-it ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### llama-3.1-8b-instant ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### llama-3.3-70b-versatile ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### llama-guard-3-8b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### llama3-70b-8192 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### llama3-8b-8192 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/llama-4-maverick-17b-128e-instruct ### Basic -- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/llama-4-scout-17b-16e-instruct ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/llama-guard-4-12b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/llama-prompt-guard-2-22m ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### meta-llama/llama-prompt-guard-2-86m ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### mistral-saba-24b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### playai-tts ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### playai-tts-arabic ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### qwen-qwq-32b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### qwen/qwen3-32b ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### whisper-large-v3 ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417) ### whisper-large-v3-turbo ### Basic -- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L463) -- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L704) -- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L731) -- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L345) -- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L368) -- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L566) -- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L488) -- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L513) -- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L539) -- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L392) +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392) ### Token Streaming -- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L674) -- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L593) -- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L619) -- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L646) +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646) ### Multimodal -- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) -- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)